mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-27 22:37:13 +00:00
Compare commits
30 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4eba959910 | |||
| e4c2a373d5 | |||
| b33cf840b5 | |||
| 2f8d82ac50 | |||
| 59bc6d7c94 | |||
| a56104206e | |||
| be485e6cab | |||
| fbd94e129d | |||
| cf5a9e0ba3 | |||
| 254592717a | |||
| aba9e2e9e6 | |||
| 924700a24f | |||
| b8eef694c7 | |||
| 26d55bb04e | |||
| c9f7e48a44 | |||
| 11e1662c90 | |||
| 7ea3aa2a93 | |||
| 4e48c390e7 | |||
| 02030f90a3 | |||
| d4d416ed47 | |||
| fa0a056bc1 | |||
| 1880bdca91 | |||
| 1c2f69eba9 | |||
| e193d34444 | |||
| fca714efe8 | |||
| 65bd2d575b | |||
| 58cab30351 | |||
| 5e863badc2 | |||
| b78e2ad6e7 | |||
| 605176fe75 |
@@ -6,7 +6,17 @@
|
||||
- Route every new sync/flush through the durability-mode and bucket override
|
||||
gates; mode `none` must not pay the new fsync.
|
||||
- Keep blocking filesystem/CPU work off async runtime threads, but do not split
|
||||
one small operation into many `spawn_blocking` round trips.
|
||||
one small operation into many `spawn_blocking` round trips; coalesce `open`,
|
||||
`read`/`write`, `fsync`, and `close` into one blocking closure per disk.
|
||||
- Check for per-disk syscall amplification (`disks × parts/files`): pin a
|
||||
directory once (`openat2` on Linux) and reuse the handle across sibling reads
|
||||
instead of re-walking ancestor components per file or re-opening identical
|
||||
`source_parent == destination_parent` directories.
|
||||
- After `fstat`/`file.metadata()`, require `Vec::with_capacity(len + 1)` before
|
||||
`read_to_end` so small metadata reads do not trigger 6–7 exponential-growth
|
||||
`read()` syscalls.
|
||||
- Avoid redundant `store.load()` or `root_missing()` round-trips under the same
|
||||
exclusive lock when the state or original bytes were just loaded or persisted.
|
||||
- Measure lock hold time across I/O and compare acquisition order for ABBA.
|
||||
- Keep cleanup, extra stat/rename, and diagnostics out of the PUT commit critical
|
||||
section when they need not be there.
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-linux=9bd4b5ed4d135c6f61e0d144ddeb6b897fd5a6037b51f31088693cf1b23e0718
|
||||
sha256-darwin=9bd4b5ed4d135c6f61e0d144ddeb6b897fd5a6037b51f31088693cf1b23e0718
|
||||
sha256-linux=6190506b9e4bdb60fd392544a7bc47be708077bb4fe7a2ef689d047455e1af86
|
||||
sha256-darwin=6190506b9e4bdb60fd392544a7bc47be708077bb4fe7a2ef689d047455e1af86
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=da00472db3d8181ed413b8e9919f43e86fe644ccee444d92035ce189a33ab037
|
||||
sha256-linux=42e95c9480e5bfa0d717be6652c6500b43e3a337cb63907869104bb155b5bd91
|
||||
sha256-darwin=a67a347f26be186c598b5beaf68e6bcbe1855350a35e1423dd5b8a81bcba06a5
|
||||
sha256-linux=ad4fabe811c6cca1aba89fad34ccd404b5832a6a955875c2826f6e0fa1ea4f1f
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=347a0d4e5e547d5388703062ce66a66167a845b6e57ec591ccb4c732a567c68d
|
||||
sha256-linux=4f9efa47194279185bb68249e82c788fa147abe944b6c764050e3d47f31e7926
|
||||
sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81
|
||||
sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
{
|
||||
"invariant": "list-uncommitted-version",
|
||||
"suite": "rustfs-filemeta",
|
||||
"name": "metacache::tests::resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge"
|
||||
"name": "metacache::tests::resolve_merge_keeps_the_newest_version_at_listing_quorum"
|
||||
},
|
||||
{
|
||||
"invariant": "minio-object-fixture",
|
||||
|
||||
@@ -52,6 +52,7 @@ script-tests: ## Run shell script tests
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_glibc.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_upgrade_workflow.py
|
||||
|
||||
@@ -68,6 +68,17 @@ command = ['sh', '-c', 'echo RUST_MIN_STACK=33554432 >> "$NEXTEST_ENV"']
|
||||
# boundaries. Reserve the run's capacity so unrelated fixtures cannot starve
|
||||
# their existing deadlines and typed-failure assertions (rustfs/backlog#2706).
|
||||
# Keep each test's internal concurrency and zero-retry policy unchanged.
|
||||
[[profile.default.overrides]]
|
||||
filter = """
|
||||
package(rustfs-ecstore) & (
|
||||
test(/^set_disk::ops::multipart::tests::complete_multipart_(quorum_success_releases_logical_guards_before_physical_tail|waits_for_tail_before_releasing_guards_and_marking_capacity)$/)
|
||||
| test(=store::heal::tests::full_format_heal_lost_after_last_save_does_not_publish_or_renew_and_latches)
|
||||
| test(/^store::init::tests::(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page|dispatch_manifest_shutdown_(drains_admitted_delete_batch_without_tail_admission|stops_aborting_validation_tail_reads|stops_authorized_tail_reads_before_receipt_or_cas))$/)
|
||||
| (binary(metadata_read_lock_reuse_test) & test(=replica_multipart_completes_while_metadata_writer_waits_for_its_snapshot))
|
||||
)
|
||||
"""
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & test(/^connect::diagnostics::schedule::tests::/)'
|
||||
threads-required = "num-test-threads"
|
||||
@@ -351,6 +362,17 @@ path = "junit.xml"
|
||||
|
||||
# Match the local durable-progress reservations. All members, deadlines,
|
||||
# assertions, and internal concurrency remain enabled without retries.
|
||||
[[profile.ci.overrides]]
|
||||
filter = """
|
||||
package(rustfs-ecstore) & (
|
||||
test(/^set_disk::ops::multipart::tests::complete_multipart_(quorum_success_releases_logical_guards_before_physical_tail|waits_for_tail_before_releasing_guards_and_marking_capacity)$/)
|
||||
| test(=store::heal::tests::full_format_heal_lost_after_last_save_does_not_publish_or_renew_and_latches)
|
||||
| test(/^store::init::tests::(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page|dispatch_manifest_shutdown_(drains_admitted_delete_batch_without_tail_admission|stops_aborting_validation_tail_reads|stops_authorized_tail_reads_before_receipt_or_cas))$/)
|
||||
| (binary(metadata_read_lock_reuse_test) & test(=replica_multipart_completes_while_metadata_writer_waits_for_its_snapshot))
|
||||
)
|
||||
"""
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & test(/^connect::diagnostics::schedule::tests::/)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
@@ -74,7 +74,6 @@ Notes:
|
||||
- `.github/workflows/e2e-s3tests.yml` (weekly full s3-tests sweep)
|
||||
- `.github/workflows/mint.yml` (weekly multi-SDK mint run)
|
||||
- `.github/workflows/fuzz.yml` (nightly fuzz corpus jobs)
|
||||
- `.github/workflows/performance-ab.yml` (nightly warp A/B gate)
|
||||
- **Pending:** the ci-7 e2e nightly-full workflow does not exist yet
|
||||
(backlog#1149 ci-7). When it lands, wire it up with the snippet above.
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
"never_ran_grace_until": "2026-09-08T00:00:00Z"
|
||||
},
|
||||
{ "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
|
||||
{
|
||||
"workflow": ".github/workflows/runner-hygiene.yml",
|
||||
"max_age_hours": 792,
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
This repository is public. For a public repository, GitHub's standard hosted runners (`ubuntu-latest`, `windows-latest`, `macos-latest`) cost nothing and have no minute cap, whatever plan the organisation is on. GitHub bills only two things: larger hosted runners, which have custom labels and are not used here, and standard runners in private repositories.
|
||||
|
||||
Every other label in this repository is capacity RustFS pays for by the hour whether or not a job is running well:
|
||||
Every other label in this repository is capacity RustFS pays for. ARC pods and lab machines accrue cost while the machine exists. A Ubicloud label is metered for the minutes a job holds its VM:
|
||||
|
||||
| Label | Paid by | What it is | Use it for |
|
||||
|---|---|---|---|
|
||||
@@ -15,7 +15,7 @@ Every other label in this repository is capacity RustFS pays for by the hour whe
|
||||
| `windows-latest`, `macos-latest` | nobody | GitHub-hosted standard runners | Windows and macOS targets |
|
||||
| `sm-standard-2` | RustFS | ARC pod, Ubuntu 24.04, minimal image: no `gh`, no C toolchain; Docker only on the `dind-` label | only a job that a hosted runner cannot run (see the placement rule) |
|
||||
| `sm-standard-4` | RustFS | ARC pod, Ubuntu 24.04, 14 CPUs and 28 GB (the label is not the core count), large disk | workspace-scale Cargo builds |
|
||||
| `sm-standard-4-arm` | RustFS | ARC pod, aarch64; reached through the `build.yml` matrix, so `actionlint.yaml` does not list it | native aarch64 release builds |
|
||||
| `ubicloud-standard-16-arm` | RustFS | Ubicloud VM, alias of `ubicloud-standard-16-arm-ubuntu-2404`: Ubuntu 24.04 arm64, 16 vCPU, 48 GB, 300 GB disk. Reached through the `build.yml` matrix, so `actionlint.yaml` does not list it | native aarch64 release builds (`linux-aarch64-musl`, `linux-aarch64-gnu`) |
|
||||
| `dind-sm-standard-2` | RustFS | ARC pod with Docker-in-Docker | image builds that need `buildx` and a registry push |
|
||||
| `smoke-testing`, `pf-testing` | RustFS | long-lived lab machines | the packaged functional chain and performance suites |
|
||||
|
||||
@@ -40,7 +40,7 @@ A hosted runner is also the only choice for a few things the pods cannot do: `mo
|
||||
|
||||
- Every `shared-key` has exactly one writer. `cache-warm.yml` owns `ci-dev`, `ci-feat-rio`, `ci-feat-proto` and `ci-uring`; every other `./.github/actions/setup` call states `cache-save-if` explicitly, and `scripts/security/check_cache_save_if.sh` fails the build when one does not.
|
||||
- All readers of a key run on the same image as its writer. Keys that live on hosted runners say so in their name (`ci-uring` is warmed on `ubuntu-latest`; `ci-e2e-distributed-hosted`, `e2e-upgrade-server-hosted`, `e2e-upgrade-tests-hosted`).
|
||||
- Moving a job between pools therefore means renaming its key. The new key is empty until a run from `main` (a scheduled run, or a manual dispatch of that workflow from `main`) saves it; until then that lane does a cold build.
|
||||
- Moving a job between pools therefore means renaming its key. The new key is empty until a run from `main` (a scheduled run, or a manual dispatch of that workflow from `main`) saves it; until then that lane does a cold build. The `build.yml` aarch64 release legs append `-ubicloud-arm` for this reason. `cache-warm.yml` does not write those keys.
|
||||
- `CARGO_BUILD_JOBS` and every other `CARGO*`, `CC*`, `CFLAGS*`, `CXX*`, `CMAKE*` and `RUST*` variable is hashed into the key. `ci.yml` sets `CARGO_BUILD_JOBS` at step level for that reason; promoting it to job level rotates the key for every lane that shares it.
|
||||
|
||||
## Where self-hosted minutes leak
|
||||
@@ -62,7 +62,7 @@ gh api "repos/rustfs/rustfs/actions/runs/<run_id>/jobs?per_page=100" \
|
||||
--jq '.jobs[] | "\(.conclusion)\t\(.labels | join(","))\t\(.started_at)\t\(.completed_at)\t\(.name)"'
|
||||
```
|
||||
|
||||
Only rows whose label starts with `sm-`, `dind-`, `smoke-` or `pf-` cost anything. At the October 2026 review, a full pull-request run of `ci.yml` spent about 150 self-hosted minutes, a third of it in jobs that never compiled; treat that as the scale of what a placement mistake costs, not as a current figure.
|
||||
Only rows whose label starts with `sm-`, `dind-`, `ubicloud-`, `smoke-` or `pf-` cost anything. At the October 2026 review, a full pull-request run of `ci.yml` spent about 150 self-hosted minutes, a third of it in jobs that never compiled; treat that as the scale of what a placement mistake costs, not as a current figure.
|
||||
|
||||
## Checklist for a new or moved job
|
||||
|
||||
|
||||
@@ -29,7 +29,6 @@ on:
|
||||
- 'scripts/test_package_service_scripts.sh'
|
||||
- 'scripts/test_package_versions.sh'
|
||||
- 'scripts/test_docker_workflow.py'
|
||||
- 'scripts/security/check_performance_ab_workflow.sh'
|
||||
- 'scripts/security/check_preview_release_workflow.sh'
|
||||
- 'scripts/security/check_tier_artifact_workflow.sh'
|
||||
- 'scripts/security/check_workflow_pins.sh'
|
||||
@@ -48,7 +47,6 @@ on:
|
||||
- 'scripts/test_package_service_scripts.sh'
|
||||
- 'scripts/test_package_versions.sh'
|
||||
- 'scripts/test_docker_workflow.py'
|
||||
- 'scripts/security/check_performance_ab_workflow.sh'
|
||||
- 'scripts/security/check_preview_release_workflow.sh'
|
||||
- 'scripts/security/check_tier_artifact_workflow.sh'
|
||||
- 'scripts/security/check_workflow_pins.sh'
|
||||
@@ -155,9 +153,6 @@ jobs:
|
||||
- name: Check preview release workflow policy
|
||||
run: ./scripts/security/check_preview_release_workflow.sh
|
||||
|
||||
- name: Check performance A/B workflow trust boundary
|
||||
run: ./scripts/security/check_performance_ab_workflow.sh
|
||||
|
||||
- name: Check tier evidence workflow isolation
|
||||
run: ./scripts/security/check_tier_artifact_workflow.sh
|
||||
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
name: BTF prototype reference
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "scripts/inspect_btf_prototypes.py"
|
||||
- "scripts/test_inspect_btf_prototypes.py"
|
||||
- "scripts/fixtures/btf_prototypes.c"
|
||||
- ".github/workflows/btf-prototype-reference.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: btf-prototype-reference-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
reference:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Check prototype and CLI contracts
|
||||
run: python3 -B scripts/test_inspect_btf_prototypes.py
|
||||
- name: Install only missing native reference tools
|
||||
run: |
|
||||
select_bpftool() {
|
||||
local candidate
|
||||
local candidates=()
|
||||
if command -v bpftool >/dev/null; then
|
||||
candidates+=("$(command -v bpftool)")
|
||||
fi
|
||||
for candidate in /usr/lib/linux-tools-*/bpftool /usr/lib/linux-tools/*/bpftool; do
|
||||
if [[ -x "$candidate" ]]; then
|
||||
candidates+=("$candidate")
|
||||
fi
|
||||
done
|
||||
while IFS= read -r candidate; do
|
||||
if [[ -x "$candidate" ]] && file -Lb "$candidate" | grep -q 'ELF' \
|
||||
&& "$candidate" version >/dev/null 2>&1; then
|
||||
printf '%s\n' "$candidate"
|
||||
return 0
|
||||
fi
|
||||
done < <(printf '%s\n' "${candidates[@]}" | sort -Vr)
|
||||
return 1
|
||||
}
|
||||
packages=()
|
||||
command -v clang >/dev/null || packages+=(clang)
|
||||
command -v llvm-objcopy >/dev/null || packages+=(llvm)
|
||||
select_bpftool >/dev/null || packages+=(linux-tools-generic)
|
||||
if (( ${#packages[@]} )); then
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y "${packages[@]}"
|
||||
fi
|
||||
# File parsing needs a working binary, not a host-kernel wrapper.
|
||||
selected_bpftool="$(select_bpftool)"
|
||||
"$selected_bpftool" version
|
||||
printf 'BPFTOOL=%s\n' "$selected_bpftool" >> "$GITHUB_ENV"
|
||||
- name: Compare compiled declarations with bpftool
|
||||
run: |
|
||||
clang -target bpf -O2 -g -c scripts/fixtures/btf_prototypes.c -o "$RUNNER_TEMP/prototypes.o"
|
||||
llvm-objcopy --dump-section ".BTF=$RUNNER_TEMP/prototypes.btf" "$RUNNER_TEMP/prototypes.o"
|
||||
python3 -B scripts/test_inspect_btf_prototypes.py --native-reference "$RUNNER_TEMP/prototypes.btf"
|
||||
@@ -181,12 +181,18 @@ jobs:
|
||||
selected="all"
|
||||
fi
|
||||
|
||||
|
||||
# linux-aarch64-* need a real aarch64 host (placement rule: hardware)
|
||||
# so the packaged binary and console smoke execute the artifact.
|
||||
# ubicloud-standard-16-arm is that host: Ubuntu 24.04 arm64, alias of
|
||||
# ubicloud-standard-16-arm-ubuntu-2404 (16 vCPU, 48 GB). linux-x86_64-* stay on the
|
||||
# ARC sm-standard-4 pool. GNU rows keep cross=true so cargo-zigbuild
|
||||
# pins the glibc 2.31 floor on the same architecture; musl stays a
|
||||
# native cargo build.
|
||||
all='{"include":[
|
||||
{"target_id":"linux-x86_64-musl","os":"sm-standard-4","target":"x86_64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
|
||||
{"target_id":"linux-aarch64-musl","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
|
||||
{"target_id":"linux-aarch64-musl","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
|
||||
{"target_id":"linux-x86_64-gnu","os":"sm-standard-4","target":"x86_64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"linux-aarch64-gnu","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"linux-aarch64-gnu","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
|
||||
{"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":"","features":"","glibc":""}
|
||||
]}'
|
||||
@@ -244,7 +250,11 @@ jobs:
|
||||
with:
|
||||
rust-version: stable
|
||||
target: ${{ matrix.target }}
|
||||
cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}
|
||||
# rust-cache keys include OS and architecture but not the runner image.
|
||||
# The Ubicloud aarch64 legs therefore use their own suffix so they
|
||||
# cannot restore a cache written on a different arm64 image.
|
||||
# x86_64, macOS, and Windows keys stay unsuffixed.
|
||||
cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}${{ matrix.os == 'ubicloud-standard-16-arm' && '-ubicloud-arm' || '' }}
|
||||
# main only. A cache saved on refs/tags/X is scoped to that tag: no
|
||||
# other tag, no main run and no PR can restore it, so every release
|
||||
# cycle wrote up to 12 entries of 1-2GB (preview tag plus final tag,
|
||||
@@ -764,9 +774,9 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# The self-hosted Linux runners do not ship the aws CLI (GitHub-hosted
|
||||
# images did). Install the official CLI bundle on demand so R2 uploads
|
||||
# survive a fresh runner without relying on a distro package.
|
||||
# ARC pods (the x86_64 legs) do not ship the aws CLI. Ubicloud arm64
|
||||
# images follow the GitHub runner image and often do. Install the
|
||||
# official CLI bundle when it is missing so R2 uploads work on either.
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
echo "aws CLI not found on runner; installing..."
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
|
||||
@@ -45,7 +45,7 @@ on:
|
||||
# 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
|
||||
# build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
|
||||
# (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
|
||||
# e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
|
||||
# and e2e-replication-nightly (04:00) lanes.
|
||||
- cron: "43 7 * * 0"
|
||||
|
||||
concurrency:
|
||||
|
||||
@@ -38,8 +38,8 @@ name: e2e-nightly
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00),
|
||||
# stale (01:30) and performance-ab (06:00).
|
||||
# 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00)
|
||||
# and stale (01:30).
|
||||
- cron: "29 4 * * *"
|
||||
|
||||
# Only alert-on-failure needs more than read access; it declares its own
|
||||
|
||||
@@ -1,384 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Warp A/B budget gate for the hotpath series (rustfs/backlog#935 HP-14, item 4).
|
||||
#
|
||||
# Two entry points, honestly scoped:
|
||||
# * schedule (nightly, on main): post-merge detection — catches a regression
|
||||
# within 24h of landing, not before merge.
|
||||
# * workflow_dispatch: an explicitly selected trusted ref.
|
||||
# The dispatch input can run the gate with --allow-regression so a deliberate
|
||||
# correctness cost (e.g. the #4221 fsync durability fix) is recorded, not
|
||||
# blocked (rustfs/backlog#935 correction 1).
|
||||
|
||||
name: Performance A/B
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "31 6 * * *" # 06:31 UTC nightly, against main
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
duration:
|
||||
description: "warp duration per round"
|
||||
required: false
|
||||
default: "12s"
|
||||
type: string
|
||||
allow_regression:
|
||||
description: "Pass the gate despite a FAIL (deliberate tradeoff)"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
warp-ab:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Warp A/B budget gate
|
||||
runs-on: sm-standard-2
|
||||
# A normal nightly restores the last successful binary and builds only the
|
||||
# candidate; daily access keeps that cache warm. A cache miss may build both
|
||||
# and needs room for the A/B run plus artifact and cache publication.
|
||||
timeout-minutes: 180
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0 # baseline may be an earlier successful scheduled head
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
|
||||
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Install warp
|
||||
run: |
|
||||
set -euo pipefail
|
||||
WARP_VERSION="1.0.0"
|
||||
curl -fsSL "https://github.com/minio/warp/releases/download/v${WARP_VERSION}/warp_Linux_x86_64.tar.gz" \
|
||||
| sudo tar -xz -C /usr/local/bin warp
|
||||
warp --version
|
||||
|
||||
- name: Decide exemption
|
||||
id: exempt
|
||||
env:
|
||||
INPUT_ALLOW_REGRESSION: ${{ github.event.inputs.allow_regression }}
|
||||
run: |
|
||||
allow="false"
|
||||
if [[ "$INPUT_ALLOW_REGRESSION" == "true" ]]; then
|
||||
allow="true"
|
||||
fi
|
||||
echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# A failed regression run must keep comparing against the last known-good
|
||||
# scheduled head. Otherwise the next nightly would absorb the regression
|
||||
# into its baseline and turn green without a fix.
|
||||
- name: Find last successful scheduled baseline
|
||||
id: scheduled_baseline
|
||||
if: github.event_name == 'schedule'
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
result-encoding: string
|
||||
script: |
|
||||
const { data } = await github.rest.actions.listWorkflowRuns({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
workflow_id: "performance-ab.yml",
|
||||
event: "schedule",
|
||||
status: "success",
|
||||
per_page: 1,
|
||||
});
|
||||
return data.workflow_runs[0]?.head_sha ?? "";
|
||||
|
||||
# Manual runs compare a selected ref with current main. Scheduled runs
|
||||
# compare current main with the last successful scheduled head. With no
|
||||
# history, the first run measures the candidate against itself and seeds
|
||||
# that head only if the complete rig succeeds.
|
||||
- name: Resolve baseline / candidate commits
|
||||
id: commits
|
||||
env:
|
||||
SCHEDULED_BASELINE_SHA: ${{ steps.scheduled_baseline.outputs.result }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
candidate_sha="$(git rev-parse HEAD)"
|
||||
if [[ "${{ github.event_name }}" == "schedule" ]]; then
|
||||
baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}"
|
||||
else
|
||||
baseline_sha="$(git rev-parse origin/main)"
|
||||
fi
|
||||
git cat-file -e "${baseline_sha}^{commit}"
|
||||
if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
|
||||
echo "::error::baseline $baseline_sha is not an ancestor of candidate $candidate_sha; update the selected ref before comparing" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
|
||||
echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
|
||||
echo "baseline commit: $baseline_sha"
|
||||
echo "candidate commit: $candidate_sha"
|
||||
|
||||
# Exact-key restore of the candidate binary saved by its successful
|
||||
# scheduled run. A miss leaves cache-hit unset and falls back to a source
|
||||
# build of that known-good head.
|
||||
- name: Restore cached baseline binary
|
||||
id: baseline_cache
|
||||
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: baseline-bin/rustfs
|
||||
key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}
|
||||
|
||||
# Self-heal: on a nightly/dispatch run where the candidate commit IS the
|
||||
# baseline commit, a cache miss would make the rig build the same commit
|
||||
# twice (~65min per side with the post-#4806 LTO profile — no job budget
|
||||
# fits that). Build it once here, reuse it for both phases, and save it
|
||||
# back to the cache so the next run hits.
|
||||
- name: Build baseline on cache miss (same-commit self-heal)
|
||||
id: selfheal
|
||||
if: >-
|
||||
steps.baseline_cache.outputs.cache-hit != 'true' &&
|
||||
steps.commits.outputs.baseline_sha == steps.commits.outputs.candidate_sha
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cargo build --release --bin rustfs
|
||||
mkdir -p baseline-bin
|
||||
cp target/release/rustfs baseline-bin/rustfs
|
||||
echo "built=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build baseline on cache miss (different candidate)
|
||||
id: baseline_build
|
||||
if: >-
|
||||
steps.baseline_cache.outputs.cache-hit != 'true' &&
|
||||
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
run: |
|
||||
set -euo pipefail
|
||||
baseline_root="$RUNNER_TEMP/rustfs-baseline-${{ github.run_id }}"
|
||||
baseline_target="$RUNNER_TEMP/rustfs-baseline-target-${{ github.run_id }}"
|
||||
git worktree add --detach "$baseline_root" "${{ steps.commits.outputs.baseline_sha }}"
|
||||
cargo build --release --manifest-path "$baseline_root/Cargo.toml" --bin rustfs --target-dir "$baseline_target"
|
||||
mkdir -p baseline-bin
|
||||
cp "$baseline_target/release/rustfs" baseline-bin/rustfs
|
||||
git worktree remove --force "$baseline_root"
|
||||
echo "built=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build candidate binary
|
||||
id: candidate_build
|
||||
if: steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cargo build --release --bin rustfs
|
||||
mkdir -p candidate-bin
|
||||
cp target/release/rustfs candidate-bin/rustfs
|
||||
echo "built=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Save self-healed baseline to cache
|
||||
if: steps.selfheal.outputs.built == 'true' || steps.baseline_build.outputs.built == 'true'
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: baseline-bin/rustfs
|
||||
key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}
|
||||
|
||||
- name: Run warp A/B and gate
|
||||
id: ab
|
||||
env:
|
||||
INPUT_DURATION: ${{ github.event.inputs.duration }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# The formal runner executes A1 baseline -> B1 candidate -> B2 candidate
|
||||
# -> A2 baseline for each workload and drive-sync cell. It requires three
|
||||
# rounds per leg to emit tail latency and error-rate evidence.
|
||||
# --health-timeout 180 outlasts the server's own 120s startup-readiness
|
||||
# budget, which the rig's previous 60s health poll undershot (the first
|
||||
# two nightly failures). perf-6 recalibrates these once the noise study
|
||||
# lands.
|
||||
duration="${INPUT_DURATION:-12s}"
|
||||
baseline_sha="${{ steps.commits.outputs.baseline_sha }}"
|
||||
candidate_sha="${{ steps.commits.outputs.candidate_sha }}"
|
||||
baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}"
|
||||
selfheal_built="${{ steps.selfheal.outputs.built }}"
|
||||
baseline_built="${{ steps.baseline_build.outputs.built }}"
|
||||
candidate_built="${{ steps.candidate_build.outputs.built }}"
|
||||
|
||||
args=(--duration "$duration" --rounds 3 --cooldown 5 --health-timeout 180 --baseline-revision "$baseline_sha" --candidate-revision "$candidate_sha")
|
||||
|
||||
if [[ "$baseline_hit" == "true" || "$selfheal_built" == "true" || "$baseline_built" == "true" ]]; then
|
||||
chmod +x baseline-bin/rustfs
|
||||
base_bin="$PWD/baseline-bin/rustfs"
|
||||
args+=(--baseline-bin "$base_bin")
|
||||
if [[ "$baseline_hit" == "true" ]]; then
|
||||
base_src="actions-cache (rustfs-baseline-$baseline_sha)"
|
||||
elif [[ "$selfheal_built" == "true" ]]; then
|
||||
base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
|
||||
else
|
||||
base_src="isolated baseline source build (saved as rustfs-baseline-$baseline_sha)"
|
||||
fi
|
||||
if [[ "$candidate_sha" == "$baseline_sha" ]]; then
|
||||
# No commits landed since the last successful baseline, so reuse
|
||||
# the one binary for both phases and measure only rig drift.
|
||||
args+=(--candidate-bin "$base_bin")
|
||||
cand_src="same binary as baseline (same commit)"
|
||||
elif [[ "$candidate_built" == "true" ]]; then
|
||||
chmod +x candidate-bin/rustfs
|
||||
args+=(--candidate-bin "$PWD/candidate-bin/rustfs")
|
||||
cand_src="source build of the checked-out ref"
|
||||
else
|
||||
echo "::error::candidate binary was not built" >&2
|
||||
exit 2
|
||||
fi
|
||||
else
|
||||
echo "::error::baseline binary was not restored or built" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo "baseline binary: $base_src"
|
||||
echo "candidate binary: $cand_src"
|
||||
|
||||
if [[ "${{ steps.exempt.outputs.allow_regression }}" == "true" ]]; then
|
||||
args+=(--allow-regression --exemption-reason "workflow dispatch override")
|
||||
fi
|
||||
# Do not let a gate FAIL abort the job here; capture status and surface
|
||||
# it after the step summary is written.
|
||||
set +e
|
||||
bash scripts/run_hotpath_warp_abba.sh "${args[@]}"
|
||||
echo "status=$?" >> "$GITHUB_OUTPUT"
|
||||
set -e
|
||||
# Locate the newest run dir + gate.md for the summary/comment/artifact
|
||||
# steps. On a startup failure there is no gate.md, but the run dir still
|
||||
# holds server-logs/ for diagnosis.
|
||||
# Run dirs are UTC-timestamp names (no special chars); ls is safe here.
|
||||
# shellcheck disable=SC2012
|
||||
run_dir="$(ls -td target/hotpath-abba/*/ 2>/dev/null | head -n1 || true)"
|
||||
echo "run_dir=${run_dir%/}" >> "$GITHUB_OUTPUT"
|
||||
# shellcheck disable=SC2012
|
||||
gate_md="$(ls -t target/hotpath-abba/*/candidate_gate.md 2>/dev/null | head -n1 || true)"
|
||||
echo "gate_md=$gate_md" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload A/B results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: hotpath-warp-ab-${{ github.run_number }}
|
||||
# Includes per-cell median_summary.csv / baseline_compare.csv, both gates,
|
||||
# and server-logs/ (rustfs.log + startup env per phase) so a failed run
|
||||
# is diagnosable. Short retention: this is churny nightly debug data.
|
||||
path: target/hotpath-abba/
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
||||
- name: Write gate summary
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
status="${{ steps.ab.outputs.status }}"
|
||||
gate_md="${{ steps.ab.outputs.gate_md }}"
|
||||
run_dir="${{ steps.ab.outputs.run_dir }}"
|
||||
{
|
||||
echo "## Hotpath warp A/B — run ${{ github.run_number }}"
|
||||
echo
|
||||
if [[ "$status" == "0" ]]; then
|
||||
echo "Rig/gate exit: \`0\` (pass or warn)."
|
||||
else
|
||||
echo "Rig/gate exit: \`${status:-unknown}\` — **FAILED**."
|
||||
fi
|
||||
echo
|
||||
if [[ -n "$gate_md" && -f "$gate_md" ]]; then
|
||||
cat "$gate_md"
|
||||
else
|
||||
echo "No \`gate.md\` produced — the rig failed **before** the gate"
|
||||
echo "(most likely server startup / health). Failing phase(s) below;"
|
||||
echo "full logs in the \`hotpath-warp-ab-${{ github.run_number }}\` artifact."
|
||||
if [[ -n "$run_dir" && -d "$run_dir/server-logs" ]]; then
|
||||
for f in "$run_dir"/server-logs/*.log; do
|
||||
[[ -f "$f" ]] || continue
|
||||
echo
|
||||
echo "<details><summary>$(basename "$f")</summary>"
|
||||
echo
|
||||
echo '```'
|
||||
tail -n 30 "$f"
|
||||
echo '```'
|
||||
echo "</details>"
|
||||
done
|
||||
fi
|
||||
fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stage successful candidate baseline
|
||||
if: >-
|
||||
steps.ab.outputs.status == '0' &&
|
||||
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cp candidate-bin/rustfs baseline-bin/rustfs
|
||||
|
||||
- name: Cache successful candidate baseline
|
||||
if: >-
|
||||
steps.ab.outputs.status == '0' &&
|
||||
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: baseline-bin/rustfs
|
||||
key: rustfs-baseline-${{ steps.commits.outputs.candidate_sha }}
|
||||
|
||||
# Scheduled failure alerting is handled by the alert-on-failure job below
|
||||
# (perf-2 consuming ci-8's schedule-failure-issue composite action).
|
||||
|
||||
- name: Enforce gate
|
||||
if: always()
|
||||
run: |
|
||||
status="${{ steps.ab.outputs.status }}"
|
||||
if [[ -z "$status" ]]; then
|
||||
echo "::error::warp A/B setup failed before the rig ran. Check the first failed workflow step." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$status" != "0" ]]; then
|
||||
echo "::error::warp A/B budget gate failed (exit $status). See the step summary / gate.md artifact." >&2
|
||||
exit "$status"
|
||||
fi
|
||||
echo "warp A/B budget gate passed."
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [warp-ab]
|
||||
# `always()` is required: without it this job is skipped when a needed
|
||||
# job fails. Alerts only for scheduled (nightly) runs (backlog#1149
|
||||
# ci-8); manual dispatch failures are already watched by a human.
|
||||
# `cancelled` is included alongside `failure` on purpose: a job that hits
|
||||
# timeout-minutes ends as `cancelled`, and the 2026-07-11..07-14 nightly
|
||||
# timeouts went silent precisely because the guard was failure-only. The
|
||||
# composite action already reports cancelled/timed-out jobs in the issue
|
||||
# body.
|
||||
if: >-
|
||||
github.repository == 'rustfs/rustfs' && (
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
@@ -16,8 +16,8 @@
|
||||
#
|
||||
# Forces a job failure and then runs .github/actions/schedule-failure-issue
|
||||
# through the exact `needs` + `always() && contains(needs.*.result,
|
||||
# 'failure')` wiring used by the real consumers (e2e-s3tests, mint, fuzz,
|
||||
# performance-ab). Dispatch it twice to verify both the issue-creation and
|
||||
# 'failure')` wiring used by the real consumers (e2e-s3tests, mint, fuzz).
|
||||
# Dispatch it twice to verify both the issue-creation and
|
||||
# the dedupe-comment paths, then close the resulting
|
||||
# "[scheduled-failure] Schedule Failure Alert Drill" issue in rustfs/backlog
|
||||
# (the default target repository). Requires the BACKLOG_ISSUE_TOKEN secret
|
||||
|
||||
@@ -28,7 +28,6 @@ on:
|
||||
- "mint"
|
||||
- "minio-interop"
|
||||
- "Nightly GNU Build"
|
||||
- "Performance A/B"
|
||||
- "Runner Hygiene"
|
||||
types: [completed]
|
||||
|
||||
|
||||
Generated
+169
-86
@@ -365,9 +365,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
|
||||
|
||||
[[package]]
|
||||
name = "arrow"
|
||||
version = "59.3.1"
|
||||
version = "59.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c25317ebddb8f2e6f006759336c95e5ded847a446f3524c27853bfe7d77a69b"
|
||||
checksum = "3f04e0a89cc175eaf3740282537851e6fd9d7276f97730b9d0ea2859c11a85df"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -386,9 +386,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-arith"
|
||||
version = "59.3.1"
|
||||
version = "59.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ab555c0ffe9cb817041c0cee80baa9663a22225d0a035693579771f2693cadc"
|
||||
checksum = "157943c720919e8f5b6731407ce5cd4c9a5ef47a75bf10edaf24b7081b5763ef"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -400,9 +400,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-array"
|
||||
version = "59.3.1"
|
||||
version = "59.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2dae1d91737421e92c6dd7c7f9d0f2d25198fd6fc539738b628f4dad10adf533"
|
||||
checksum = "72b5b1682924f11ecf6929689c16cbc69efbb6dd141b06ad98ee1cb10d858c43"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow-buffer",
|
||||
@@ -419,9 +419,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-buffer"
|
||||
version = "59.3.1"
|
||||
version = "59.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fe9cd0064f2d7cf8ecfaf77cd70e67a980bc71aea92b63102bd37a7d3379a09"
|
||||
checksum = "b5d687afc6c927965a8a759f3b68de91933d9dfa2e1bb6e156bac84b85665e92"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"half",
|
||||
@@ -431,9 +431,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-cast"
|
||||
version = "59.3.1"
|
||||
version = "59.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee510bc252781d700400979a96bd1eadebeae23901132bbabd92118da1b376c5"
|
||||
checksum = "3f23c3379feede42135ef27ca60e67f6e643cd4343ae74e4c8b16c83f55afd85"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -453,9 +453,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-csv"
|
||||
version = "59.3.1"
|
||||
version = "59.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ace83d5935421b950fea3a0a3f9972c0837c297ef98ade57127a5661b681112c"
|
||||
checksum = "85b1984628b091b8319b3c305fce21d04f9e9b2578bb733b0d0108462ae1d698"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-cast",
|
||||
@@ -468,9 +468,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-data"
|
||||
version = "59.3.1"
|
||||
version = "59.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "54f128c73bb7fe9f7137218f910edd5b436f390e0f040b1956f5028390787897"
|
||||
checksum = "bb0a031c7ff5e2bdb443117054fb466af1d4df5b7f0ecbe197713dc8b113b1ba"
|
||||
dependencies = [
|
||||
"arrow-buffer",
|
||||
"arrow-schema",
|
||||
@@ -481,9 +481,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-ipc"
|
||||
version = "59.3.1"
|
||||
version = "59.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e244f39b82fa7a823e8b5984a0bd988234f9c92c1df79378fb85090e6e4c821"
|
||||
checksum = "f5dd1afadda2eb821aaf851bb4c0adef60c30cc0f1a29bd02fb99dd07fb9f58d"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -497,9 +497,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-json"
|
||||
version = "59.3.1"
|
||||
version = "59.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "727b21ebf8894dcd9504d6ab66bffdac338391e1f85cde594526a0210299ea40"
|
||||
checksum = "e2bc4d56a0dad84cc51c5812ccff48b642047a1f3be80349cd84318983e79edb"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -522,9 +522,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-ord"
|
||||
version = "59.3.1"
|
||||
version = "59.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70d376b9b883239ae0f9963cdeeb59126ccaac08d60adf208a6ad3e2dd4048c3"
|
||||
checksum = "561b8209faa2cd489764b7fd07d7cb9e8d274d78048c38de59f77c41140cb834"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -535,9 +535,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-row"
|
||||
version = "59.3.1"
|
||||
version = "59.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48096811f92b5c3a0c59d33cfecd7d7d2ed037d81e54a27a42d232e7700e6de2"
|
||||
checksum = "0d8c68b87a569f92b90c46cb0a09d86bd9609ddc9b1bf1fc5a503c8c6907c14e"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -548,9 +548,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-schema"
|
||||
version = "59.3.1"
|
||||
version = "59.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37b6c8a6b12cf3869d299ce36ed8e1cd6fe9836264b68fb6c6538473a36124a1"
|
||||
checksum = "e87acb0c6bf6237fca7a50d7b51e801c98a18a5fe6f3bb34f7d77b85ca165388"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_json",
|
||||
@@ -558,9 +558,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-select"
|
||||
version = "59.3.1"
|
||||
version = "59.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e523e85fa9fb3181a391bf0edfa6a1ae0dc46df8cbf7f6b96034dcb1e2a7a04a"
|
||||
checksum = "c844141958d805b47d1208e0d12aac53fcf388ce8011639fe82aa492adde0bb4"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow-array",
|
||||
@@ -572,9 +572,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-string"
|
||||
version = "59.3.1"
|
||||
version = "59.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b931a905a0e4b8832d9f54fb369cf6e545f9dc03cb3d9611216a7bd2653df60a"
|
||||
checksum = "bc130892b1a9552da77971924b390e06cf375d1f7e4344a7b7b40ca4426fa470"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -1514,6 +1514,23 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bindgen"
|
||||
version = "0.73.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "787ef8ef523575546b106a58213d6e6b06198a05c2f757258c68a74273670cfa"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"cexpr",
|
||||
"clang-sys",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
"shlex",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "binrw"
|
||||
version = "0.15.2"
|
||||
@@ -1726,13 +1743,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brotli"
|
||||
version = "9.0.0"
|
||||
version = "9.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8b851b75c23ca7873623d612fe49bd1989aeb03d08fb9432187eb253d3d4c6b"
|
||||
checksum = "6737ce8b81460bf9407909f73ff50bf13f7cb8577e1e76715f57ed0eba58ba44"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib 3.0.0",
|
||||
"alloc-stdlib 0.3.0",
|
||||
"brotli-decompressor 6.0.1",
|
||||
"brotli-decompressor 6.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1747,9 +1764,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brotli-decompressor"
|
||||
version = "6.0.1"
|
||||
version = "6.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "941cd9bd4ddab83cb46fa5a2d428f1c857b24ac78cb876cf7beb710840934bd7"
|
||||
checksum = "34c6783f42c21c952ad7112b354f0ba2f10760e9decea5b1b60523d6e270faaf"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib 3.0.0",
|
||||
"alloc-stdlib 0.3.0",
|
||||
@@ -1967,6 +1984,15 @@ dependencies = [
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cexpr"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
|
||||
dependencies = [
|
||||
"nom 7.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.5"
|
||||
@@ -2083,6 +2109,17 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clang-sys"
|
||||
version = "1.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a"
|
||||
dependencies = [
|
||||
"glob",
|
||||
"libc",
|
||||
"libloading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.7"
|
||||
@@ -2199,7 +2236,7 @@ version = "0.4.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe"
|
||||
dependencies = [
|
||||
"brotli 9.0.0",
|
||||
"brotli 9.1.0",
|
||||
"bzip2",
|
||||
"compression-core",
|
||||
"flate2",
|
||||
@@ -2207,7 +2244,7 @@ dependencies = [
|
||||
"lz4",
|
||||
"memchr",
|
||||
"zstd 0.14.3",
|
||||
"zstd-safe 8.2.0",
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2542,9 +2579,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-bigint"
|
||||
version = "0.7.7"
|
||||
version = "0.7.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc907c45bb7e9ef8fc3778defd0f3cd8c81288f1b4e748eba064948f20c47122"
|
||||
checksum = "9c035cb64cba03dad2fb921f3e218785e8cf9a41790da5eb3a1400f62fa2be11"
|
||||
dependencies = [
|
||||
"cpubits",
|
||||
"ctutils",
|
||||
@@ -2620,9 +2657,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ctutils"
|
||||
version = "0.4.5"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0"
|
||||
checksum = "fe3e9d21d39e6f39ff46e03d5b621424f88ee23d8a1672c6bfc9ccc7f2da4f6a"
|
||||
dependencies = [
|
||||
"cmov",
|
||||
"subtle",
|
||||
@@ -3632,9 +3669,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "der"
|
||||
version = "0.8.2"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a"
|
||||
checksum = "6a996a108feb8b427a6e12fcf6af4aa83409b79d72c15337d7741368131ee004"
|
||||
dependencies = [
|
||||
"const-oid 0.10.2",
|
||||
"pem-rfc7468 1.0.0",
|
||||
@@ -4040,6 +4077,7 @@ dependencies = [
|
||||
"rustfs-utils",
|
||||
"rustls",
|
||||
"s3s",
|
||||
"same-file",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
@@ -4066,7 +4104,7 @@ version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0"
|
||||
dependencies = [
|
||||
"der 0.8.2",
|
||||
"der 0.8.3",
|
||||
"digest 0.11.3",
|
||||
"elliptic-curve",
|
||||
"rfc6979",
|
||||
@@ -4131,9 +4169,9 @@ checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
|
||||
|
||||
[[package]]
|
||||
name = "elliptic-curve"
|
||||
version = "0.14.2"
|
||||
version = "0.14.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ad4496221fcfc21015fd8c0e26fcfab00504832dc0604f52acdd7bad2a8dc81"
|
||||
checksum = "900f37e7e18d74a2347389af6cd69e3a30d5cc9e697f40004865679aa75e9c7c"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"crypto-bigint",
|
||||
@@ -5235,9 +5273,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath"
|
||||
version = "0.29.1"
|
||||
version = "0.29.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62f2351902fa259b758c7303505273393f6b08172ab5bdd5739a2d2bc6d554e8"
|
||||
checksum = "5b4af2b5ecf2650278fe33b0e49b68c8c484d2306475c127628316dea740cf5c"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -5270,15 +5308,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-drain"
|
||||
version = "0.29.1"
|
||||
version = "0.29.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83dab2340134d9976d62f9f5172f9c6584c7f521b092f258e0e58e412e160b53"
|
||||
checksum = "a5cceb6c15fbaaeb00b51acc7be7fad533d0b56cb422b1cfc16cf9ca1624d33a"
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros"
|
||||
version = "0.29.1"
|
||||
version = "0.29.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a04bbe5701b097f224a08187a78b75bb9d9b4bbaff6a279c2a50ba73edbb7aac"
|
||||
checksum = "08c0a42eb299ebb7ec6b9161489e5bf132603a34ca12cecadb8490937aa0492a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -6398,7 +6436,7 @@ dependencies = [
|
||||
"hashbrown 0.16.1",
|
||||
"indexmap 2.14.2",
|
||||
"metrics",
|
||||
"ordered-float 5.5.0",
|
||||
"ordered-float 5.5.1",
|
||||
"quanta",
|
||||
"radix_trie",
|
||||
"rand 0.9.5",
|
||||
@@ -7332,9 +7370,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ordered-float"
|
||||
version = "5.5.0"
|
||||
version = "5.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0"
|
||||
checksum = "7f59006ce1e204c30956638535b74d317bca99477b9356dfaec54c37f6f600ab"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
@@ -7451,9 +7489,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pageant"
|
||||
version = "0.2.4"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0786409b5aaa884655c9b4065681db0835db0cc185a6fb19fb17a5d38656866"
|
||||
checksum = "e0bc922d9aa8cfcf598de27b3c6122cb2eeda9d6fe71527a22ebcb7f633540ff"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"byteorder",
|
||||
@@ -7522,9 +7560,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "parquet"
|
||||
version = "59.3.1"
|
||||
version = "59.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b3d2ef0fb136e9341570f219ecb168902e76079c6d81ad555f0a4840b1e08a3"
|
||||
checksum = "8eb260d566f2fbdabe356d26aaa5d13403aac29b5b4e44463b8f9bd7b2b12680"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow-array",
|
||||
@@ -7809,7 +7847,7 @@ version = "0.8.0-rc.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "986d2e952779af96ea048f160fd9194e1751b4faea78bcf3ceb456efe008088e"
|
||||
dependencies = [
|
||||
"der 0.8.2",
|
||||
"der 0.8.3",
|
||||
"spki 0.8.1",
|
||||
]
|
||||
|
||||
@@ -7852,7 +7890,7 @@ dependencies = [
|
||||
"aes 0.9.3",
|
||||
"aes-gcm",
|
||||
"cbc 0.2.1",
|
||||
"der 0.8.2",
|
||||
"der 0.8.3",
|
||||
"pbkdf2 0.13.0",
|
||||
"rand_core 0.10.1",
|
||||
"scrypt 0.12.0",
|
||||
@@ -7876,7 +7914,7 @@ version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7"
|
||||
dependencies = [
|
||||
"der 0.8.2",
|
||||
"der 0.8.3",
|
||||
"pkcs5 0.8.1",
|
||||
"rand_core 0.10.1",
|
||||
"spki 0.8.1",
|
||||
@@ -9138,9 +9176,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "russh"
|
||||
version = "0.64.1"
|
||||
version = "0.65.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba61e87b9ec9a39a59a6bbed4c0b8ff7fb07073405b24a767a1f2c8560ac8f58"
|
||||
checksum = "3bb77a4b259312b2abbc541bca1c13cb4e9b61c560a7fd894068c3b92e0c9dc3"
|
||||
dependencies = [
|
||||
"aes 0.9.3",
|
||||
"aws-lc-rs",
|
||||
@@ -9155,7 +9193,7 @@ dependencies = [
|
||||
"curve25519-dalek 5.0.0",
|
||||
"data-encoding",
|
||||
"delegate",
|
||||
"der 0.8.2",
|
||||
"der 0.8.3",
|
||||
"digest 0.11.3",
|
||||
"ecdsa",
|
||||
"ed25519-dalek 3.0.0",
|
||||
@@ -10846,7 +10884,7 @@ version = "1.0.2"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"blake2",
|
||||
"brotli 9.0.0",
|
||||
"brotli 9.1.0",
|
||||
"bytes",
|
||||
"convert_case 0.12.0",
|
||||
"crc-fast",
|
||||
@@ -11307,7 +11345,7 @@ checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"ctutils",
|
||||
"der 0.8.2",
|
||||
"der 0.8.3",
|
||||
"hybrid-array",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -11473,6 +11511,15 @@ dependencies = [
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_spanned"
|
||||
version = "1.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_urlencoded"
|
||||
version = "0.7.1"
|
||||
@@ -11752,9 +11799,9 @@ checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256"
|
||||
|
||||
[[package]]
|
||||
name = "sketches-ddsketch"
|
||||
version = "0.3.1"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
|
||||
checksum = "45f82fa313d625dd72aaa65ec0c9f8c95407add0510c1bef3c18ea7257ec34ce"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
@@ -11883,7 +11930,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der 0.8.2",
|
||||
"der 0.8.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12632,6 +12679,30 @@ dependencies = [
|
||||
"webpki-roots 0.26.11",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml"
|
||||
version = "0.9.12+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"
|
||||
dependencies = [
|
||||
"indexmap 2.14.2",
|
||||
"serde_core",
|
||||
"serde_spanned",
|
||||
"toml_datetime 0.7.5+spec-1.1.0",
|
||||
"toml_parser",
|
||||
"toml_writer",
|
||||
"winnow 0.7.15",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_datetime"
|
||||
version = "0.7.5+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_datetime"
|
||||
version = "1.1.2+spec-1.1.0"
|
||||
@@ -12648,9 +12719,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c"
|
||||
dependencies = [
|
||||
"indexmap 2.14.2",
|
||||
"toml_datetime",
|
||||
"toml_datetime 1.1.2+spec-1.1.0",
|
||||
"toml_parser",
|
||||
"winnow",
|
||||
"winnow 1.0.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12659,9 +12730,15 @@ version = "1.1.5+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c"
|
||||
dependencies = [
|
||||
"winnow",
|
||||
"winnow 1.0.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_writer"
|
||||
version = "1.1.3+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770"
|
||||
|
||||
[[package]]
|
||||
name = "tonic"
|
||||
version = "0.14.6"
|
||||
@@ -13688,6 +13765,12 @@ version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "0.7.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "1.0.4"
|
||||
@@ -13954,9 +14037,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.1.0"
|
||||
version = "9.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2886b00003a72c4ccee3f42cfc34908d33b2508a74d827dc2a4244706475c704"
|
||||
checksum = "ec2e3161de27091f18a831d9494e9c53378a34833af5c1dda35ad85849da18d6"
|
||||
dependencies = [
|
||||
"aes 0.9.3",
|
||||
"bzip2",
|
||||
@@ -14008,10 +14091,18 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
|
||||
dependencies = [
|
||||
"zstd-safe 7.3.0",
|
||||
"apache-avro",
|
||||
"arrow-array",
|
||||
"arrow-ipc",
|
||||
"arrow-schema",
|
||||
"bytes",
|
||||
"kafka-protocol",
|
||||
"parquet",
|
||||
"semver",
|
||||
"toml",
|
||||
"zstd 0.14.3",
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -14020,16 +14111,7 @@ version = "0.14.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70"
|
||||
dependencies = [
|
||||
"zstd-safe 8.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-safe"
|
||||
version = "7.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882"
|
||||
dependencies = [
|
||||
"zstd-sys",
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -14047,6 +14129,7 @@ version = "2.1.2+zstd.1.5.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
+7
-4
@@ -66,6 +66,7 @@ members = [
|
||||
"crates/io-metrics", # I/O metrics collection for performance analysis
|
||||
"crates/io-core", # Buffered Bytes, mmap-then-copy, and aligned pread I/O helpers
|
||||
"crates/zip", # ZIP file handling and compression
|
||||
"crates/zstd-compat", # Fixed codec for dependencies requiring zstd 0.13
|
||||
]
|
||||
resolver = "3"
|
||||
|
||||
@@ -260,7 +261,7 @@ aws-smithy-http-client = { default-features = false, version = "1.5.1" }
|
||||
aws-smithy-runtime-api = { version = "1.19.1" }
|
||||
aws-smithy-types = { version = "1.9.0" }
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "9.0.0"
|
||||
brotli = "9.1.0"
|
||||
clap = { version = "4.6.7" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.12.0"
|
||||
@@ -302,6 +303,7 @@ num_cpus = { version = "1.17.0" }
|
||||
nvml-wrapper = "0.13.0"
|
||||
object = { version = "=0.40.0", default-features = false }
|
||||
parking_lot = "0.13.0"
|
||||
same-file = "1.0.6"
|
||||
path-absolutize = "4.0.1"
|
||||
percent-encoding = "2.3.2"
|
||||
# Server-side QR rendering for TOTP enrollment, so neither the console nor the
|
||||
@@ -354,7 +356,7 @@ winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.1.0"
|
||||
zip = "9.1.2"
|
||||
zstd = "0.14.3"
|
||||
|
||||
# Observability and Metrics
|
||||
@@ -375,7 +377,7 @@ libunftp = { version = "0.23.1" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.64.1" }
|
||||
russh = { version = "0.65.0" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
@@ -384,7 +386,7 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.2" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.29.1", default-features = false }
|
||||
hotpath = { version = "0.29.2", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
@@ -424,6 +426,7 @@ strip = "none"
|
||||
|
||||
# Pin upstream fixes until compatible crates.io releases are available.
|
||||
[patch.crates-io]
|
||||
zstd = { path = "crates/zstd-compat" }
|
||||
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
|
||||
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
|
||||
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
|
||||
|
||||
@@ -100,6 +100,7 @@ when restoring `/rustfs/console` for a custom OEM build.
|
||||
## S3 API environment variables
|
||||
|
||||
- `RUSTFS_API_OBJECT_MAX_VERSIONS` caps the number of retained versions for a single object. It defaults to `9223372036854775807`, matching MinIO's practical-unlimited default. Set a positive integer to enforce a lower per-object metadata bound.
|
||||
- `RUSTFS_VEEAM_SOSAPI_ENABLE` serves Veeam SOSAPI `system.xml` and `capacity.xml` for an authenticated GET or HEAD of those exact bucket-root keys. It defaults to `true`. Set it to `false` to restore the normal object path, including deletion of a real object previously stored at that key.
|
||||
- `MINIO_API_OBJECT_MAX_VERSIONS` is accepted as a compatibility alias when the canonical RustFS variable is not set.
|
||||
|
||||
## Distributed endpoint locality
|
||||
|
||||
@@ -222,6 +222,16 @@ pub const ENV_RUSTFS_TLS_PATH: &str = "RUSTFS_TLS_PATH";
|
||||
/// Environment variable for server KMS enable.
|
||||
pub const ENV_RUSTFS_KMS_ENABLE: &str = "RUSTFS_KMS_ENABLE";
|
||||
|
||||
/// Environment variable for Veeam SOSAPI virtual system objects.
|
||||
///
|
||||
/// When true (the default), authenticated GET and HEAD of the two Veeam
|
||||
/// SOSAPI keys return generated XML. When false, those keys use the normal
|
||||
/// object path so an operator can remove a real object stored there earlier.
|
||||
pub const ENV_VEEAM_SOSAPI_ENABLE: &str = "RUSTFS_VEEAM_SOSAPI_ENABLE";
|
||||
|
||||
/// Default Veeam SOSAPI enablement.
|
||||
pub const DEFAULT_VEEAM_SOSAPI_ENABLE: bool = true;
|
||||
|
||||
/// Default KMS enable for server-side encryption
|
||||
/// This is the default value for enabling KMS encryption for server-side encryption.
|
||||
/// Default value: false
|
||||
|
||||
@@ -146,6 +146,7 @@ zip.workspace = true
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
|
||||
[dev-dependencies]
|
||||
same-file.workspace = true
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["test-util"] }
|
||||
rustfs-heal.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
||||
@@ -20,15 +20,14 @@
|
||||
//! offline before heal converges, a strict latest-listing quorum of 3 can only
|
||||
//! ever observe 2 copies, so ListObjectsV2 silently dropped the object even
|
||||
//! though GetObject (read quorum 2) still succeeded. Exposed by the flaky
|
||||
//! "Mixed-version rolling upgrade from rc.2" CI lane (run 33478999853); the
|
||||
//! product fix relaxes the listing's required object quorum by the number of
|
||||
//! set drives the listing could not consult (see
|
||||
//! `latest_listing_required_object_quorum` in
|
||||
//! `crates/ecstore/src/store/list_objects.rs`).
|
||||
//! "Mixed-version rolling upgrade from rc.2" CI lane (run 33478999853). The
|
||||
//! listing now follows MinIO's rule: an entry needs `ceil(asked/2)` copies
|
||||
//! among the online drives it asked, so three online drives need two copies
|
||||
//! (see `SetDisks::list_path` in `crates/ecstore/src/store/list_objects.rs`).
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestClusterEnvironment, init_logging};
|
||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use bytes::Bytes;
|
||||
use std::collections::HashSet;
|
||||
@@ -51,13 +50,13 @@ mod tests {
|
||||
format!("degraded-object-{idx:02}")
|
||||
}
|
||||
|
||||
async fn list_all_keys(client: &Client) -> Result<HashSet<String>, Box<dyn Error + Send + Sync>> {
|
||||
async fn list_all_keys(client: &Client, bucket: &str) -> Result<HashSet<String>, Box<dyn Error + Send + Sync>> {
|
||||
let mut keys = HashSet::new();
|
||||
let mut continuation_token: Option<String> = None;
|
||||
loop {
|
||||
let response = client
|
||||
.list_objects_v2()
|
||||
.bucket(BUCKET)
|
||||
.bucket(bucket)
|
||||
.set_continuation_token(continuation_token.clone())
|
||||
.send()
|
||||
.await?;
|
||||
@@ -147,7 +146,7 @@ mod tests {
|
||||
let expected: HashSet<String> = (0..OBJECT_COUNT).map(object_key).collect();
|
||||
let deadline = Instant::now() + LISTING_DEADLINE;
|
||||
let listed = loop {
|
||||
let listed = match list_all_keys(&client).await {
|
||||
let listed = match list_all_keys(&client, BUCKET).await {
|
||||
Ok(keys) => keys,
|
||||
Err(error) if Instant::now() < deadline => {
|
||||
info!("retrying degraded listing: {error}");
|
||||
@@ -171,4 +170,97 @@ mod tests {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 4-node x 4-drive cluster (one 16-drive set), #8828 shape:
|
||||
/// 1. PUT objects while the storage class is EC:8, so each is 8+8.
|
||||
/// 2. Restart with the EC:4 default, as after a storage-class change.
|
||||
/// 3. Stop two nodes (8 drives). Each object keeps eight data shards, so
|
||||
/// GET succeeds and ListObjectsV2 must list every key: the listing
|
||||
/// quorum is half of the eight drives it can ask.
|
||||
/// 4. Stop a third node. Four online drives are below the set's read
|
||||
/// quorum, so LIST must report 503 instead of a partial page.
|
||||
#[tokio::test]
|
||||
async fn ec8_objects_stay_listable_with_half_of_an_ec4_set_offline() -> TestResult {
|
||||
const EC8_BUCKET: &str = "degraded-listing-ec8";
|
||||
init_logging();
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, 4)).await?;
|
||||
cluster.set_env("RUSTFS_HEAL_ENABLED", "false");
|
||||
cluster.set_env("RUSTFS_SCANNER_ENABLED", "false");
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:8");
|
||||
cluster.start().await?;
|
||||
cluster.create_test_bucket(EC8_BUCKET).await?;
|
||||
let client = cluster.create_s3_client(0)?;
|
||||
for idx in 0..OBJECT_COUNT {
|
||||
let key = object_key(idx);
|
||||
client
|
||||
.put_object()
|
||||
.bucket(EC8_BUCKET)
|
||||
.key(&key)
|
||||
.body(Bytes::from(format!("ec8 listing payload {idx}")).into())
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format!("EC:8 PUT for {key} failed: {error}"))?;
|
||||
}
|
||||
|
||||
info!("restarting with the EC:4 default storage class");
|
||||
cluster.stop();
|
||||
// A later entry for the same variable wins when the node process starts.
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4");
|
||||
cluster.start().await?;
|
||||
let client = cluster.create_s3_client(0)?;
|
||||
|
||||
info!("stopping nodes 2 and 3 (eight of sixteen drives)");
|
||||
cluster.stop_node(2)?;
|
||||
cluster.stop_node(3)?;
|
||||
for idx in 0..OBJECT_COUNT {
|
||||
let key = object_key(idx);
|
||||
let deadline = Instant::now() + GET_RETRY_DEADLINE;
|
||||
loop {
|
||||
match client.get_object().bucket(EC8_BUCKET).key(&key).send().await {
|
||||
Ok(response) => {
|
||||
let body = response.body.collect().await?.into_bytes();
|
||||
assert_eq!(body.as_ref(), format!("ec8 listing payload {idx}").as_bytes());
|
||||
break;
|
||||
}
|
||||
Err(error) if Instant::now() < deadline => {
|
||||
info!("retrying EC:8 GET for {key}: {error}");
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
Err(error) => return Err(format!("EC:8 object {key} failed GET with eight drives: {error}").into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let expected: HashSet<String> = (0..OBJECT_COUNT).map(object_key).collect();
|
||||
let deadline = Instant::now() + LISTING_DEADLINE;
|
||||
loop {
|
||||
match list_all_keys(&client, EC8_BUCKET).await {
|
||||
Ok(listed) if listed == expected => break,
|
||||
Ok(listed) => assert!(
|
||||
Instant::now() < deadline,
|
||||
"EC:8 objects readable with eight drives stayed missing from ListObjectsV2: listed={listed:?}"
|
||||
),
|
||||
Err(error) => assert!(
|
||||
Instant::now() < deadline,
|
||||
"ListObjectsV2 kept failing with eight of sixteen drives online: {error}"
|
||||
),
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
|
||||
info!("stopping node 1 (four of sixteen drives remain)");
|
||||
cluster.stop_node(1)?;
|
||||
let deadline = Instant::now() + LISTING_DEADLINE;
|
||||
loop {
|
||||
match client.list_objects_v2().bucket(EC8_BUCKET).send().await {
|
||||
Err(error) if error.raw_response().is_some_and(|response| response.status().as_u16() == 503) => break,
|
||||
Err(error) => assert!(Instant::now() < deadline, "below the set read quorum LIST must report 503, got {error}"),
|
||||
Ok(response) => panic!("below the set read quorum LIST must fail, but listed {} keys", response.contents().len()),
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,18 +13,20 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, retrying_get_equals, unique_bucket,
|
||||
wait_for_ready, wait_until,
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, get_object_bytes, payload_for, put_object,
|
||||
retrying_get_equals, retrying_put, s3_probe_client, s3_probe_error_code, unique_bucket, wait_for_ready, wait_until,
|
||||
};
|
||||
use crate::chaos::{census_object_version_on_disk, signed_admin_post};
|
||||
use crate::common::{build_test_s3_config, init_logging};
|
||||
use crate::fault_proxy::FaultMode;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::operation::{get_object::GetObjectError, put_object::PutObjectError};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{Barrier, mpsc};
|
||||
use tokio::time::timeout;
|
||||
use tokio::time::{Instant, timeout};
|
||||
|
||||
#[tokio::test]
|
||||
async fn kill_and_restart_node_preserves_objects() -> TestResult {
|
||||
@@ -44,6 +46,133 @@ async fn kill_and_restart_node_preserves_objects() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ec4_partial_node_return_restores_read_write_quorum() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start_with_env(
|
||||
DistLayout::FourByFour,
|
||||
&[
|
||||
("RUSTFS_ERASURE_SET_DRIVE_COUNT", "16"),
|
||||
("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
let storage = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
|
||||
let storage: serde_json::Value = serde_json::from_str(&storage)?;
|
||||
let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
|
||||
assert_eq!(storage.disks.len(), 16);
|
||||
assert_eq!(storage.backend.total_sets, vec![1]);
|
||||
assert_eq!(storage.backend.drives_per_set, vec![16]);
|
||||
assert_eq!(storage.backend.standard_sc_parity, Some(4));
|
||||
|
||||
let bucket = unique_bucket("partial-ec4");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = s3_probe_client(&dist.client(0)?);
|
||||
let baseline = payload_for("partial-recovery/baseline", 8 * 1024 * 1024);
|
||||
put_object(&client, &bucket, "baseline.bin", baseline.clone()).await?;
|
||||
wait_until(
|
||||
Duration::from_secs(30),
|
||||
|| async {
|
||||
for node in &dist.cluster.nodes {
|
||||
for directory in &node.data_dirs {
|
||||
let census = census_object_version_on_disk(std::path::Path::new(directory), &bucket, "baseline.bin", None)?;
|
||||
if !census.is_complete() {
|
||||
return Ok(false);
|
||||
}
|
||||
assert_eq!(census.data_blocks, Some(12), "the object must require twelve data shards");
|
||||
assert_eq!(census.parity_blocks, Some(4), "the object must have four parity shards");
|
||||
assert!(!census.expected_part_numbers.is_empty(), "the baseline must use physical shards");
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
"the baseline is complete on all sixteen EC:4 drives",
|
||||
)
|
||||
.await?;
|
||||
|
||||
dist.cluster.stop_node(2)?;
|
||||
dist.cluster.stop_node(3)?;
|
||||
let read_error = timeout(Duration::from_secs(60), get_object_bytes(&client, &bucket, "baseline.bin"))
|
||||
.await?
|
||||
.expect_err("eight unavailable drives must prevent the baseline GET");
|
||||
assert!(
|
||||
matches!(
|
||||
s3_probe_error_code(read_error.as_ref()),
|
||||
Some("SlowDown" | "SlowDownRead" | "ServiceUnavailable")
|
||||
),
|
||||
"quorum loss must produce an availability error: {read_error:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
read_error
|
||||
.downcast_ref::<SdkError<GetObjectError>>()
|
||||
.and_then(SdkError::raw_response)
|
||||
.map(|response| response.status().as_u16()),
|
||||
Some(503),
|
||||
"read quorum loss must remain a service availability failure"
|
||||
);
|
||||
let write_error = timeout(Duration::from_secs(60), put_object(&client, &bucket, "unavailable.bin", baseline.clone()))
|
||||
.await?
|
||||
.expect_err("eight unavailable drives must prevent a successful PUT");
|
||||
assert!(
|
||||
matches!(s3_probe_error_code(write_error.as_ref()), Some("SlowDown" | "ServiceUnavailable")),
|
||||
"quorum loss must produce an availability error: {write_error:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
write_error
|
||||
.downcast_ref::<SdkError<PutObjectError>>()
|
||||
.and_then(SdkError::raw_response)
|
||||
.map(|response| response.status().as_u16()),
|
||||
Some(503),
|
||||
"write quorum loss must remain a service availability failure"
|
||||
);
|
||||
|
||||
// Recover only four drives. The remaining node must stay stopped while
|
||||
// every live coordinator serves old data and acknowledges new writes.
|
||||
dist.cluster.start_node_process(2).await?;
|
||||
let recovery_deadline = Instant::now() + Duration::from_secs(90);
|
||||
let mut recovered_objects = Vec::new();
|
||||
for node in 0..3 {
|
||||
assert!(dist.cluster.nodes[3].process.is_none(), "the fourth node must remain unavailable");
|
||||
let client = s3_probe_client(&dist.client(node)?);
|
||||
retrying_get_equals(
|
||||
&client,
|
||||
&bucket,
|
||||
"baseline.bin",
|
||||
&baseline,
|
||||
recovery_deadline.saturating_duration_since(Instant::now()),
|
||||
)
|
||||
.await?;
|
||||
let key = format!("partial-{node}.bin");
|
||||
let body = payload_for(&key, 1024 * 1024);
|
||||
retrying_put(
|
||||
&client,
|
||||
&bucket,
|
||||
&key,
|
||||
body.clone(),
|
||||
recovery_deadline.saturating_duration_since(Instant::now()),
|
||||
)
|
||||
.await?;
|
||||
recovered_objects.push((key, body));
|
||||
}
|
||||
for node in 0..3 {
|
||||
assert!(dist.cluster.nodes[3].process.is_none(), "reads must complete before all drives return");
|
||||
let client = s3_probe_client(&dist.client(node)?);
|
||||
assert_object_bytes(&client, &bucket, "baseline.bin", &baseline).await?;
|
||||
for (key, body) in &recovered_objects {
|
||||
assert_object_bytes(&client, &bucket, key, body).await?;
|
||||
}
|
||||
}
|
||||
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let client = s3_probe_client(&dist.client(3)?);
|
||||
assert_object_bytes(&client, &bucket, "baseline.bin", &baseline).await?;
|
||||
for (key, body) in &recovered_objects {
|
||||
assert_object_bytes(&client, &bucket, key, body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_cluster_restart_preserves_objects() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -1181,11 +1181,16 @@ pub(crate) async fn retrying_get_equals(
|
||||
)
|
||||
.into()),
|
||||
Err(error) => {
|
||||
if matches!(s3_probe_error_code(error.as_ref()), Some("SlowDown" | "ServiceUnavailable")) {
|
||||
Ok(false)
|
||||
} else {
|
||||
Err(error)
|
||||
}
|
||||
let retryable = match s3_probe_error_code(error.as_ref()) {
|
||||
Some("SlowDown" | "ServiceUnavailable") => true,
|
||||
// GET preserves this compatibility code for a lost data quorum.
|
||||
Some("SlowDownRead") => error
|
||||
.downcast_ref::<SdkError<GetObjectError>>()
|
||||
.and_then(SdkError::raw_response)
|
||||
.is_some_and(|response| response.status().as_u16() == 503),
|
||||
_ => false,
|
||||
};
|
||||
if retryable { Ok(false) } else { Err(error) }
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -1324,6 +1329,62 @@ mod retry_tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn retrying_get_allows_read_quorum_recovery() -> TestResult {
|
||||
use aws_sdk_s3::config::Region;
|
||||
use aws_smithy_http_client::test_util::{ReplayEvent, StaticReplayClient};
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
|
||||
let endpoint = "http://read-quorum.example.invalid";
|
||||
let event = |status: u16, body: &'static str| {
|
||||
ReplayEvent::new(
|
||||
http::Request::builder()
|
||||
.method(Method::GET)
|
||||
.uri(format!("{endpoint}/retry-probe/key?x-id=GetObject"))
|
||||
.body(SdkBody::empty())
|
||||
.expect("build replay request"),
|
||||
http::Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len())
|
||||
.body(SdkBody::from(body))
|
||||
.expect("build replay response"),
|
||||
)
|
||||
};
|
||||
for status in [503, 403, 500] {
|
||||
let transport = StaticReplayClient::new(vec![
|
||||
event(status, "<Error><Code>SlowDownRead</Code></Error>"),
|
||||
event(200, "expected bytes"),
|
||||
]);
|
||||
let client = Client::from_conf(
|
||||
aws_sdk_s3::Config::builder()
|
||||
.behavior_version_latest()
|
||||
.credentials_provider(Credentials::new(FAKE_ACCESS_KEY, FAKE_SECRET_KEY, None, None, "read-quorum-test"))
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(endpoint)
|
||||
.force_path_style(true)
|
||||
.retry_config(RetryConfig::disabled())
|
||||
.http_client(transport.clone())
|
||||
.build(),
|
||||
);
|
||||
let result = retrying_get_equals(&client, "retry-probe", "key", b"expected bytes", Duration::from_secs(5)).await;
|
||||
if status == 503 {
|
||||
result?;
|
||||
assert_eq!(
|
||||
transport.actual_requests().count(),
|
||||
2,
|
||||
"the probe must retry the read-quorum error exactly once"
|
||||
);
|
||||
transport.assert_requests_match(&[]);
|
||||
} else {
|
||||
let error = result.expect_err("the compatibility code must not hide a non-503 failure");
|
||||
assert_eq!(s3_probe_error_code(error.as_ref()), Some("SlowDownRead"));
|
||||
assert_eq!(transport.actual_requests().count(), 1, "non-availability errors must fail immediately");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_polls_distinguish_committed_and_replicated_keys() -> TestResult {
|
||||
let target = FakeS3Target::start().await?;
|
||||
|
||||
@@ -31,6 +31,7 @@ mod multipart_fence_acceptance_test;
|
||||
mod multipart_rpc_delay_test;
|
||||
mod object_lock_test;
|
||||
mod observability_test;
|
||||
mod publication_log;
|
||||
mod replication_delete_marker_test;
|
||||
mod replication_quota_test;
|
||||
mod s3_basic_test;
|
||||
|
||||
@@ -1638,6 +1638,191 @@ async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation(
|
||||
"refused late packets must not recreate staging"
|
||||
);
|
||||
}
|
||||
late_part_packets_preserve_retired_and_fresh_uploads().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
|
||||
for (method, settles) in [("PreparePartTransaction", false), ("SettlePartTransaction", true)] {
|
||||
for recreate_bucket in [false, true] {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let key = "late-part-transaction";
|
||||
let previous = payload_for("committed-before-late-part", 4096);
|
||||
clients[0]
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(previous.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let upload = clients[0].create_multipart_upload().bucket(&bucket).key(key).send().await?;
|
||||
let upload_id = upload.upload_id().ok_or("missing original upload ID")?;
|
||||
clients[0]
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(vec![0x41; 9 * 1024 * 1024]))
|
||||
.send()
|
||||
.await?;
|
||||
let path = upload_path_scope(&bucket, key, upload_id)?;
|
||||
let hash = sha256_hex(format!("{bucket}/{key}").as_bytes());
|
||||
let roots = cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.flat_map(|node| &node.data_dirs)
|
||||
.map(|root| Path::new(root).join(RUSTFS_META_MULTIPART_BUCKET).join(&path))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(roots.len(), 4);
|
||||
let original = multipart_sources(&cluster, Some(&hash))?;
|
||||
assert_eq!(original.len(), 4);
|
||||
assert!(
|
||||
original.values().all(|source| source
|
||||
.files
|
||||
.keys()
|
||||
.any(|path| path.file_name().and_then(|name| name.to_str()) == Some("part.1"))),
|
||||
"all selected peers originally own the acknowledged part"
|
||||
);
|
||||
let scope = upload_scope(upload_id)?;
|
||||
// Retain only the original attempt; recovery of the same upload
|
||||
// must be able to issue fresh settlement requests through the gate.
|
||||
let pauses = proxies[1..3]
|
||||
.iter()
|
||||
.map(|proxy| proxy.pause_late(&[method], 1, &scope))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let replacement = clients[0]
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(vec![0x62; 8 * 1024 * 1024]));
|
||||
let caller = tokio::spawn(async move { replacement.send().await });
|
||||
wait_for_faults(&pauses, &[method]).await?;
|
||||
let failure = tokio::time::timeout(DEADLINE, caller)
|
||||
.await??
|
||||
.expect_err("two retained peers cannot acknowledge part replacement");
|
||||
assert_unavailable(&failure, "storage_write_quorum", None);
|
||||
assert!(pauses.iter().all(|pause| pause.hits() == 1 && pause.responses() == 0));
|
||||
let expected_actions = if settles { (1, 0) } else { (0, 0) };
|
||||
assert!(
|
||||
pauses.iter().all(|pause| pause.retained_settle_actions() == expected_actions),
|
||||
"settlement capture must be a real Commit, not Rollback"
|
||||
);
|
||||
abort_after_fault(&clients[0], &bucket, key, upload_id, false).await?;
|
||||
assert!(
|
||||
roots.iter().all(|root| !root.exists()),
|
||||
"ordinary Abort physically retires all original roots"
|
||||
);
|
||||
assert!(multipart_sources(&cluster, Some(&hash))?.is_empty());
|
||||
for client in &clients {
|
||||
assert_object_bytes_after_fault(client, &bucket, key, &previous).await?;
|
||||
}
|
||||
let protected = if recreate_bucket {
|
||||
clients[0].delete_object().bucket(&bucket).key(key).send().await?;
|
||||
clients[0].delete_bucket().bucket(&bucket).send().await?;
|
||||
clients[1].create_bucket().bucket(&bucket).send().await?;
|
||||
let fresh = payload_for("recreated-before-late-part", 4096);
|
||||
clients[1]
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(fresh.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
fresh
|
||||
} else {
|
||||
previous
|
||||
};
|
||||
let fresh = clients[2].create_multipart_upload().bucket(&bucket).key(key).send().await?;
|
||||
let fresh_id = fresh.upload_id().ok_or("missing fresh upload ID")?;
|
||||
let fresh_body = payload_for("fresh-active-part", 4096);
|
||||
let fresh_part = clients[2]
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(fresh_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(fresh_body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let before = multipart_sources(&cluster, Some(&hash))?;
|
||||
assert_eq!(before.len(), 4, "the new upload exists on every original participant");
|
||||
for pause in &pauses {
|
||||
pause.release();
|
||||
}
|
||||
wait_until(
|
||||
DEADLINE,
|
||||
|| async {
|
||||
Ok(pauses.iter().all(|pause| {
|
||||
pause.responses() == pause.hits()
|
||||
&& if settles {
|
||||
pause.successful_mutation_replies() == pause.hits() && pause.refused_mutations() == 0
|
||||
} else {
|
||||
pause.refused_mutations() == pause.hits() && pause.successful_mutation_replies() == 0
|
||||
}
|
||||
}))
|
||||
},
|
||||
"every original part packet returns its typed authenticated terminal outcome",
|
||||
)
|
||||
.await?;
|
||||
assert!(roots.iter().all(|root| !root.exists()), "late part packets cannot recreate retired roots");
|
||||
assert!(
|
||||
multipart_sources(&cluster, Some(&hash))? == before,
|
||||
"late packets cannot change a fresh upload's shard or control bytes"
|
||||
);
|
||||
for client in &clients {
|
||||
assert_object_bytes_after_fault(client, &bucket, key, &protected).await?;
|
||||
let listed = client
|
||||
.list_parts()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(fresh_id)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.parts().len(), 1);
|
||||
assert_eq!(listed.parts()[0].e_tag(), fresh_part.e_tag());
|
||||
let missing = client
|
||||
.list_parts()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("old upload remains retired");
|
||||
assert_eq!(missing.raw_response().expect("old upload service response").status().as_u16(), 404);
|
||||
assert_eq!(missing.as_service_error().and_then(|error| error.meta().code()), Some("NoSuchUpload"));
|
||||
}
|
||||
clients[2]
|
||||
.complete_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(fresh_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(fresh_part.e_tag().ok_or("missing fresh part ETag")?)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
for client in &clients {
|
||||
assert_object_bytes_after_fault(client, &bucket, key, &fresh_body).await?;
|
||||
}
|
||||
assert!(multipart_sources(&cluster, Some(&hash))?.is_empty());
|
||||
assert_no_uploads_after_fault(&clients, &bucket).await?;
|
||||
eprintln!(
|
||||
"MPU late part: method={method}, recreate_bucket={recreate_bucket}, retired_roots=4, delivered=2, refused={}, successful_replies={}, fresh_sources_unchanged=4, verified_readers=4",
|
||||
pauses.iter().map(RpcPause::refused_mutations).sum::<usize>(),
|
||||
pauses.iter().map(RpcPause::successful_mutation_replies).sum::<usize>()
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,413 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Follow the same append-only log files without rescanning previous JSON rows.
|
||||
|
||||
use same_file::Handle;
|
||||
use serde::{Deserialize, Deserializer};
|
||||
use serde_json::Value;
|
||||
use std::fs::File;
|
||||
use std::io::{self, Read};
|
||||
use std::path::PathBuf;
|
||||
|
||||
// Captured RPC spans already produce valid rows above 2 MiB. Keep complete
|
||||
// rows bounded independently from the much smaller selected-event history.
|
||||
const MAX_LINE_BYTES: usize = 4 << 20;
|
||||
const MAX_EVENTS: usize = 128;
|
||||
const MAX_EVENT_BYTES: usize = 1 << 20;
|
||||
|
||||
fn invalid(reason: &'static str) -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, reason)
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize)]
|
||||
struct Selector {
|
||||
#[serde(default)]
|
||||
event: Value,
|
||||
#[serde(default)]
|
||||
op: Value,
|
||||
#[serde(default)]
|
||||
bucket: Value,
|
||||
#[serde(default)]
|
||||
object: Value,
|
||||
}
|
||||
|
||||
impl Selector {
|
||||
fn matches(&self, bucket: &str, object: &str) -> bool {
|
||||
self.event.as_str() == Some("object_namespace_lock")
|
||||
&& self.op.as_str() == Some("data_movement_publication")
|
||||
&& self.bucket.as_str() == Some(bucket)
|
||||
&& self.object.as_str() == Some(object)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Probe {
|
||||
#[serde(default)]
|
||||
event: Value,
|
||||
#[serde(default)]
|
||||
op: Value,
|
||||
#[serde(default)]
|
||||
bucket: Value,
|
||||
#[serde(default)]
|
||||
object: Value,
|
||||
// A present null is not a missing fields object. The old reader uses the
|
||||
// root only when this key is absent, including for non-object JSON values.
|
||||
#[serde(default, deserialize_with = "present_fields")]
|
||||
fields: Option<Option<Selector>>,
|
||||
}
|
||||
|
||||
fn present_fields<'de, D: Deserializer<'de>>(d: D) -> Result<Option<Option<Selector>>, D::Error> {
|
||||
Option::<Selector>::deserialize(d).map(Some)
|
||||
}
|
||||
|
||||
impl Probe {
|
||||
fn matches(&self, bucket: &str, object: &str) -> bool {
|
||||
match &self.fields {
|
||||
Some(Some(fields)) => fields.matches(bucket, object),
|
||||
Some(None) => false,
|
||||
None => {
|
||||
self.event.as_str() == Some("object_namespace_lock")
|
||||
&& self.op.as_str() == Some("data_movement_publication")
|
||||
&& self.bucket.as_str() == Some(bucket)
|
||||
&& self.object.as_str() == Some(object)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn select(line: &[u8], bucket: &str, object: &str, full_decodes: &mut u64) -> io::Result<Option<Value>> {
|
||||
let text = std::str::from_utf8(line).map_err(|_| invalid("invalid UTF-8 in a completed log row"))?;
|
||||
// Skip unknown fields without allocating a Value tree for every RPC trace.
|
||||
// A shape/duplicate-field error falls back to the original Value parser;
|
||||
// it must not discard valid escaped strings or last-key-wins JSON.
|
||||
if let Ok(probe) = serde_json::from_str::<Probe>(text)
|
||||
&& !probe.matches(bucket, object)
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
*full_decodes += 1;
|
||||
let Ok(row) = serde_json::from_str::<Value>(text) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let fields = row.get("fields").unwrap_or(&row);
|
||||
if fields.get("event").and_then(Value::as_str) != Some("object_namespace_lock")
|
||||
|| fields.get("op").and_then(Value::as_str) != Some("data_movement_publication")
|
||||
|| fields.get("bucket").and_then(Value::as_str) != Some(bucket)
|
||||
|| fields.get("object").and_then(Value::as_str) != Some(object)
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let mut event = fields.clone();
|
||||
event["timestamp"] = row["timestamp"].clone();
|
||||
Ok(Some(event))
|
||||
}
|
||||
|
||||
struct Tail {
|
||||
path: PathBuf,
|
||||
handle: Handle,
|
||||
offset: u64,
|
||||
pending: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Tail {
|
||||
fn open(path: PathBuf) -> io::Result<Self> {
|
||||
let file = File::open(&path)?;
|
||||
if !file.metadata()?.is_file() {
|
||||
return Err(invalid("publication log is not a regular file"));
|
||||
}
|
||||
Ok(Self {
|
||||
path,
|
||||
handle: Handle::from_file(file)?,
|
||||
offset: 0,
|
||||
pending: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn check_identity(&self) -> io::Result<()> {
|
||||
if Handle::from_path(&self.path)? != self.handle {
|
||||
return Err(invalid("publication log file was replaced"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn poll(&mut self, bytes_read: &mut u64, mut accept: impl FnMut(&[u8]) -> io::Result<()>) -> io::Result<()> {
|
||||
self.check_identity()?;
|
||||
let end = self.handle.as_file().metadata()?.len();
|
||||
if end < self.offset {
|
||||
return Err(invalid("publication log file was truncated"));
|
||||
}
|
||||
// A fixed high-water snapshot bounds each poll even if writers append
|
||||
// faster than the reader. Bytes after it remain for the next poll.
|
||||
let mut chunk = [0_u8; 64 * 1024];
|
||||
while self.offset < end {
|
||||
let size = usize::try_from((end - self.offset).min(64 * 1024)).expect("a log chunk fits usize");
|
||||
self.handle.as_file_mut().read_exact(&mut chunk[..size])?;
|
||||
self.offset += u64::try_from(size).expect("a bounded log read fits u64");
|
||||
*bytes_read += u64::try_from(size).expect("a bounded log read fits u64");
|
||||
let appended_at = self.pending.len();
|
||||
self.pending.extend_from_slice(&chunk[..size]);
|
||||
let mut start = 0;
|
||||
for end in self.pending[appended_at..]
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(i, byte)| (*byte == b'\n').then_some(appended_at + i + 1))
|
||||
{
|
||||
if end - start > MAX_LINE_BYTES {
|
||||
return Err(invalid("publication log row exceeds its fixed bound"));
|
||||
}
|
||||
accept(&self.pending[start..end])?;
|
||||
start = end;
|
||||
}
|
||||
self.pending.drain(..start);
|
||||
if self.pending.len() > MAX_LINE_BYTES {
|
||||
return Err(invalid("unfinished publication log row exceeds its fixed bound"));
|
||||
}
|
||||
}
|
||||
self.check_identity()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct PublicationEvents {
|
||||
tails: Vec<Tail>,
|
||||
bucket: String,
|
||||
object: String,
|
||||
events: Vec<Value>,
|
||||
per_file: Vec<Vec<Value>>,
|
||||
event_count: usize,
|
||||
event_bytes: usize,
|
||||
bytes_read: u64,
|
||||
full_decodes: u64,
|
||||
failed: bool,
|
||||
}
|
||||
|
||||
impl PublicationEvents {
|
||||
pub(super) fn new(paths: Vec<PathBuf>, bucket: &str, object: &str) -> io::Result<Self> {
|
||||
if paths.is_empty() || paths.len() > 4 {
|
||||
return Err(invalid("publication capture requires one to four log files"));
|
||||
}
|
||||
let tails: Vec<Tail> = paths.into_iter().map(Tail::open).collect::<io::Result<_>>()?;
|
||||
let mut reader = Self {
|
||||
per_file: vec![Vec::new(); tails.len()],
|
||||
tails,
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
events: Vec::new(),
|
||||
event_bytes: 0,
|
||||
event_count: 0,
|
||||
bytes_read: 0,
|
||||
full_decodes: 0,
|
||||
failed: false,
|
||||
};
|
||||
// Prime before holding remote DELETE or starting a movement attempt.
|
||||
// Retain earlier matching events too: the caller still applies its
|
||||
// original timestamp boundary, and evidence keeps the complete history.
|
||||
reader.poll()?;
|
||||
Ok(reader)
|
||||
}
|
||||
|
||||
pub(super) fn poll(&mut self) -> io::Result<&[Value]> {
|
||||
if self.failed {
|
||||
return Err(invalid("publication log reader was already invalidated"));
|
||||
}
|
||||
let result = (|| {
|
||||
for (tail, history) in self.tails.iter_mut().zip(&mut self.per_file) {
|
||||
tail.poll(&mut self.bytes_read, |line| {
|
||||
if let Some(event) = select(line, &self.bucket, &self.object, &mut self.full_decodes)? {
|
||||
let bytes = serde_json::to_vec(&event).map_err(io::Error::other)?.len();
|
||||
if self.event_count >= MAX_EVENTS || bytes > MAX_EVENT_BYTES - self.event_bytes {
|
||||
return Err(invalid("publication event history exceeds its fixed bound"));
|
||||
}
|
||||
self.event_bytes += bytes;
|
||||
self.event_count += 1;
|
||||
history.push(event);
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
// Preserve the old file-then-row order, even when a later poll appends
|
||||
// to an earlier file. Only bounded selected history is copied here.
|
||||
self.events = self.per_file.iter().flatten().cloned().collect();
|
||||
if let Err(error) = result {
|
||||
self.failed = true;
|
||||
return Err(error);
|
||||
}
|
||||
Ok(&self.events)
|
||||
}
|
||||
|
||||
/// Diagnostics only: after a read error this history can be incomplete.
|
||||
pub(super) fn cached(&self) -> &[Value] {
|
||||
&self.events
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
use std::io::Write;
|
||||
|
||||
fn event(id: &str) -> Value {
|
||||
json!({"timestamp":"2026-10-26T00:00:00Z","event":"object_namespace_lock","op":"data_movement_publication","bucket":"bucket","object":"object","lock_attempt_id":id,"state":"acquiring","extra":{"kept":[1,2]}})
|
||||
}
|
||||
fn append(path: &std::path::Path, raw: &[u8]) {
|
||||
std::fs::OpenOptions::new()
|
||||
.append(true)
|
||||
.open(path)
|
||||
.expect("open append log")
|
||||
.write_all(raw)
|
||||
.expect("append log bytes");
|
||||
}
|
||||
fn legacy(raw: &[u8]) -> Vec<Value> {
|
||||
std::str::from_utf8(raw)
|
||||
.expect("fixture UTF-8")
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
let row: Value = serde_json::from_str(line).ok()?;
|
||||
let fields = row.get("fields").unwrap_or(&row);
|
||||
if fields.get("event").and_then(Value::as_str) == Some("object_namespace_lock")
|
||||
&& fields.get("op").and_then(Value::as_str) == Some("data_movement_publication")
|
||||
&& fields.get("bucket").and_then(Value::as_str) == Some("bucket")
|
||||
&& fields.get("object").and_then(Value::as_str) == Some("object")
|
||||
{
|
||||
let mut result = fields.clone();
|
||||
result["timestamp"] = row["timestamp"].clone();
|
||||
Some(result)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn appended_events_keep_full_fields_without_history_rescans() {
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
let mut raw = (json!({"message":"unrelated trace","payload":"x".repeat(4096)}).to_string() + "\n")
|
||||
.repeat(64)
|
||||
.into_bytes();
|
||||
raw.extend_from_slice(
|
||||
format!("{}\n", json!({"event":"object_namespace_lock","op":"get_object","bucket":"bucket","object":"object","spans":"x".repeat(2401832)})).as_bytes(),
|
||||
);
|
||||
raw.extend_from_slice(format!("{}\n", event("first")).as_bytes());
|
||||
std::fs::write(file.path(), &raw).expect("seed log");
|
||||
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("prime log");
|
||||
assert_eq!(reader.cached(), legacy(&raw));
|
||||
assert_eq!(reader.full_decodes, 1);
|
||||
let bytes = reader.bytes_read;
|
||||
assert_eq!(reader.poll().expect("unchanged poll"), legacy(&raw));
|
||||
assert_eq!(reader.bytes_read, bytes);
|
||||
let next = format!("{}\n", event("second"));
|
||||
append(file.path(), next.as_bytes());
|
||||
raw.extend_from_slice(next.as_bytes());
|
||||
assert_eq!(reader.poll().expect("new rows"), legacy(&raw));
|
||||
assert_eq!(reader.bytes_read, u64::try_from(raw.len()).expect("fixture length"));
|
||||
assert_eq!(reader.full_decodes, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn later_appends_preserve_the_old_file_then_row_order() {
|
||||
let first = tempfile::NamedTempFile::new().expect("first log");
|
||||
let second = tempfile::NamedTempFile::new().expect("second log");
|
||||
std::fs::write(first.path(), format!("{}\n", event("a1"))).expect("seed first");
|
||||
std::fs::write(second.path(), format!("{}\n", event("b1"))).expect("seed second");
|
||||
let mut reader = PublicationEvents::new(vec![first.path().to_owned(), second.path().to_owned()], "bucket", "object")
|
||||
.expect("prime both");
|
||||
append(first.path(), format!("{}\n", event("a2")).as_bytes());
|
||||
let mut expected = legacy(&std::fs::read(first.path()).expect("first bytes"));
|
||||
expected.extend(legacy(&std::fs::read(second.path()).expect("second bytes")));
|
||||
assert_eq!(reader.poll().expect("append first after second was read"), expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn root_nested_escaped_and_duplicate_fields_match_the_old_reader() {
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
let root = event("root");
|
||||
let mut nested = json!({"timestamp":root["timestamp"],"fields":root});
|
||||
nested["fields"]["extra"] = json!({"unicode":"é"});
|
||||
let null = json!({"fields":null,"event":"object_namespace_lock","op":"data_movement_publication","bucket":"bucket","object":"object"});
|
||||
let escaped = root
|
||||
.to_string()
|
||||
.replace("object_namespace_lock", "\\u006fbject_namespace_lock");
|
||||
let duplicate = root.to_string().replacen("{", "{\"event\":\"earlier-value\",", 1);
|
||||
let raw = format!("{root}\n{nested}\n{null}\n{escaped}\n{duplicate}\nnot json\n{{\"fields\":[]}}\n");
|
||||
std::fs::write(file.path(), &raw).expect("mixed log");
|
||||
let reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("probe and fallback");
|
||||
assert_eq!(reader.cached(), legacy(raw.as_bytes()));
|
||||
assert_eq!(reader.cached().len(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incomplete_rows_and_split_utf8_wait_for_the_newline() {
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
let mut row = event("partial");
|
||||
row["extra"] = json!("é");
|
||||
let raw = format!("{row}\n").into_bytes();
|
||||
let cut = raw.iter().position(|b| *b == 0xc3).expect("UTF-8 prefix") + 1;
|
||||
std::fs::write(file.path(), &raw[..cut]).expect("partial code point");
|
||||
let mut reader =
|
||||
PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("incomplete row is pending");
|
||||
assert!(reader.cached().is_empty());
|
||||
append(file.path(), &raw[cut..raw.len() - 1]);
|
||||
assert!(reader.poll().expect("complete JSON without delimiter").is_empty());
|
||||
append(file.path(), b"\n");
|
||||
assert_eq!(reader.poll().expect("published row"), legacy(&raw));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncation_is_sticky_even_after_the_file_grows_again() {
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
std::fs::write(file.path(), format!("{}\n", event("first"))).expect("seed log");
|
||||
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("prime log");
|
||||
file.as_file().set_len(0).expect("truncate");
|
||||
assert!(reader.poll().is_err());
|
||||
append(file.path(), format!("{}\n", event("longer replacement")).as_bytes());
|
||||
assert!(reader.poll().is_err());
|
||||
assert_eq!(reader.cached().len(), 1);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn a_replaced_path_does_not_switch_to_the_new_log() {
|
||||
let dir = tempfile::tempdir().expect("log dir");
|
||||
let path = dir.path().join("node.log");
|
||||
std::fs::write(&path, b"").expect("log file");
|
||||
let mut reader = PublicationEvents::new(vec![path.clone()], "bucket", "object").expect("open log");
|
||||
std::fs::rename(&path, dir.path().join("retired.log")).expect("replace log");
|
||||
std::fs::write(&path, format!("{}\n", event("other-generation"))).expect("replacement");
|
||||
assert!(reader.poll().is_err());
|
||||
assert!(reader.cached().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_utf8_and_oversized_or_excessive_records_fail_closed() {
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("empty log");
|
||||
append(file.path(), b"\xff\n");
|
||||
assert!(reader.poll().is_err());
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("empty log");
|
||||
append(file.path(), &vec![b'x'; MAX_LINE_BYTES + 1]);
|
||||
assert!(reader.poll().is_err());
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("empty log");
|
||||
for _ in 0..=MAX_EVENTS {
|
||||
append(file.path(), format!("{}\n", event("attempt")).as_bytes());
|
||||
}
|
||||
assert!(reader.poll().is_err());
|
||||
assert_eq!(reader.cached().len(), MAX_EVENTS);
|
||||
assert!(reader.poll().is_err());
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("empty log");
|
||||
let mut large = event("large");
|
||||
large["extra"] = json!("x".repeat(600 * 1024));
|
||||
for _ in 0..2 {
|
||||
append(file.path(), format!("{large}\n").as_bytes());
|
||||
}
|
||||
assert!(reader.poll().is_err());
|
||||
assert_eq!(reader.cached().len(), 1);
|
||||
assert!(reader.poll().is_err());
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ use super::harness::{
|
||||
rebalance_status_json, sha256_hex, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
|
||||
wait_for_rebalance_complete,
|
||||
};
|
||||
use super::publication_log::PublicationEvents;
|
||||
use super::scanner_heal_topology_fixture::{
|
||||
MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL, STAGE_AFTER_RESPONSIBILITY_PERSISTED, STAGE_BEFORE_SOURCE_CLEANUP,
|
||||
TopologyRun, assert_bucket_durability_strict, assert_process_durability_strict, require_test_hooks,
|
||||
@@ -174,29 +175,6 @@ async fn transition_version(
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_events(run: &TopologyRun, bucket: &str) -> TestResult<Vec<Value>> {
|
||||
let mut events = Vec::new();
|
||||
for path in run.node_log_paths() {
|
||||
let log = std::fs::read_to_string(path)?;
|
||||
for line in log.lines() {
|
||||
let Ok(row) = serde_json::from_str::<Value>(line) else {
|
||||
continue;
|
||||
};
|
||||
let fields = row.get("fields").unwrap_or(&row);
|
||||
if fields.get("event").and_then(Value::as_str) == Some("object_namespace_lock")
|
||||
&& fields.get("op").and_then(Value::as_str) == Some("data_movement_publication")
|
||||
&& fields.get("bucket").and_then(Value::as_str) == Some(bucket)
|
||||
&& fields.get("object").and_then(Value::as_str) == Some(KEY)
|
||||
{
|
||||
let mut event = fields.clone();
|
||||
event["timestamp"] = row["timestamp"].clone();
|
||||
events.push(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(events)
|
||||
}
|
||||
|
||||
fn event_time(event: &Value) -> TestResult<OffsetDateTime> {
|
||||
Ok(OffsetDateTime::parse(
|
||||
event["timestamp"].as_str().ok_or("lock event omitted its timestamp")?,
|
||||
@@ -204,10 +182,10 @@ fn event_time(event: &Value) -> TestResult<OffsetDateTime> {
|
||||
)?)
|
||||
}
|
||||
|
||||
async fn pending_publication(run: &TopologyRun, bucket: &str, started: OffsetDateTime) -> TestResult<Value> {
|
||||
async fn pending_publication(events: &mut PublicationEvents, started: OffsetDateTime) -> TestResult<Value> {
|
||||
loop {
|
||||
let events = lock_events(run, bucket)?;
|
||||
for event in &events {
|
||||
let observed = events.poll()?;
|
||||
for event in observed {
|
||||
if event_time(event)? < started {
|
||||
continue;
|
||||
}
|
||||
@@ -216,7 +194,7 @@ async fn pending_publication(run: &TopologyRun, bucket: &str, started: OffsetDat
|
||||
"mover acquired its publication lock while remote DELETE was held"
|
||||
);
|
||||
if event["state"] == "acquiring"
|
||||
&& !events
|
||||
&& !observed
|
||||
.iter()
|
||||
.any(|later| later["lock_attempt_id"] == event["lock_attempt_id"] && later["state"] != "acquiring")
|
||||
{
|
||||
@@ -228,14 +206,14 @@ async fn pending_publication(run: &TopologyRun, bucket: &str, started: OffsetDat
|
||||
}
|
||||
}
|
||||
|
||||
async fn acquired_publication(run: &TopologyRun, bucket: &str, pending: &Value) -> TestResult<Value> {
|
||||
async fn acquired_publication(events: &mut PublicationEvents, pending: &Value) -> TestResult<Value> {
|
||||
loop {
|
||||
for event in lock_events(run, bucket)? {
|
||||
for event in events.poll()? {
|
||||
if event["lock_attempt_id"] != pending["lock_attempt_id"] {
|
||||
continue;
|
||||
}
|
||||
match event["state"].as_str() {
|
||||
Some("acquired") => return Ok(event),
|
||||
Some("acquired") => return Ok(event.clone()),
|
||||
Some("failed" | "cancelled") => return Err(format!("held publication attempt did not acquire: {event}").into()),
|
||||
_ => {}
|
||||
}
|
||||
@@ -403,6 +381,7 @@ async fn tier_delete_during_movement(decommission: bool) -> TestResult {
|
||||
// started a worker. Install a separate release nonce before holding DELETE.
|
||||
let cleanup_barrier = run.movement_barrier.next_phase();
|
||||
cleanup_barrier.arm_movement(operation, &bucket, KEY, STAGE_BEFORE_SOURCE_CLEANUP)?;
|
||||
let mut publication_events = PublicationEvents::new(run.node_log_paths(), &bucket, KEY)?;
|
||||
let mut delete_gate = target.hold_delete_object(&remote_bucket, &old_remote.key);
|
||||
// The public DELETE persists a free owner and enqueues its remote cleanup.
|
||||
let client = run.dist.client(0)?;
|
||||
@@ -470,11 +449,11 @@ async fn tier_delete_during_movement(decommission: bool) -> TestResult {
|
||||
})?;
|
||||
drop(decommission_start);
|
||||
let movement_acknowledged_at = OffsetDateTime::now_utc();
|
||||
let pending = timeout_at(overlap_deadline, pending_publication(&run, &bucket, movement_started))
|
||||
let pending = timeout_at(overlap_deadline, pending_publication(&mut publication_events, movement_started))
|
||||
.await
|
||||
.map_err(|error| format!(
|
||||
"G-TIER {operation} publication pending timed out within DELETE overlap: {error}; startup={startup}; lock_events={:?}; cleanup_barrier={:?}",
|
||||
lock_events(&run, &bucket), cleanup_barrier.reached_observations()
|
||||
publication_events.cached(), cleanup_barrier.reached_observations()
|
||||
))?
|
||||
.map_err(|error| format!("G-TIER {operation} publication pending failed within DELETE overlap: {error}"))?;
|
||||
// Positive gate entry, exact physical ownership and the pending lock attempt
|
||||
@@ -484,11 +463,11 @@ async fn tier_delete_during_movement(decommission: bool) -> TestResult {
|
||||
let released_at = OffsetDateTime::now_utc();
|
||||
let delete_overlap_elapsed_ms = overlap_started.elapsed().as_millis();
|
||||
drop(delete_gate);
|
||||
let acquired = timeout(DELETE_OVERLAP_TIMEOUT, acquired_publication(&run, &bucket, &pending))
|
||||
let acquired = timeout(DELETE_OVERLAP_TIMEOUT, acquired_publication(&mut publication_events, &pending))
|
||||
.await
|
||||
.map_err(|error| format!(
|
||||
"G-TIER {operation} publication acquired after DELETE release timed out: {error}; pending={pending}; lock_events={:?}; cleanup_barrier={:?}",
|
||||
lock_events(&run, &bucket), cleanup_barrier.reached_observations()
|
||||
publication_events.cached(), cleanup_barrier.reached_observations()
|
||||
))?
|
||||
.map_err(|error| format!("G-TIER {operation} publication acquired after DELETE release failed: {error}"))?;
|
||||
assert!(event_time(&pending)? >= movement_started && event_time(&pending)? < released_at);
|
||||
@@ -496,6 +475,7 @@ async fn tier_delete_during_movement(decommission: bool) -> TestResult {
|
||||
event_time(&acquired)? >= released_at,
|
||||
"the same publication attempt must acquire after DELETE is released"
|
||||
);
|
||||
drop(publication_events);
|
||||
let reached = cleanup_barrier
|
||||
.wait_reached_during(
|
||||
&run.dist.cluster,
|
||||
|
||||
@@ -1129,12 +1129,14 @@ mod tests {
|
||||
/// and the marker plus the cursor tag.
|
||||
///
|
||||
/// Bug Reference: backlog#1047
|
||||
/// The V2 continuation token appends a `[rustfs_cache:...]` cursor tag to the
|
||||
/// last returned key. The orchestration layer passed that raw string to
|
||||
/// `forward_past`, so any key whose byte after the shared stem sorts below
|
||||
/// `[` (0x5B) — `.`, `/`, `-`, digits, uppercase — was silently dropped on
|
||||
/// the next page: with keys `a`, `a.txt`, `zz` and max_keys=1, page 2
|
||||
/// returned `zz` and `a.txt` was never listed.
|
||||
/// Earlier releases appended a `[rustfs_cache:...]` cursor tag to the last
|
||||
/// returned key inside the V2 continuation token. The orchestration layer
|
||||
/// passed that raw string to `forward_past`, so any key whose byte after the
|
||||
/// shared stem sorts below `[` (0x5B) — `.`, `/`, `-`, digits, uppercase —
|
||||
/// was silently dropped on the next page: with keys `a`, `a.txt`, `zz` and
|
||||
/// max_keys=1, page 2 returned `zz` and `a.txt` was never listed. The token
|
||||
/// is now base64 of the plain key, and this test keeps guarding the
|
||||
/// continuation boundary.
|
||||
#[tokio::test]
|
||||
async fn test_list_objects_v2_continuation_keeps_keys_after_marker_stem() {
|
||||
init_logging();
|
||||
|
||||
@@ -19,6 +19,19 @@ use http::{Method, StatusCode};
|
||||
use tokio::time::{Duration, sleep, timeout};
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// A successful reporting response can still have unknown scanner usage.
|
||||
fn reported_quota_usage(response: &serde_json::Value) -> std::io::Result<Option<u64>> {
|
||||
let value = response
|
||||
.get("current_usage")
|
||||
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "quota response is missing current_usage"))?;
|
||||
if value.is_null() {
|
||||
return Ok(None);
|
||||
}
|
||||
value.as_u64().map(Some).ok_or_else(|| {
|
||||
std::io::Error::new(std::io::ErrorKind::InvalidData, "quota current_usage is not an unsigned integer or null")
|
||||
})
|
||||
}
|
||||
|
||||
/// Test environment setup for quota tests
|
||||
pub struct QuotaTestEnv {
|
||||
pub env: RustFSTestEnvironment,
|
||||
@@ -87,35 +100,69 @@ impl QuotaTestEnv {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_bucket_quota_stats(&self) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!("{}/rustfs/admin/v3/quota-stats/{}", self.env.url, self.bucket_name);
|
||||
let response = awscurl_get(&url, &self.env.access_key, &self.env.secret_key).await?;
|
||||
|
||||
if response.contains("error") {
|
||||
Err(format!("Failed to get quota stats: {}", response).into())
|
||||
} else {
|
||||
Ok(serde_json::from_str(&response)?)
|
||||
async fn wait_for_usage_response(
|
||||
&self,
|
||||
method: Method,
|
||||
path: &str,
|
||||
body: Option<String>,
|
||||
expected_usage: u64,
|
||||
) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let convergence = async {
|
||||
loop {
|
||||
let (status, response) = admin_request(
|
||||
&self.env.url,
|
||||
method.clone(),
|
||||
path,
|
||||
body.clone(),
|
||||
&self.env.access_key,
|
||||
&self.env.secret_key,
|
||||
)
|
||||
.await?;
|
||||
if status.is_success() {
|
||||
let response: serde_json::Value = serde_json::from_str(&response)?;
|
||||
if let Some(usage) = reported_quota_usage(&response)?
|
||||
&& usage == expected_usage
|
||||
{
|
||||
return Ok::<_, Box<dyn std::error::Error + Send + Sync>>(response);
|
||||
}
|
||||
} else if status != StatusCode::SERVICE_UNAVAILABLE {
|
||||
return Err(format!("quota usage request failed for {path}: {status} {response}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
};
|
||||
match timeout(Duration::from_secs(30), convergence).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(format!("quota usage did not converge to {expected_usage} for {path} within 30 seconds").into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn check_bucket_quota(
|
||||
async fn wait_for_bucket_quota_stats(
|
||||
&self,
|
||||
bucket: &str,
|
||||
expected_usage: u64,
|
||||
) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.wait_for_usage_response(Method::GET, &format!("/rustfs/admin/v3/quota-stats/{bucket}"), None, expected_usage)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn wait_for_bucket_quota_check(
|
||||
&self,
|
||||
operation_type: &str,
|
||||
operation_size: u64,
|
||||
expected_usage: u64,
|
||||
) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!("{}/rustfs/admin/v3/quota-check/{}", self.env.url, self.bucket_name);
|
||||
let check_request = serde_json::json!({
|
||||
"operation_type": operation_type,
|
||||
"operation_size": operation_size
|
||||
});
|
||||
|
||||
let response = awscurl_post(&url, &check_request.to_string(), &self.env.access_key, &self.env.secret_key).await?;
|
||||
|
||||
if response.contains("error") {
|
||||
Err(format!("Failed to check quota: {}", response).into())
|
||||
} else {
|
||||
Ok(serde_json::from_str(&response)?)
|
||||
}
|
||||
self.wait_for_usage_response(
|
||||
Method::POST,
|
||||
&format!("/rustfs/admin/v3/quota-check/{}", self.bucket_name),
|
||||
Some(check_request.to_string()),
|
||||
expected_usage,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn upload_object(&self, key: &str, size_bytes: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
@@ -145,30 +192,9 @@ impl QuotaTestEnv {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_bucket_usage(&self) -> Result<u64, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let stats = self.get_bucket_quota_stats().await?;
|
||||
stats
|
||||
.get("current_usage")
|
||||
.and_then(|value| value.as_u64())
|
||||
.ok_or_else(|| format!("bucket quota usage is not authoritative for {}", self.bucket_name).into())
|
||||
}
|
||||
|
||||
async fn wait_for_bucket_usage(&self, expected: u64) -> Result<u64, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let convergence = async {
|
||||
loop {
|
||||
let stats = self.get_bucket_quota_stats().await?;
|
||||
if let Some(usage) = stats.get("current_usage").and_then(|value| value.as_u64())
|
||||
&& usage == expected
|
||||
{
|
||||
return Ok::<u64, Box<dyn std::error::Error + Send + Sync>>(usage);
|
||||
}
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
};
|
||||
match timeout(Duration::from_secs(30), convergence).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(format!("bucket usage did not converge to {expected} bytes within 30 seconds").into()),
|
||||
}
|
||||
let stats = self.wait_for_bucket_quota_stats(&self.bucket_name, expected).await?;
|
||||
reported_quota_usage(&stats)?.ok_or_else(|| "converged quota usage is missing".into())
|
||||
}
|
||||
|
||||
pub async fn set_bucket_quota_for(
|
||||
@@ -176,8 +202,6 @@ impl QuotaTestEnv {
|
||||
bucket: &str,
|
||||
quota_bytes: u64,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.wait_for_quota_usage_for(bucket).await?;
|
||||
|
||||
let quota_path = format!("/rustfs/admin/v3/quota/{bucket}");
|
||||
let quota_config = serde_json::json!({
|
||||
"quota": quota_bytes,
|
||||
@@ -211,48 +235,6 @@ impl QuotaTestEnv {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn wait_for_quota_usage_for(&self, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let stats_path = format!("/rustfs/admin/v3/quota-stats/{bucket}");
|
||||
let readiness = async {
|
||||
loop {
|
||||
let (status, response) =
|
||||
admin_request(&self.env.url, Method::GET, &stats_path, None, &self.env.access_key, &self.env.secret_key)
|
||||
.await?;
|
||||
if status.is_success() {
|
||||
let stats: serde_json::Value = serde_json::from_str(&response)?;
|
||||
if stats.get("current_usage").and_then(|value| value.as_u64()).is_some() {
|
||||
return Ok::<(), Box<dyn std::error::Error + Send + Sync>>(());
|
||||
}
|
||||
} else if status != StatusCode::SERVICE_UNAVAILABLE {
|
||||
return Err(format!("quota usage readiness failed for {bucket}: {status} {response}").into());
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
};
|
||||
match timeout(Duration::from_secs(30), readiness).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(format!("quota usage did not become authoritative for {bucket} within 30 seconds").into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get bucket quota statistics for specific bucket
|
||||
pub async fn get_bucket_quota_stats_for(
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
|
||||
debug!("Getting quota stats for bucket: {}", bucket);
|
||||
|
||||
let url = format!("{}/rustfs/admin/v3/quota-stats/{}", self.env.url, bucket);
|
||||
let response = awscurl_get(&url, &self.env.access_key, &self.env.secret_key).await?;
|
||||
|
||||
if response.contains("error") {
|
||||
Err(format!("Failed to get quota stats: {}", response).into())
|
||||
} else {
|
||||
let stats: serde_json::Value = serde_json::from_str(&response)?;
|
||||
Ok(stats)
|
||||
}
|
||||
}
|
||||
|
||||
/// Upload an object to specific bucket
|
||||
pub async fn upload_object_to_bucket(
|
||||
&self,
|
||||
@@ -277,6 +259,132 @@ impl QuotaTestEnv {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod usage_response_tests {
|
||||
use super::*;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
#[test]
|
||||
fn unknown_usage_is_not_zero() {
|
||||
assert_eq!(
|
||||
reported_quota_usage(&serde_json::json!({"current_usage": null})).expect("valid unknown usage"),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
reported_quota_usage(&serde_json::json!({"current_usage": 0})).expect("known zero usage"),
|
||||
Some(0)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_or_invalid_usage_is_not_readiness() {
|
||||
for response in [
|
||||
serde_json::json!({}),
|
||||
serde_json::json!({"current_usage": -1}),
|
||||
serde_json::json!({"current_usage": "0"}),
|
||||
serde_json::json!({"current_usage": 1.5}),
|
||||
serde_json::json!({"current_usage": false}),
|
||||
] {
|
||||
let error = reported_quota_usage(&response).expect_err("malformed usage must fail");
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
}
|
||||
|
||||
async fn mock_environment(
|
||||
responses: Vec<(StatusCode, &'static str)>,
|
||||
) -> Result<(QuotaTestEnv, JoinSet<()>), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let listener = TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)).await?;
|
||||
let address = listener.local_addr()?;
|
||||
let mut tasks = JoinSet::new();
|
||||
tasks.spawn(async move {
|
||||
for (status, body) in responses {
|
||||
let (mut stream, _) = listener.accept().await.expect("accept quota reporting request");
|
||||
let mut request = Vec::new();
|
||||
while !request.windows(4).any(|part| part == b"\r\n\r\n") {
|
||||
let mut buffer = [0_u8; 1024];
|
||||
let read = stream.read(&mut buffer).await.expect("read reporting request");
|
||||
assert!(read > 0 && request.len() + read <= 8192, "bounded HTTP request headers");
|
||||
request.extend_from_slice(&buffer[..read]);
|
||||
}
|
||||
let response = format!(
|
||||
"HTTP/1.1 {} Test\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
|
||||
status.as_u16(),
|
||||
body.len(),
|
||||
body
|
||||
);
|
||||
stream
|
||||
.write_all(response.as_bytes())
|
||||
.await
|
||||
.expect("write quota reporting response");
|
||||
stream.shutdown().await.expect("close reporting response");
|
||||
}
|
||||
});
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.url = format!("http://{address}");
|
||||
let client = env.create_s3_client();
|
||||
Ok((
|
||||
QuotaTestEnv {
|
||||
env,
|
||||
client,
|
||||
bucket_name: "quota-reporting-fixture".to_owned(),
|
||||
},
|
||||
tasks,
|
||||
))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reporting_waits_for_known_matching_usage_without_selecting_allowed()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let (env, mut tasks) = mock_environment(vec![
|
||||
(StatusCode::SERVICE_UNAVAILABLE, "unavailable"),
|
||||
(StatusCode::OK, r#"{"current_usage":null}"#),
|
||||
(StatusCode::OK, r#"{"current_usage":1}"#),
|
||||
(StatusCode::OK, r#"{"current_usage":0,"allowed":false}"#),
|
||||
])
|
||||
.await?;
|
||||
let response = env.wait_for_usage_response(Method::GET, "/reporting", None, 0).await?;
|
||||
assert_eq!(response["current_usage"], 0);
|
||||
assert_eq!(response["allowed"], false, "readiness must not select a desired admission outcome");
|
||||
timeout(Duration::from_secs(1), tasks.join_next())
|
||||
.await?
|
||||
.expect("reporting mock task")
|
||||
.expect("all responses consumed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reporting_rejects_malformed_success_instead_of_retrying() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let (env, mut tasks) = mock_environment(vec![(StatusCode::OK, r#"{"current_usage":"unknown"}"#)]).await?;
|
||||
let error = env
|
||||
.wait_for_usage_response(Method::GET, "/reporting", None, 0)
|
||||
.await
|
||||
.expect_err("malformed usage must fail");
|
||||
assert!(error.to_string().contains("not an unsigned integer or null"));
|
||||
timeout(Duration::from_secs(1), tasks.join_next())
|
||||
.await?
|
||||
.expect("reporting mock task")
|
||||
.expect("response consumed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reporting_does_not_retry_authorization_failure() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let (env, mut tasks) = mock_environment(vec![(StatusCode::FORBIDDEN, "denied")]).await?;
|
||||
let error = env
|
||||
.wait_for_usage_response(Method::GET, "/reporting", None, 0)
|
||||
.await
|
||||
.expect_err("authorization error must fail");
|
||||
assert!(error.to_string().contains("403"));
|
||||
timeout(Duration::from_secs(1), tasks.join_next())
|
||||
.await?
|
||||
.expect("reporting mock task")
|
||||
.expect("response consumed");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod integration_tests {
|
||||
use super::*;
|
||||
@@ -489,7 +597,7 @@ mod integration_tests {
|
||||
env.upload_object("file2.txt", 256 * 1024).await?;
|
||||
|
||||
// Check usage
|
||||
let usage = env.get_bucket_usage().await?;
|
||||
let usage = env.wait_for_bucket_usage((512 + 256) * 1024).await?;
|
||||
assert_eq!(usage, (512 + 256) * 1024);
|
||||
|
||||
// Delete a file
|
||||
@@ -524,7 +632,7 @@ mod integration_tests {
|
||||
env.upload_object("file2.txt", 512 * 1024).await?;
|
||||
|
||||
// Get detailed quota statistics
|
||||
let stats = env.get_bucket_quota_stats().await?;
|
||||
let stats = env.wait_for_bucket_quota_stats(&env.bucket_name, (1024 + 512) * 1024).await?;
|
||||
|
||||
assert_eq!(stats.get("bucket").unwrap().as_str().unwrap(), env.bucket_name);
|
||||
assert_eq!(stats.get("quota_limit").unwrap().as_u64().unwrap(), 2 * 1024 * 1024);
|
||||
@@ -553,22 +661,22 @@ mod integration_tests {
|
||||
env.upload_object("existing_file.txt", 512 * 1024).await?;
|
||||
|
||||
// Check if we can upload another 512KB (should succeed, exactly fill quota)
|
||||
let check_result = env.check_bucket_quota("PUT", 512 * 1024).await?;
|
||||
let check_result = env.wait_for_bucket_quota_check("PUT", 512 * 1024, 512 * 1024).await?;
|
||||
assert!(check_result.get("allowed").unwrap().as_bool().unwrap());
|
||||
assert_eq!(check_result.get("remaining_quota").unwrap().as_u64().unwrap(), 0);
|
||||
|
||||
// Note: we haven't actually uploaded the second file yet, so current_usage is still 512KB
|
||||
// Check if we can upload 1KB (should succeed - we haven't used the full quota yet)
|
||||
let check_result = env.check_bucket_quota("PUT", 1024).await?;
|
||||
let check_result = env.wait_for_bucket_quota_check("PUT", 1024, 512 * 1024).await?;
|
||||
assert!(check_result.get("allowed").unwrap().as_bool().unwrap());
|
||||
assert_eq!(check_result.get("remaining_quota").unwrap().as_u64().unwrap(), 523264); // 511 * 1024
|
||||
|
||||
// Check if we can upload 600KB (should fail - would exceed quota)
|
||||
let check_result = env.check_bucket_quota("PUT", 600 * 1024).await?;
|
||||
let check_result = env.wait_for_bucket_quota_check("PUT", 600 * 1024, 512 * 1024).await?;
|
||||
assert!(!check_result.get("allowed").unwrap().as_bool().unwrap());
|
||||
|
||||
// Check delete operation (should always be allowed)
|
||||
let check_result = env.check_bucket_quota("DELETE", 512 * 1024).await?;
|
||||
let check_result = env.wait_for_bucket_quota_check("DELETE", 512 * 1024, 512 * 1024).await?;
|
||||
assert!(check_result.get("allowed").unwrap().as_bool().unwrap());
|
||||
|
||||
env.cleanup_bucket().await?;
|
||||
@@ -600,8 +708,8 @@ mod integration_tests {
|
||||
env.upload_object_to_bucket(&bucket2, "another_file.txt", 512 * 1024).await?;
|
||||
|
||||
// Verify statistics are independent
|
||||
let stats1 = env.get_bucket_quota_stats_for(&bucket1).await?;
|
||||
let stats2 = env.get_bucket_quota_stats_for(&bucket2).await?;
|
||||
let stats1 = env.wait_for_bucket_quota_stats(&bucket1, 1024 * 1024).await?;
|
||||
let stats2 = env.wait_for_bucket_quota_stats(&bucket2, (1024 + 512) * 1024).await?;
|
||||
|
||||
assert_eq!(stats1.get("current_usage").unwrap().as_u64().unwrap(), 1024 * 1024);
|
||||
assert_eq!(stats2.get("current_usage").unwrap().as_u64().unwrap(), (1024 + 512) * 1024);
|
||||
@@ -662,7 +770,6 @@ mod integration_tests {
|
||||
let env = QuotaTestEnv::new().await?;
|
||||
|
||||
env.create_bucket().await?;
|
||||
env.wait_for_quota_usage_for(&env.bucket_name).await?;
|
||||
|
||||
// Test 1: GET quota for bucket without quota config
|
||||
let url = format!("{}/rustfs/admin/v3/quota/{}", env.env.url, env.bucket_name);
|
||||
|
||||
@@ -27,8 +27,8 @@ use hyper_util::{
|
||||
};
|
||||
use prost::Message as _;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
BatchGenerallyLockRequest, DeleteRequest, GenerallyLockRequest, ReadVersionRequest, RecoverMultipartSourceRequest,
|
||||
RenameDataResponse,
|
||||
BatchGenerallyLockRequest, DeleteRequest, GenerallyLockRequest, PreparePartTransactionResponse, ReadVersionRequest,
|
||||
RecoverMultipartSourceRequest, RenameDataResponse, SettlePartTransactionRequest, SettlePartTransactionResponse,
|
||||
};
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
@@ -56,6 +56,35 @@ enum Disposition {
|
||||
Discard,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum MutationReply {
|
||||
None,
|
||||
Rename,
|
||||
PreparePart,
|
||||
SettlePart,
|
||||
}
|
||||
|
||||
impl MutationReply {
|
||||
fn outcome(self, bytes: &[u8]) -> io::Result<Option<(bool, bool)>> {
|
||||
let result = match self {
|
||||
Self::None => return Ok(None),
|
||||
Self::Rename => {
|
||||
let reply = RenameDataResponse::decode(unary_payload(bytes)?).map_err(io::Error::other)?;
|
||||
(reply.success, reply.error.is_some())
|
||||
}
|
||||
Self::PreparePart => {
|
||||
let reply = PreparePartTransactionResponse::decode(unary_payload(bytes)?).map_err(io::Error::other)?;
|
||||
(reply.success, reply.error.is_some())
|
||||
}
|
||||
Self::SettlePart => {
|
||||
let reply = SettlePartTransactionResponse::decode(unary_payload(bytes)?).map_err(io::Error::other)?;
|
||||
(reply.success, reply.error.is_some())
|
||||
}
|
||||
};
|
||||
Ok(Some(result))
|
||||
}
|
||||
}
|
||||
|
||||
struct PauseState {
|
||||
methods: Vec<&'static str>,
|
||||
scope: Option<Vec<u8>>,
|
||||
@@ -64,6 +93,9 @@ struct PauseState {
|
||||
limit: usize,
|
||||
retain_after_disconnect: bool,
|
||||
refused_mutations: AtomicUsize,
|
||||
successful_mutation_replies: AtomicUsize,
|
||||
settle_commits: AtomicUsize,
|
||||
settle_rollbacks: AtomicUsize,
|
||||
responses: AtomicUsize,
|
||||
matched: AtomicUsize,
|
||||
blocked: AtomicUsize,
|
||||
@@ -122,6 +154,17 @@ impl RpcPause {
|
||||
self.state.refused_mutations.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn successful_mutation_replies(&self) -> usize {
|
||||
self.state.successful_mutation_replies.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn retained_settle_actions(&self) -> (usize, usize) {
|
||||
(
|
||||
self.state.settle_commits.load(Ordering::SeqCst),
|
||||
self.state.settle_rollbacks.load(Ordering::SeqCst),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn unsupported_source_inspection_routes(&self) -> HashSet<String> {
|
||||
self.state
|
||||
.unsupported_inspection_routes
|
||||
@@ -394,6 +437,9 @@ impl RpcFaultProxy {
|
||||
limit,
|
||||
retain_after_disconnect,
|
||||
refused_mutations: AtomicUsize::new(0),
|
||||
successful_mutation_replies: AtomicUsize::new(0),
|
||||
settle_commits: AtomicUsize::new(0),
|
||||
settle_rollbacks: AtomicUsize::new(0),
|
||||
responses: AtomicUsize::new(0),
|
||||
matched: AtomicUsize::new(0),
|
||||
blocked: AtomicUsize::new(0),
|
||||
@@ -509,6 +555,7 @@ async fn forward_request(
|
||||
if let Some(method) = method.filter(|name| pause.methods.contains(name)) {
|
||||
let mut route = None;
|
||||
let mut lock_owners = Vec::new();
|
||||
let mut settle_action = None;
|
||||
let matches = if let Some(scope) = pause.scope.as_deref() {
|
||||
// Inspect bounded control routing while preserving the exact
|
||||
// signed bytes and trailers forwarded to the backend.
|
||||
@@ -522,12 +569,19 @@ async fn forward_request(
|
||||
inspection = Some((pause.clone(), disk));
|
||||
}
|
||||
let mut matches = scope_matches(method, &bytes, scope)?;
|
||||
if matches && method == "SettlePartTransaction" {
|
||||
settle_action = Some(
|
||||
SettlePartTransactionRequest::decode(unary_payload(&bytes)?)
|
||||
.map_err(io::Error::other)?
|
||||
.rollback,
|
||||
);
|
||||
}
|
||||
if matches && (pause.record_lock_owners || pause.lock_owner.is_some()) {
|
||||
lock_owners = scoped_lock_owners(method, &bytes, scope)?;
|
||||
if let Some(owner) = &pause.lock_owner {
|
||||
matches = lock_owners.iter().any(|candidate| candidate == owner);
|
||||
if !matches && pause.record_lock_owners {
|
||||
observed = Some((pause.clone(), None, std::mem::take(&mut lock_owners), false));
|
||||
observed = Some((pause.clone(), None, std::mem::take(&mut lock_owners), MutationReply::None));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -549,6 +603,14 @@ async fn forward_request(
|
||||
true
|
||||
};
|
||||
if matches && pause.matched.fetch_add(1, Ordering::SeqCst) < pause.limit {
|
||||
if let Some(rollback) = settle_action {
|
||||
let counter = if rollback {
|
||||
&pause.settle_rollbacks
|
||||
} else {
|
||||
&pause.settle_commits
|
||||
};
|
||||
counter.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
let mut released = pause.release.subscribe();
|
||||
pause.blocked.fetch_add(1, Ordering::SeqCst);
|
||||
pause.reached.notify_one();
|
||||
@@ -568,10 +630,14 @@ async fn forward_request(
|
||||
pause,
|
||||
route,
|
||||
lock_owners,
|
||||
matches!(
|
||||
method,
|
||||
"RenameData" | "RenameMultipartData" | "RenameDataAtIncarnation" | "RenameDataWithQuotaFence"
|
||||
),
|
||||
match method {
|
||||
"RenameData" | "RenameMultipartData" | "RenameDataAtIncarnation" | "RenameDataWithQuotaFence" => {
|
||||
MutationReply::Rename
|
||||
}
|
||||
"PreparePartTransaction" => MutationReply::PreparePart,
|
||||
"SettlePartTransaction" => MutationReply::SettlePart,
|
||||
_ => MutationReply::None,
|
||||
},
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -604,7 +670,7 @@ async fn forward_request(
|
||||
.expect("inspection response routes")
|
||||
.insert(disk);
|
||||
}
|
||||
if let Some((pause, route, lock_owners, rename)) = observed {
|
||||
if let Some((pause, route, lock_owners, mutation_reply)) = observed {
|
||||
pause.responses.fetch_add(1, Ordering::SeqCst);
|
||||
if response.status().is_success() && response.headers().get("grpc-status").is_none_or(|status| status == "0") {
|
||||
pause.forwarded.fetch_add(1, Ordering::SeqCst);
|
||||
@@ -626,10 +692,14 @@ async fn forward_request(
|
||||
.headers
|
||||
.get("grpc-status")
|
||||
.or_else(|| trailers.as_ref()?.get("grpc-status"));
|
||||
if rename && parts.status.is_success() && status.is_none_or(|status| status == "0") {
|
||||
let result = RenameDataResponse::decode(unary_payload(&bytes)?).map_err(io::Error::other)?;
|
||||
if !result.success && result.error.is_some() {
|
||||
if parts.status.is_success()
|
||||
&& status.is_some_and(|status| status == "0")
|
||||
&& let Some((success, has_error)) = mutation_reply.outcome(&bytes)?
|
||||
{
|
||||
if !success && has_error {
|
||||
pause.refused_mutations.fetch_add(1, Ordering::SeqCst);
|
||||
} else if success && !has_error {
|
||||
pause.successful_mutation_replies.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
let body = Full::new(bytes)
|
||||
@@ -676,6 +746,10 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn backend_with_status(status: Option<&'static str>) -> io::Result<Backend> {
|
||||
backend_with_reply(status, None).await
|
||||
}
|
||||
|
||||
async fn backend_with_reply(status: Option<&'static str>, reply: Option<Bytes>) -> io::Result<Backend> {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||
let address = listener.local_addr()?;
|
||||
let received = Arc::new(AtomicUsize::new(0));
|
||||
@@ -687,9 +761,11 @@ mod tests {
|
||||
accepted = listener.accept() => {
|
||||
let Ok((socket, _)) = accepted else { break };
|
||||
let received = received_task.clone();
|
||||
let reply = reply.clone();
|
||||
children.spawn(async move {
|
||||
let service = service_fn(move |request: Request<Incoming>| {
|
||||
let received = received.clone();
|
||||
let reply = reply.clone();
|
||||
async move {
|
||||
received.fetch_add(1, Ordering::SeqCst);
|
||||
let authority = request.headers().get(http::header::HOST).cloned().or_else(|| {
|
||||
@@ -698,6 +774,7 @@ mod tests {
|
||||
})
|
||||
});
|
||||
let body = request.into_body().collect().await?.to_bytes();
|
||||
let body = reply.unwrap_or(body);
|
||||
let mut trailers = http::HeaderMap::new();
|
||||
trailers.insert("grpc-status", http::HeaderValue::from_static(status.unwrap_or("0")));
|
||||
trailers.insert("x-proof", http::HeaderValue::from_static("unchanged-proof"));
|
||||
@@ -1434,6 +1511,93 @@ mod tests {
|
||||
.await
|
||||
.expect("scoped discard deadline");
|
||||
assert_late_packet_ownership().await;
|
||||
assert_late_part_reply_refusal().await;
|
||||
}
|
||||
|
||||
async fn assert_late_part_reply_refusal() {
|
||||
use rustfs_protos::proto_gen::node_service::{Error, PreparePartTransactionRequest};
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
// method, success, error, gRPC status, rollback, expected successes/refusals
|
||||
for (method, success, has_error, status, rollback, expected) in [
|
||||
("PreparePartTransaction", false, true, None, false, (0, 1)),
|
||||
("PreparePartTransaction", true, false, None, false, (1, 0)),
|
||||
("PreparePartTransaction", false, false, None, false, (0, 0)),
|
||||
("SettlePartTransaction", true, false, None, false, (1, 0)),
|
||||
("SettlePartTransaction", false, true, None, true, (0, 1)),
|
||||
("SettlePartTransaction", true, true, None, false, (0, 0)),
|
||||
("SettlePartTransaction", false, true, Some("16"), false, (0, 0)),
|
||||
("PreparePartTransaction", false, true, Some("16"), false, (0, 0)),
|
||||
("RenameMultipartData", false, true, None, false, (0, 1)),
|
||||
] {
|
||||
let error = has_error.then(|| Error {
|
||||
code: 1,
|
||||
error_info: "retired source".to_owned(),
|
||||
});
|
||||
let reply = match method {
|
||||
"PreparePartTransaction" => unary_frame(&PreparePartTransactionResponse { success, error }),
|
||||
"SettlePartTransaction" => unary_frame(&SettlePartTransactionResponse { success, error }),
|
||||
_ => unary_frame(&RenameDataResponse {
|
||||
success,
|
||||
error,
|
||||
..Default::default()
|
||||
}),
|
||||
};
|
||||
// An unknown field must survive typed inspection byte-for-byte.
|
||||
let mut framed = reply.to_vec();
|
||||
framed.extend_from_slice(&[0xfa, 0x03, 0x03, b'x', b'y', b'z']);
|
||||
let payload_len = u32::try_from(framed.len() - 5).expect("small reply");
|
||||
framed[1..5].copy_from_slice(&payload_len.to_be_bytes());
|
||||
let reply = Bytes::from(framed);
|
||||
let backend = backend_with_reply(status, Some(reply.clone()))
|
||||
.await
|
||||
.expect("typed reply backend");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy");
|
||||
let pause = proxy.pause_late(&[method], 1, "late-upload").expect("part gate");
|
||||
let request = match method {
|
||||
"PreparePartTransaction" => unary_frame(&PreparePartTransactionRequest {
|
||||
disk: "disk".to_owned(),
|
||||
dst_path: "late-upload/part.1".to_owned(),
|
||||
..Default::default()
|
||||
}),
|
||||
"SettlePartTransaction" => unary_frame(&SettlePartTransactionRequest {
|
||||
disk: "disk".to_owned(),
|
||||
path: "late-upload/part.1".to_owned(),
|
||||
rollback,
|
||||
..Default::default()
|
||||
}),
|
||||
_ => Bytes::from_static(b"late-upload"),
|
||||
};
|
||||
let address = proxy.local_addr();
|
||||
let caller = tokio::spawn(async move {
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let response = call_body(&client, address, method, request).await.expect("retained reply");
|
||||
response.into_body().collect().await.expect("typed response frames")
|
||||
});
|
||||
pause.wait_for_hits(1).await.expect("actual retained request");
|
||||
pause.release();
|
||||
let response = caller.await.expect("caller task");
|
||||
let trailers = response.trailers().expect("reply trailers");
|
||||
assert_eq!(trailers["x-proof"], "unchanged-proof");
|
||||
assert_eq!(trailers["grpc-status"], status.unwrap_or("0"));
|
||||
assert_eq!(response.to_bytes(), reply);
|
||||
assert_eq!(pause.responses(), 1);
|
||||
assert_eq!(
|
||||
(pause.successful_mutation_replies(), pause.refused_mutations()),
|
||||
expected,
|
||||
"typed {method} outcome"
|
||||
);
|
||||
let actions = if method == "SettlePartTransaction" {
|
||||
if rollback { (0, 1) } else { (1, 0) }
|
||||
} else {
|
||||
(0, 0)
|
||||
};
|
||||
assert_eq!(pause.retained_settle_actions(), actions);
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("part reply deadline");
|
||||
}
|
||||
async fn assert_late_packet_ownership() {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
|
||||
@@ -600,6 +600,142 @@ mod tests {
|
||||
info!("✅ PASSED: Veeam backup workflow simulation completed successfully");
|
||||
}
|
||||
|
||||
/// Veeam B&R discovers SOSAPI by GET/HEAD of two well-known keys.
|
||||
#[tokio::test]
|
||||
async fn test_veeam_sosapi_system_and_capacity() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
use crate::common::local_http_client;
|
||||
|
||||
const SYSTEM_KEY: &str = ".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml";
|
||||
const CAPACITY_KEY: &str = ".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml";
|
||||
|
||||
init_logging();
|
||||
info!("🧪 TEST: Veeam SOSAPI system.xml and capacity.xml");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let client = create_s3_client(&env);
|
||||
let bucket = "veeam-sosapi";
|
||||
create_bucket(&client, bucket).await?;
|
||||
|
||||
let system = client.get_object().bucket(bucket).key(SYSTEM_KEY).send().await?;
|
||||
let system_etag = system.e_tag().map(str::to_string);
|
||||
let system_len = system.content_length();
|
||||
assert_eq!(system.content_type(), Some("application/xml"));
|
||||
let system_body = system.body.collect().await?.into_bytes();
|
||||
let system_xml = std::str::from_utf8(&system_body)?;
|
||||
assert!(system_xml.contains("<ProtocolVersion>\"1.0\"</ProtocolVersion>"), "{system_xml}");
|
||||
assert!(
|
||||
system_xml.contains("<ModelName>\"RustFS ") && system_xml.contains("\"</ModelName>"),
|
||||
"{system_xml}"
|
||||
);
|
||||
assert!(system_xml.contains("<CapacityInfo>true</CapacityInfo>"));
|
||||
assert!(system_xml.contains("<UploadSessions>false</UploadSessions>"));
|
||||
assert!(system_xml.contains("<IAMSTS>false</IAMSTS>"));
|
||||
assert!(!system_xml.contains("SystemRecommendations"));
|
||||
assert!(!system_xml.contains("APIEndpoints"));
|
||||
assert!(!system_xml.contains("""));
|
||||
|
||||
let head = client.head_object().bucket(bucket).key(SYSTEM_KEY).send().await?;
|
||||
assert_eq!(head.content_type(), Some("application/xml"));
|
||||
assert_eq!(head.content_length(), system_len);
|
||||
assert_eq!(head.e_tag().map(str::to_string), system_etag);
|
||||
assert!(head.last_modified().is_some());
|
||||
|
||||
let ranged = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(SYSTEM_KEY)
|
||||
.range("bytes=0-3")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(ranged.content_length(), Some(4));
|
||||
let ranged_header = ranged.content_range().unwrap_or("");
|
||||
assert!(ranged_header.starts_with("bytes 0-3/"), "{ranged_header}");
|
||||
let ranged_body = ranged.body.collect().await?.into_bytes();
|
||||
assert_eq!(&ranged_body[..], &system_body[..4]);
|
||||
|
||||
let capacity = client.get_object().bucket(bucket).key(CAPACITY_KEY).send().await?;
|
||||
assert_eq!(capacity.content_type(), Some("application/xml"));
|
||||
let capacity_body = capacity.body.collect().await?.into_bytes();
|
||||
let capacity_xml = std::str::from_utf8(&capacity_body)?;
|
||||
let capacity_value = xml_u64(capacity_xml, "Capacity")?;
|
||||
let available_value = xml_u64(capacity_xml, "Available")?;
|
||||
let used_value = xml_u64(capacity_xml, "Used")?;
|
||||
assert!(used_value <= capacity_value, "{capacity_xml}");
|
||||
assert!(available_value <= capacity_value.saturating_sub(used_value), "{capacity_xml}");
|
||||
|
||||
let put_err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(SYSTEM_KEY)
|
||||
.body(ByteStream::from_static(b"overwrite"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("SOSAPI put must fail");
|
||||
assert_eq!(put_err.code(), Some("MethodNotAllowed"));
|
||||
let delete_err = client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(CAPACITY_KEY)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("SOSAPI delete must fail");
|
||||
assert_eq!(delete_err.code(), Some("MethodNotAllowed"));
|
||||
|
||||
let normal_key = "Veeam/Backup/job.dat";
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(normal_key)
|
||||
.body(ByteStream::from_static(b"backup-bytes"))
|
||||
.send()
|
||||
.await?;
|
||||
let normal = client.get_object().bucket(bucket).key(normal_key).send().await?;
|
||||
let normal_body = normal.body.collect().await?.into_bytes();
|
||||
assert_eq!(&normal_body[..], b"backup-bytes");
|
||||
|
||||
let anon = local_http_client()
|
||||
.get(format!("{}/{bucket}/{SYSTEM_KEY}", env.url))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(anon.status(), reqwest::StatusCode::FORBIDDEN);
|
||||
let anon_body = anon.text().await?;
|
||||
assert!(!anon_body.contains("ProtocolVersion"), "{anon_body}");
|
||||
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
let scope_date = format!("{:04}{:02}{:02}", now.year(), u8::from(now.month()), now.day());
|
||||
let amz_date = format!("{scope_date}T{:02}{:02}{:02}Z", now.hour(), now.minute(), now.second());
|
||||
let bad = local_http_client()
|
||||
.get(format!("{}/{bucket}/{SYSTEM_KEY}", env.url))
|
||||
.header("x-amz-date", &amz_date)
|
||||
.header("x-amz-content-sha256", "UNSIGNED-PAYLOAD")
|
||||
.header(
|
||||
"authorization",
|
||||
format!(
|
||||
"AWS4-HMAC-SHA256 Credential=bad/{scope_date}/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=0000000000000000000000000000000000000000000000000000000000000000"
|
||||
),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(bad.status(), reqwest::StatusCode::FORBIDDEN);
|
||||
let bad_body = bad.text().await?;
|
||||
assert!(!bad_body.contains("ProtocolVersion"), "{bad_body}");
|
||||
|
||||
info!("✅ PASSED: Veeam SOSAPI system.xml and capacity.xml");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn xml_u64(xml: &str, element: &str) -> Result<u64, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let start = format!("<{element}>");
|
||||
let end = format!("</{element}>");
|
||||
let start_idx = xml.find(&start).ok_or_else(|| format!("missing {element} in {xml}"))? + start.len();
|
||||
let end_idx = xml[start_idx..]
|
||||
.find(&end)
|
||||
.ok_or_else(|| format!("unclosed {element} in {xml}"))?
|
||||
+ start_idx;
|
||||
Ok(xml[start_idx..end_idx].parse()?)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_terraform_put_after_delete() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -12,6 +12,9 @@ Applies to `crates/ecstore/`.
|
||||
|
||||
- Be careful with allocations and locking in hot paths.
|
||||
- Keep network and disk operations async-friendly; avoid introducing unnecessary blocking.
|
||||
- Coalesce related `open`, `read`/`write`, `fsync`, and `close` filesystem steps into a single `blocking_queue` or `run_blocking_namespace_operation` closure per disk rather than chaining multiple `tokio::fs` or `spawn_blocking` dispatches.
|
||||
- Pin a parent directory once (`openat2(BENEATH | NO_SYMLINKS)` on Linux) and reuse the pinned guard across sibling metadata reads (`read_optional_in_pinned`, `part.N.meta` loops) instead of re-walking ancestor components per file or re-opening when `source_parent == destination_parent`.
|
||||
- After `file.metadata()`, pre-size read buffers (`Vec::with_capacity(len + 1)`) before `read_to_end` to avoid `std::io::read_to_end`'s 32-byte exponential growth (`6–7` `read()` syscalls per small metadata file).
|
||||
- Benchmark-sensitive changes should include measurable rationale.
|
||||
|
||||
## Allocation Discipline in Hot Paths
|
||||
|
||||
@@ -252,7 +252,7 @@ pub mod bucket {
|
||||
pub mod quota {
|
||||
pub use crate::bucket::quota::{
|
||||
BucketQuota, QUOTA_RESERVATION_PROTOCOL_V1, QUOTA_RESERVATION_PROTOCOL_V2, QuotaCheckResult, QuotaError,
|
||||
QuotaMigrationSkipReason, QuotaOperation, QuotaProtocolMigration,
|
||||
QuotaMigrationSkipReason, QuotaOperation, QuotaProtocolMigration, QuotaType,
|
||||
};
|
||||
|
||||
pub mod checker {
|
||||
|
||||
@@ -11314,13 +11314,7 @@ mod tests {
|
||||
let marker = marker
|
||||
.as_deref()
|
||||
.expect("production checkpoint should persist an object marker");
|
||||
let marker_suffix = marker
|
||||
.strip_prefix(&expected_marker)
|
||||
.expect("production checkpoint should stop at the first page boundary");
|
||||
assert!(
|
||||
marker_suffix.is_empty() || marker_suffix.starts_with("[rustfs_cache:v2,"),
|
||||
"unexpected production checkpoint marker suffix: {marker_suffix}"
|
||||
);
|
||||
assert_eq!(marker, expected_marker, "production checkpoint should stop at the first page boundary");
|
||||
assert_eq!(version_marker.as_deref(), Some("null"));
|
||||
}
|
||||
|
||||
|
||||
@@ -43,11 +43,6 @@ const EVENT_METACACHE_LISTING: &str = "metacache_listing";
|
||||
pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = Result<(), DiskError>> + Send>> + Send + 'static>;
|
||||
pub type PartialFn =
|
||||
Box<dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
|
||||
pub(crate) type PartialResultFn = Box<
|
||||
dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = disk::error::Result<()>> + Send>>
|
||||
+ Send
|
||||
+ 'static,
|
||||
>;
|
||||
type FinishedFn = Box<dyn Fn(&[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
@@ -60,6 +55,7 @@ impl FallbackClaimTracker {
|
||||
self.claimed.lock().await.insert(disk.endpoint().to_string());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn claimed_keys(&self) -> HashSet<String> {
|
||||
self.claimed.lock().await.clone()
|
||||
}
|
||||
@@ -317,7 +313,17 @@ pub struct ListPathRawOutcome {
|
||||
pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
|
||||
let rx = rx.child_token();
|
||||
let _cancel_guard = rx.clone().drop_guard();
|
||||
list_path_raw_inner(rx, opts, None, None).await.map(|_| ())
|
||||
list_path_raw_inner(rx, opts, None).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// Like [`list_path_raw`], but reports how the merged stream ended.
|
||||
pub(crate) async fn list_path_raw_with_outcome(
|
||||
rx: CancellationToken,
|
||||
opts: ListPathRawOptions,
|
||||
) -> disk::error::Result<ListPathRawOutcome> {
|
||||
let rx = rx.child_token();
|
||||
let _cancel_guard = rx.clone().drop_guard();
|
||||
list_path_raw_inner(rx, opts, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn list_path_raw_with_claim_tracker(
|
||||
@@ -327,25 +333,13 @@ pub(crate) async fn list_path_raw_with_claim_tracker(
|
||||
) -> disk::error::Result<ListPathRawOutcome> {
|
||||
let rx = rx.child_token();
|
||||
let _cancel_guard = rx.clone().drop_guard();
|
||||
list_path_raw_inner(rx, opts, Some(claim_tracker), None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn list_path_raw_with_partial_result(
|
||||
rx: CancellationToken,
|
||||
opts: ListPathRawOptions,
|
||||
claim_tracker: FallbackClaimTracker,
|
||||
partial_result: PartialResultFn,
|
||||
) -> disk::error::Result<ListPathRawOutcome> {
|
||||
let rx = rx.child_token();
|
||||
let _cancel_guard = rx.clone().drop_guard();
|
||||
list_path_raw_inner(rx, opts, Some(claim_tracker), Some(partial_result)).await
|
||||
list_path_raw_inner(rx, opts, Some(claim_tracker)).await
|
||||
}
|
||||
|
||||
async fn list_path_raw_inner(
|
||||
rx: CancellationToken,
|
||||
opts: ListPathRawOptions,
|
||||
fallback_claim_tracker: Option<FallbackClaimTracker>,
|
||||
partial_result: Option<PartialResultFn>,
|
||||
) -> disk::error::Result<ListPathRawOutcome> {
|
||||
if opts.disks.is_empty() {
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
@@ -1020,13 +1014,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(partial_fn) = partial_result.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
result = partial_fn(MetaCacheEntries(top_entries), &errs) => result?,
|
||||
}
|
||||
} else if let Some(partial_fn) = opts.partial.as_ref() {
|
||||
if let Some(partial_fn) = opts.partial.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
@@ -1308,7 +1296,6 @@ mod tests {
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("listing should complete");
|
||||
@@ -1333,7 +1320,6 @@ mod tests {
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("listing should complete");
|
||||
@@ -1827,39 +1813,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_propagates_partial_callback_failure() {
|
||||
let first = MetaCacheEntry {
|
||||
name: "bucket/object-a".to_string(),
|
||||
metadata: vec![1],
|
||||
..Default::default()
|
||||
};
|
||||
let second = MetaCacheEntry {
|
||||
name: "bucket/object-b".to_string(),
|
||||
metadata: vec![2],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = list_path_raw_with_partial_result(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None, None],
|
||||
min_disks: 2,
|
||||
test_reader_behaviors: vec![
|
||||
TestReaderBehavior::Entries(vec![first]),
|
||||
TestReaderBehavior::Entries(vec![second]),
|
||||
],
|
||||
..Default::default()
|
||||
},
|
||||
FallbackClaimTracker::default(),
|
||||
Box::new(move |_, _| Box::pin(async { Err(DiskError::ErasureReadQuorum) })),
|
||||
)
|
||||
.await
|
||||
.expect_err("a resolution failure after reader advancement must fail the listing");
|
||||
|
||||
assert_eq!(err, DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_cancels_a_blocked_partial_callback() {
|
||||
let cancel = CancellationToken::new();
|
||||
|
||||
@@ -8748,6 +8748,89 @@ mod tests {
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn endpoint_reconnect_admits_ready_drive_while_another_format_probe_is_blocked() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let reference = crate::layout::format::FormatV3::new(1, 2);
|
||||
let mut peers = Vec::new();
|
||||
let mut endpoints = Vec::new();
|
||||
for index in 0..2 {
|
||||
let mut format = reference.clone();
|
||||
format.erasure.this = reference.erasure.sets[0][index];
|
||||
let peer = TestGrpcPeer::spawn(
|
||||
Bytes::from(format.to_json().expect("format should serialize")),
|
||||
Bytes::from_static(b"recovered-peer-data"),
|
||||
)
|
||||
.await
|
||||
.expect("authenticated regression peer is required");
|
||||
endpoints.push(Endpoint {
|
||||
url: url::Url::parse(&format!("{}/data/rustfs{index}", peer.addr)).expect("endpoint should parse"),
|
||||
is_local: false,
|
||||
pool_idx: 0,
|
||||
set_idx: 0,
|
||||
disk_idx: i32::try_from(index).expect("fixture index fits"),
|
||||
});
|
||||
peers.push(peer);
|
||||
}
|
||||
let set = crate::set_disk::SetDisks::new(
|
||||
"independent-reconnect-test".to_owned(),
|
||||
Arc::new(tokio::sync::RwLock::new(vec![None; 2])),
|
||||
2,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
endpoints,
|
||||
reference.clone(),
|
||||
Vec::new(),
|
||||
)
|
||||
.await;
|
||||
let gate = peers[0].peer.pause_next_format_read();
|
||||
let pending_set = Arc::clone(&set);
|
||||
let pending = tokio::spawn(async move { pending_set.connect_disks().await });
|
||||
time::timeout(Duration::from_secs(10), gate.entered.notified())
|
||||
.await
|
||||
.expect("the first endpoint must reach the format barrier");
|
||||
|
||||
let ready = time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
if let Some(disk) = set.disks.read().await[1].clone() {
|
||||
return disk;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
if let Ok(disk) = &ready {
|
||||
assert!(set.disks.read().await[0].is_none(), "the blocked endpoint must remain unpublished");
|
||||
assert!(!pending.is_finished(), "the sweep still owns its blocked endpoint probe");
|
||||
assert_eq!(
|
||||
disk.read_all("bucket", "object")
|
||||
.await
|
||||
.expect("the ready endpoint must serve real I/O"),
|
||||
Bytes::from_static(b"recovered-peer-data")
|
||||
);
|
||||
assert_eq!(
|
||||
peers[1].peer.object_read_all_disks(),
|
||||
vec![reference.erasure.sets[0][1].to_string()],
|
||||
"the usable handle must carry its validated slot identity"
|
||||
);
|
||||
}
|
||||
gate.release.cancel();
|
||||
pending.await.expect("all endpoint probes must finish after release");
|
||||
for disk in set.disks.read().await.iter().flatten() {
|
||||
disk.close().await.expect("stop each published handle's monitoring");
|
||||
}
|
||||
for peer in peers {
|
||||
peer.stop().await;
|
||||
}
|
||||
ready.expect("a blocked earlier endpoint must not delay a ready drive in the same set");
|
||||
assert!(
|
||||
set.disks.read().await.iter().all(Option::is_some),
|
||||
"both endpoints must eventually be admitted"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn missing_read_disks_preserve_concurrently_published_handles() {
|
||||
|
||||
@@ -41,6 +41,10 @@ use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::{Request, Response};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_REMOTE_LOCK: &str = "remote_lock";
|
||||
|
||||
fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(request: &mut Request<T>) -> std::io::Result<()> {
|
||||
set_tonic_rolling_mutation_body_digest(request)
|
||||
}
|
||||
@@ -233,6 +237,20 @@ pub struct RemoteClient {
|
||||
impl RemoteClient {
|
||||
const ONLINE_CHECK_RESOURCE: &'static str = "health-lock-online";
|
||||
|
||||
fn record_release_ack(&self, lock_id: &LockId) {
|
||||
tracing::trace!(
|
||||
target: "rustfs_ecstore::object_lock_diag",
|
||||
event = EVENT_NAMESPACE_LOCK_RELEASE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "acknowledged",
|
||||
peer_kind = "remote",
|
||||
lock_id = %lock_id,
|
||||
peer = %self.addr,
|
||||
"Remote namespace lock release acknowledged"
|
||||
);
|
||||
}
|
||||
|
||||
pub fn new(endpoint: String) -> Self {
|
||||
Self { addr: endpoint }
|
||||
}
|
||||
@@ -1204,8 +1222,6 @@ impl LockClient for RemoteClient {
|
||||
}
|
||||
|
||||
async fn release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
info!("remote release for {}", lock_id);
|
||||
|
||||
let unlock_request = Self::create_unlock_request(lock_id);
|
||||
let request_string = serde_json::to_string(&unlock_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
|
||||
@@ -1226,6 +1242,9 @@ impl LockClient for RemoteClient {
|
||||
if let Some(error_info) = resp.error_info {
|
||||
return Err(LockError::internal(error_info));
|
||||
}
|
||||
if resp.success {
|
||||
self.record_release_ack(lock_id);
|
||||
}
|
||||
Ok(resp.success)
|
||||
}
|
||||
|
||||
@@ -1261,7 +1280,13 @@ impl LockClient for RemoteClient {
|
||||
Ok(lock_ids
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, _)| resp.results.get(idx).map(|result| result.success).unwrap_or(false))
|
||||
.map(|(idx, lock_id)| {
|
||||
let result = resp.results.get(idx);
|
||||
if result.is_some_and(|result| result.success && result.error_info.is_none()) {
|
||||
self.record_release_ack(lock_id);
|
||||
}
|
||||
result.map(|result| result.success).unwrap_or(false)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
|
||||
@@ -317,6 +317,17 @@ impl Sets {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
|
||||
if let Some(exit_signal) = &self.exit_signal {
|
||||
let _ = exit_signal.send(());
|
||||
// Wait for an in-flight reconnect pass as well as its receiver to exit.
|
||||
while exit_signal.receiver_count() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep simulated peers' metadata ownership separate while sharing disks and lock clients.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
|
||||
|
||||
@@ -19,6 +19,7 @@ use tracing::Span;
|
||||
use uuid::Uuid;
|
||||
|
||||
const EVENT_OBJECT_NAMESPACE_LOCK: &str = "object_namespace_lock";
|
||||
const EVENT_OBJECT_NAMESPACE_LOCK_PEER: &str = "object_namespace_lock_peer";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
|
||||
|
||||
@@ -56,6 +57,7 @@ struct AttemptContext {
|
||||
requested_version_id: Option<String>,
|
||||
namespace: String,
|
||||
owner: String,
|
||||
lock_id: Option<String>,
|
||||
mode: &'static str,
|
||||
timeout: Duration,
|
||||
started_at: Instant,
|
||||
@@ -85,6 +87,7 @@ impl ObjectLockAttempt {
|
||||
requested_version_id: requested_version_id.map(str::to_owned),
|
||||
namespace: lock.namespace().to_owned(),
|
||||
owner: lock.owner().to_owned(),
|
||||
lock_id: None,
|
||||
mode,
|
||||
timeout,
|
||||
started_at: Instant::now(),
|
||||
@@ -105,9 +108,29 @@ impl ObjectLockAttempt {
|
||||
context.finished = true;
|
||||
context.trace("disabled");
|
||||
}
|
||||
Ok(_) => {
|
||||
Ok(guard) => {
|
||||
context.lock_id = guard.lock_id().map(ToString::to_string);
|
||||
context.acquired_at = Some(Instant::now());
|
||||
context.trace("acquired");
|
||||
if let NamespaceLockGuard::Standard(guard) = guard {
|
||||
for (peer_index, peer_lock_id) in guard.granted_lock_ids().enumerate() {
|
||||
context.parent.in_scope(|| {
|
||||
tracing::trace!(
|
||||
target: "rustfs_ecstore::object_lock_diag",
|
||||
parent: &context.parent,
|
||||
event = EVENT_OBJECT_NAMESPACE_LOCK_PEER,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT_LOCK,
|
||||
state = "granted",
|
||||
lock_attempt_id = %context.id,
|
||||
lock_id = context.lock_id.as_deref(),
|
||||
peer_lock_id = %peer_lock_id,
|
||||
peer_index,
|
||||
"Object namespace peer owner granted"
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
context.finished = true;
|
||||
@@ -171,6 +194,7 @@ impl AttemptContext {
|
||||
mode = self.mode,
|
||||
owner = %self.owner,
|
||||
lock_attempt_id = %self.id,
|
||||
lock_id = self.lock_id.as_deref(),
|
||||
acquire_ms = duration_ms(acquired_at.duration_since(self.started_at)),
|
||||
hold_ms = duration_ms(now.duration_since(acquired_at)),
|
||||
timeout_ms = duration_ms(self.timeout),
|
||||
@@ -235,15 +259,238 @@ pub(crate) mod tests {
|
||||
}
|
||||
|
||||
pub(crate) fn rows(&self) -> Vec<serde_json::Value> {
|
||||
self.all_rows()
|
||||
.into_iter()
|
||||
.filter(|row| row["fields"]["event"] == EVENT_OBJECT_NAMESPACE_LOCK)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn all_rows(&self) -> Vec<serde_json::Value> {
|
||||
let bytes = self.0.lock().expect("lock diagnostic capture mutex");
|
||||
let text = std::str::from_utf8(&bytes).expect("UTF-8 diagnostic events");
|
||||
text.lines()
|
||||
.map(|line| serde_json::from_str::<serde_json::Value>(line).expect("JSON diagnostic event"))
|
||||
.filter(|row| row["fields"]["event"] == EVENT_OBJECT_NAMESPACE_LOCK)
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn object_lock_diagnostics_join_guard_identity_to_physical_release() {
|
||||
use rustfs_lock::{
|
||||
DisabledLockManager, FastObjectLockManager, GlobalLockManager, LocalClient, LockClient, LockRequest, LockType,
|
||||
NamespaceLock, ObjectKey,
|
||||
};
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_OBJECT_LOCK_DIAG_ENABLE, Some("true"))], || {
|
||||
let capture = CapturedLockEvents::default();
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("diagnostic release runtime");
|
||||
tracing::subscriber::with_default(capture.subscriber(), || {
|
||||
runtime.block_on(async {
|
||||
let clients = (0..4)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn LockClient>)
|
||||
.collect::<Vec<_>>();
|
||||
let lock = NamespaceLockWrapper::new(
|
||||
NamespaceLock::with_clients_and_quorum("diagnostics".to_owned(), clients.clone(), 4),
|
||||
ObjectKey::new("bucket", "object"),
|
||||
"coordinator".to_owned(),
|
||||
);
|
||||
let request = tracing::info_span!("request", request_id = "original-multipart-request");
|
||||
let mut attempt = request.in_scope(|| {
|
||||
ObjectLockAttempt::start(
|
||||
"abort_multipart_upload",
|
||||
"bucket",
|
||||
"object",
|
||||
None,
|
||||
&lock,
|
||||
"write",
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
});
|
||||
let result = lock.get_write_lock(Duration::from_secs(5)).await;
|
||||
attempt.observe(&result);
|
||||
let guard = result.expect("four real namespace owners");
|
||||
let id = guard.lock_id().expect("distributed guard identity").clone();
|
||||
let peer_ids = match &guard {
|
||||
NamespaceLockGuard::Standard(guard) => {
|
||||
assert_eq!(guard.granted_lock_ids().count(), 4);
|
||||
guard.granted_lock_ids().cloned().collect::<std::collections::HashSet<_>>()
|
||||
}
|
||||
NamespaceLockGuard::Fast(_) => panic!("real distributed namespace required"),
|
||||
};
|
||||
let rows = capture.rows();
|
||||
assert_eq!(rows.last().expect("acquired event")["fields"]["lock_id"], id.to_string());
|
||||
let rows = capture.all_rows();
|
||||
let mappings = rows
|
||||
.iter()
|
||||
.filter(|row| row["fields"]["event"] == EVENT_OBJECT_NAMESPACE_LOCK_PEER)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(mappings.len(), 4, "every granted peer must map to its logical guard");
|
||||
assert!(mappings.iter().all(|row| {
|
||||
row["fields"]["lock_id"] == id.to_string()
|
||||
&& peer_ids
|
||||
.iter()
|
||||
.any(|peer_id| row["fields"]["peer_lock_id"] == peer_id.to_string())
|
||||
}));
|
||||
for client in &clients {
|
||||
let mut owned = 0;
|
||||
for peer_id in &peer_ids {
|
||||
owned += usize::from(
|
||||
client
|
||||
.check_status(peer_id)
|
||||
.await
|
||||
.expect("actual peer owner status")
|
||||
.is_some(),
|
||||
);
|
||||
}
|
||||
assert_eq!(owned, 1, "each client must hold exactly one actual granted identity");
|
||||
}
|
||||
drop((guard, attempt));
|
||||
assert!(
|
||||
capture.all_rows().iter().all(|row| row["fields"]["state"] != "released"),
|
||||
"enqueue is not release acknowledgement"
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while capture
|
||||
.all_rows()
|
||||
.iter()
|
||||
.filter(|row| {
|
||||
row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "released"
|
||||
})
|
||||
.count()
|
||||
< 4
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all physical release callbacks finish");
|
||||
for client in &clients {
|
||||
for peer_id in &peer_ids {
|
||||
assert!(
|
||||
client
|
||||
.check_status(peer_id)
|
||||
.await
|
||||
.expect("released peer owner status")
|
||||
.is_none()
|
||||
);
|
||||
assert!(!client.release(peer_id).await.expect("duplicate release is absent"));
|
||||
}
|
||||
}
|
||||
let disabled = LocalClient::with_manager(Arc::new(GlobalLockManager::Disabled(DisabledLockManager::new())));
|
||||
let disabled_request =
|
||||
LockRequest::new(ObjectKey::new("bucket", "disabled"), LockType::Exclusive, "disabled-owner".to_owned());
|
||||
assert!(
|
||||
disabled
|
||||
.acquire_lock(&disabled_request)
|
||||
.await
|
||||
.expect("disabled acquire bypass")
|
||||
.success
|
||||
);
|
||||
assert!(
|
||||
disabled
|
||||
.release(&disabled_request.lock_id)
|
||||
.await
|
||||
.expect("disabled release bypass")
|
||||
);
|
||||
let rows = capture.all_rows();
|
||||
let releases = rows
|
||||
.iter()
|
||||
.filter(|row| row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "released")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
releases.len(),
|
||||
4,
|
||||
"duplicates and disabled guards cannot acknowledge another physical release"
|
||||
);
|
||||
let physical_mappings = rows
|
||||
.iter()
|
||||
.filter(|row| row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "mapped")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(physical_mappings.len(), 4);
|
||||
assert!(releases.iter().all(|row| physical_mappings.iter().any(|mapped| {
|
||||
mapped["fields"]["physical_guard_id"] == row["fields"]["physical_guard_id"]
|
||||
&& mapped["fields"]["process_id"] == row["fields"]["process_id"]
|
||||
&& peer_ids
|
||||
.iter()
|
||||
.any(|peer_id| mapped["fields"]["lock_id"] == peer_id.to_string())
|
||||
})));
|
||||
let dropped = capture.rows().pop().expect("guard drop event");
|
||||
assert_eq!(dropped["fields"]["lock_id"], id.to_string());
|
||||
assert!(
|
||||
dropped["spans"]
|
||||
.as_array()
|
||||
.expect("retained request span")
|
||||
.iter()
|
||||
.any(|span| span["request_id"] == "original-multipart-request")
|
||||
);
|
||||
let manager = Arc::new(FastObjectLockManager::new());
|
||||
let stale_client = LocalClient::with_manager(Arc::new(GlobalLockManager::Enabled(manager.clone())));
|
||||
let stale_request = LockRequest::new(
|
||||
ObjectKey::new("bucket", "externally-released"),
|
||||
LockType::Exclusive,
|
||||
"stale-owner".to_owned(),
|
||||
);
|
||||
assert!(
|
||||
stale_client
|
||||
.acquire_lock(&stale_request)
|
||||
.await
|
||||
.expect("real stale owner")
|
||||
.success
|
||||
);
|
||||
assert_eq!(manager.force_unlock(&stale_request.resource), 1);
|
||||
let mut successor = manager
|
||||
.acquire_write_lock(stale_request.resource.clone(), "successor-owner")
|
||||
.await
|
||||
.expect("real successor guard");
|
||||
assert!(
|
||||
stale_client
|
||||
.release(&stale_request.lock_id)
|
||||
.await
|
||||
.expect("retire stale client record")
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.get_lock_info(&stale_request.resource)
|
||||
.expect("successor remains locked")
|
||||
.owner
|
||||
.as_ref(),
|
||||
"successor-owner"
|
||||
);
|
||||
assert_eq!(
|
||||
capture
|
||||
.all_rows()
|
||||
.iter()
|
||||
.filter(
|
||||
|row| row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "released"
|
||||
)
|
||||
.count(),
|
||||
4,
|
||||
"a stale guard's failed physical release must not produce an ACK"
|
||||
);
|
||||
let successor_id = successor.guard_id();
|
||||
assert!(successor.release(), "the actual successor release succeeds");
|
||||
assert!(!successor.release(), "a duplicate explicit release cannot complete again");
|
||||
drop(successor);
|
||||
assert_eq!(
|
||||
capture
|
||||
.all_rows()
|
||||
.iter()
|
||||
.filter(|row| row["fields"]["event"] == "namespace_lock_release"
|
||||
&& row["fields"]["state"] == "released"
|
||||
&& row["fields"]["physical_guard_id"] == successor_id)
|
||||
.count(),
|
||||
1,
|
||||
"explicit release followed by Drop records one physical completion"
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
fn namespace() -> NamespaceLockWrapper {
|
||||
let manager = rustfs_lock::GlobalLockManager::Enabled(Arc::new(rustfs_lock::fast_lock::FastObjectLockManager::new()));
|
||||
NamespaceLockWrapper::new(
|
||||
|
||||
@@ -1235,7 +1235,13 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
pub(crate) fn metric_epoch_for_reconnect(&self) -> Self {
|
||||
Self::new()
|
||||
let health = Self::new();
|
||||
// Explicit test faults stay attached to the drive across handle renewal.
|
||||
#[cfg(test)]
|
||||
if self.test_forced_offline.load(Ordering::Acquire) {
|
||||
health.force_offline_for_test();
|
||||
}
|
||||
health
|
||||
}
|
||||
|
||||
pub fn last_capacity_snapshot(&self) -> Option<(u64, u64, u64, u64)> {
|
||||
@@ -2664,6 +2670,11 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn is_online(&self) -> bool {
|
||||
// A matching format does not make a handle usable while its I/O gate is faulty.
|
||||
// Reconnect must renew it even if its old recovery monitor has stopped.
|
||||
if self.health.is_faulty() {
|
||||
return false;
|
||||
}
|
||||
let Ok(Some(disk_id)) = self.disk.get_disk_id().await else {
|
||||
return false;
|
||||
};
|
||||
@@ -4695,6 +4706,38 @@ mod tests {
|
||||
assert_eq!(snapshot.total_errors_availability, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_online_status_matches_io_health_and_disk_identity() {
|
||||
let dir = tempfile::tempdir().expect("create a drive root");
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 root")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let meta_dir = dir.path().join(RUSTFS_META_BUCKET);
|
||||
tokio::fs::create_dir_all(&meta_dir).await.expect("create metadata directory");
|
||||
let mut format = FormatV3::new(1, 1);
|
||||
format.erasure.this = format.erasure.sets[0][0];
|
||||
tokio::fs::write(meta_dir.join(FORMAT_CONFIG_FILE), format.to_json().expect("serialize format"))
|
||||
.await
|
||||
.expect("persist original identity");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("open formatted drive"));
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
assert!(wrapper.is_online().await, "healthy formatted drive is online");
|
||||
wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Suspect);
|
||||
assert!(wrapper.is_online().await, "suspect drives still admit foreground I/O");
|
||||
wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
|
||||
assert!(!wrapper.is_online().await, "a readable format cannot make a faulty handle usable");
|
||||
assert!(!wrapper.health.mark_recovery_success(&endpoint, "test_probe"));
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Returning);
|
||||
assert!(
|
||||
!wrapper.is_online().await,
|
||||
"returning drives retain the faulty I/O gate until recovery completes"
|
||||
);
|
||||
wrapper.reset_health_for_store_init_retry();
|
||||
wrapper.set_disk_id_state(Some(Uuid::new_v4())).await;
|
||||
assert!(!wrapper.is_online().await, "healthy state must not bypass a mismatched disk identity");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fresh_capacity_keeps_disk_identity_precheck_without_requiring_id_in_snapshot() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
@@ -159,8 +159,18 @@ pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
|
||||
std::fs::symlink_metadata(path)
|
||||
}
|
||||
|
||||
pub fn create_dir_all_std(path: &Path) -> io::Result<()> {
|
||||
match std::fs::create_dir(path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => Ok(()),
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => std::fs::create_dir_all(path),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::create_dir_all(path.as_ref()).await
|
||||
let path = path.as_ref().to_path_buf();
|
||||
tokio::task::spawn_blocking(move || create_dir_all_std(&path)).await?
|
||||
}
|
||||
|
||||
fn is_dir_error(e: &io::Error) -> bool {
|
||||
@@ -210,6 +220,7 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::create_dir(path.as_ref()).await
|
||||
}
|
||||
@@ -239,7 +250,7 @@ impl BucketExistenceCache {
|
||||
}
|
||||
}
|
||||
|
||||
fn check_exists(&self, path: &PathBuf) -> Option<bool> {
|
||||
fn check_exists(&self, path: &Path) -> Option<bool> {
|
||||
let mut cache = self.cache.lock().ok()?;
|
||||
if let Some((timestamp, exists)) = cache.get(path) {
|
||||
if timestamp.elapsed() < self.ttl {
|
||||
@@ -256,7 +267,7 @@ impl BucketExistenceCache {
|
||||
}
|
||||
}
|
||||
|
||||
fn invalidate(&self, path: &PathBuf) {
|
||||
fn invalidate(&self, path: &Path) {
|
||||
if let Ok(mut cache) = self.cache.lock() {
|
||||
cache.remove(path);
|
||||
}
|
||||
@@ -268,15 +279,16 @@ static BUCKET_EXISTENCE_CACHE: std::sync::LazyLock<BucketExistenceCache> =
|
||||
|
||||
/// Cached access check - reduces statx syscalls
|
||||
pub async fn cached_access(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
let path_buf = path.as_ref().to_path_buf();
|
||||
let path = path.as_ref();
|
||||
|
||||
if let Some(exists) = BUCKET_EXISTENCE_CACHE.check_exists(&path_buf) {
|
||||
if let Some(exists) = BUCKET_EXISTENCE_CACHE.check_exists(path) {
|
||||
if exists {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(io::Error::new(io::ErrorKind::NotFound, "bucket not found (cached)"));
|
||||
}
|
||||
|
||||
let path_buf = path.to_path_buf();
|
||||
let result = fs::metadata(&path_buf).await;
|
||||
|
||||
match &result {
|
||||
@@ -292,7 +304,7 @@ pub async fn cached_access(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
|
||||
pub fn invalidate_bucket_cache(path: impl AsRef<Path>) {
|
||||
BUCKET_EXISTENCE_CACHE.invalidate(&path.as_ref().to_path_buf());
|
||||
BUCKET_EXISTENCE_CACHE.invalidate(path.as_ref());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -115,7 +115,7 @@ use time::OffsetDateTime;
|
||||
use tokio::fs::{self, File};
|
||||
#[cfg(not(unix))]
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::io::{AsyncRead, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind, ReadBuf};
|
||||
use tokio::io::{AsyncRead, AsyncSeekExt, AsyncWrite, ErrorKind, ReadBuf};
|
||||
use tokio::sync::{Notify, RwLock, Semaphore};
|
||||
use tokio::time::{Instant, Sleep, interval_at, timeout};
|
||||
use tracing::{debug, error, info, warn};
|
||||
@@ -301,9 +301,10 @@ fn read_all_data_std(
|
||||
let size = usize::try_from(metadata.len()).map_err(|err| ReadAllError::Disk(DiskError::other(err)))?;
|
||||
let mut bytes = Vec::new();
|
||||
bytes
|
||||
.try_reserve_exact(size)
|
||||
.try_reserve_exact(size.saturating_add(1))
|
||||
.map_err(|err| ReadAllError::Disk(Error::other(err)))?;
|
||||
std::io::Read::read_to_end(&mut file, &mut bytes).map_err(|err| ReadAllError::Disk(to_file_error(err).into()))?;
|
||||
std::io::Read::read_to_end(&mut std::io::Read::take(&mut file, metadata.len()), &mut bytes)
|
||||
.map_err(|err| ReadAllError::Disk(to_file_error(err).into()))?;
|
||||
|
||||
let modtime = metadata.modified().ok().map(OffsetDateTime::from);
|
||||
Ok((bytes, modtime))
|
||||
@@ -8379,7 +8380,7 @@ impl LocalDisk {
|
||||
return Err(DiskError::FileNotFound);
|
||||
}
|
||||
|
||||
let meta_path = path_join(&[file_path.as_ref(), Path::new(STORAGE_FORMAT_FILE)]);
|
||||
let meta_path = file_path.as_ref().join(STORAGE_FORMAT_FILE);
|
||||
|
||||
let res = {
|
||||
if read_data {
|
||||
@@ -8415,8 +8416,7 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
#[hotpath::measure(impl_type = "LocalDisk")]
|
||||
async fn read_metadata_with_dmtime(&self, file_path: impl AsRef<Path>) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
|
||||
check_path_length(file_path.as_ref().to_string_lossy().as_ref())?;
|
||||
async fn read_metadata_with_dmtime(&self, file_path: PathBuf) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
|
||||
let io_permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
|
||||
|
||||
// HP-12 item 1 (sub-change A): fold the open + fstat + bounded xl.meta
|
||||
@@ -8429,9 +8429,7 @@ impl LocalDisk {
|
||||
// - is_dir -> Error::FileNotFound (NOT to_file_error(EISDIR))
|
||||
// - metadata failure -> to_file_error
|
||||
// - parse failure -> propagated verbatim from read_xl_meta_no_data_sync (`?`)
|
||||
let (data, modtime) = self
|
||||
.run_metadata_read(io_permit, file_path.as_ref().to_path_buf(), read_metadata_only_std)
|
||||
.await??;
|
||||
let (data, modtime) = self.run_metadata_read(io_permit, file_path, read_metadata_only_std).await??;
|
||||
|
||||
Ok((data, modtime))
|
||||
}
|
||||
@@ -9462,99 +9460,62 @@ impl LocalDisk {
|
||||
skip_parent
|
||||
};
|
||||
|
||||
match data {
|
||||
InternalBuf::Ref(buf) => {
|
||||
let mut f = self.open_file(file_path, O_CREATE | O_WRONLY | O_TRUNC, skip_parent).await?;
|
||||
f.write_all(buf).await.map_err(to_file_error)?;
|
||||
if sync != SyncMode::None {
|
||||
f.sync_data().await.map_err(to_file_error)?;
|
||||
// Persist the directory entry too, so a freshly created file
|
||||
// survives power loss along with its contents. Skipped for
|
||||
// FileOnly: the caller renames the file away immediately.
|
||||
if sync == SyncMode::FileAndDir
|
||||
&& let Some(parent) = file_path.parent()
|
||||
{
|
||||
os::fsync_dir(parent).await.map_err(to_file_error)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
InternalBuf::Owned(buf) => {
|
||||
let path = file_path.to_path_buf();
|
||||
if let Some(parent) = path.parent()
|
||||
&& parent != skip_parent
|
||||
{
|
||||
os::make_dir_all(parent, skip_parent).await?;
|
||||
}
|
||||
|
||||
let root = self.publication_root.clone();
|
||||
let operation = move |is_cancelled: &dyn Fn() -> bool| {
|
||||
if is_cancelled() {
|
||||
return Err(std::io::Error::other("blocking namespace operation cancelled"));
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_owned_file_write_before_open(&path);
|
||||
|
||||
let (mut file, parent_handle) = if sync == SyncMode::FileAndDir {
|
||||
root.create_truncate_with_parent(&path)?
|
||||
} else {
|
||||
(root.create_truncate(&path)?, None)
|
||||
};
|
||||
std::io::Write::write_all(&mut file, buf.as_ref())?;
|
||||
if sync != SyncMode::None {
|
||||
file.sync_data()?;
|
||||
// FileOnly callers rename the file away, so the tmp
|
||||
// directory entry never needs to become durable.
|
||||
if sync == SyncMode::FileAndDir
|
||||
&& let Some(parent) = path.parent()
|
||||
{
|
||||
os::fsync_directory_handle_std(parent, parent_handle.as_ref())?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let result = if sync == SyncMode::None {
|
||||
self.blocking_queue.run_cancellable(operation).await
|
||||
} else {
|
||||
self.blocking_queue.run_fsync_cancellable(operation).await
|
||||
};
|
||||
result.map_err(to_file_error)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn open_file(&self, path: impl AsRef<Path>, mode: usize, skip_parent: impl AsRef<Path>) -> Result<File> {
|
||||
let mut skip_parent = skip_parent.as_ref();
|
||||
if skip_parent.as_os_str().is_empty() {
|
||||
skip_parent = self.io_root();
|
||||
}
|
||||
|
||||
if let Some(parent) = path.as_ref().parent()
|
||||
&& parent != skip_parent
|
||||
{
|
||||
os::make_dir_all(parent, skip_parent).await?;
|
||||
}
|
||||
|
||||
let f = if mode == O_CREATE | O_WRONLY | O_TRUNC {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
let root = self.publication_root.clone();
|
||||
File::from_std(
|
||||
self.blocking_queue
|
||||
.run_cancellable(move |is_cancelled| {
|
||||
if is_cancelled() {
|
||||
return Err(std::io::Error::other("blocking namespace operation cancelled"));
|
||||
}
|
||||
root.create_truncate(&path)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)?,
|
||||
)
|
||||
} else {
|
||||
super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?
|
||||
#[cfg(test)]
|
||||
let is_owned = matches!(data, InternalBuf::Owned(_));
|
||||
let buf = match data {
|
||||
InternalBuf::Ref(slice) => Bytes::copy_from_slice(slice),
|
||||
InternalBuf::Owned(bytes) => bytes,
|
||||
};
|
||||
|
||||
Ok(f)
|
||||
let path = file_path.to_path_buf();
|
||||
let create_parent = if let Some(parent) = path.parent()
|
||||
&& parent != skip_parent
|
||||
{
|
||||
os::check_path_length(parent.to_string_lossy().as_ref())?;
|
||||
!(!skip_parent.as_os_str().is_empty() && skip_parent.starts_with(parent))
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
let root = self.publication_root.clone();
|
||||
let operation = move |is_cancelled: &dyn Fn() -> bool| {
|
||||
if is_cancelled() {
|
||||
return Err(std::io::Error::other("blocking namespace operation cancelled"));
|
||||
}
|
||||
if create_parent && let Some(parent) = path.parent() {
|
||||
super::fs::create_dir_all_std(parent)?;
|
||||
}
|
||||
#[cfg(test)]
|
||||
if is_owned {
|
||||
run_owned_file_write_before_open(&path);
|
||||
}
|
||||
|
||||
let (mut file, parent_handle) = if sync == SyncMode::FileAndDir {
|
||||
root.create_truncate_with_parent(&path)?
|
||||
} else {
|
||||
(root.create_truncate(&path)?, None)
|
||||
};
|
||||
std::io::Write::write_all(&mut file, buf.as_ref())?;
|
||||
if sync != SyncMode::None {
|
||||
file.sync_data()?;
|
||||
// FileOnly callers rename the file away, so the tmp
|
||||
// directory entry never needs to become durable.
|
||||
if sync == SyncMode::FileAndDir
|
||||
&& let Some(parent) = path.parent()
|
||||
{
|
||||
os::fsync_directory_handle_std(parent, parent_handle.as_ref())?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let result = if sync == SyncMode::None {
|
||||
self.blocking_queue.run_cancellable(operation).await
|
||||
} else {
|
||||
self.blocking_queue.run_fsync_cancellable(operation).await
|
||||
};
|
||||
result.map_err(to_file_error)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
@@ -10516,10 +10477,11 @@ async fn read_file_all(path: impl AsRef<Path>, root: &os::PublicationRoot) -> Re
|
||||
let root = root.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let result = (|| {
|
||||
let mut file = root.open_readonly(&path)?;
|
||||
let file = root.open_readonly(&path)?;
|
||||
let metadata = file.metadata()?;
|
||||
let mut data = Vec::new();
|
||||
std::io::Read::read_to_end(&mut file, &mut data)?;
|
||||
let size = usize::try_from(metadata.len()).map_err(std::io::Error::other)?;
|
||||
let mut data = Vec::with_capacity(size.saturating_add(1));
|
||||
std::io::Read::read_to_end(&mut std::io::Read::take(file, metadata.len()), &mut data)?;
|
||||
Ok::<_, std::io::Error>((Bytes::from(data), metadata))
|
||||
})();
|
||||
result
|
||||
@@ -10575,7 +10537,21 @@ fn local_disk_object_path(root: &Path, bucket: &str, key: &str) -> Result<PathBu
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[inline]
|
||||
fn is_clean_relative_segment(s: &str) -> bool {
|
||||
let b = s.as_bytes();
|
||||
!matches!(b.first(), None | Some(b'/' | b'.')) && !b.ends_with(b"/") && !b.windows(2).any(|w| matches!(w, b"//" | b"/."))
|
||||
}
|
||||
|
||||
fn build_local_disk_object_path(root: &Path, bucket: &str, key: &str) -> (PathBuf, PathBuf) {
|
||||
#[cfg(not(windows))]
|
||||
if is_clean_relative_segment(bucket) && is_clean_relative_segment(key) {
|
||||
let bucket_path = root.join(bucket);
|
||||
let path = bucket_path.join(key);
|
||||
return (bucket_path, path);
|
||||
}
|
||||
|
||||
let cache_key = if key.is_empty() {
|
||||
bucket.to_string()
|
||||
} else {
|
||||
@@ -10740,9 +10716,10 @@ fn reject_local_disk_symlink_components(root: &Path, path: &Path) -> Result<()>
|
||||
fn normalize_path_cow(path: &Path) -> std::borrow::Cow<'_, Path> {
|
||||
// Windows prefixes retain the existing PathBuf reconstruction semantics.
|
||||
#[cfg(not(windows))]
|
||||
if !path
|
||||
.components()
|
||||
.any(|c| matches!(c, std::path::Component::ParentDir | std::path::Component::CurDir))
|
||||
if !path.as_os_str().as_encoded_bytes().contains(&b'.')
|
||||
|| !path
|
||||
.components()
|
||||
.any(|c| matches!(c, std::path::Component::ParentDir | std::path::Component::CurDir))
|
||||
{
|
||||
return std::borrow::Cow::Borrowed(path);
|
||||
}
|
||||
@@ -12462,12 +12439,12 @@ impl DiskAPI for LocalDisk {
|
||||
let src_volume_dir = self.io_get_bucket_path(src_volume)?;
|
||||
let dst_volume_dir = self.io_get_bucket_path(dst_volume)?;
|
||||
if !skip_access_checks(src_volume) {
|
||||
access(&src_volume_dir)
|
||||
cached_access(&src_volume_dir)
|
||||
.await
|
||||
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
|
||||
}
|
||||
if !skip_access_checks(dst_volume) {
|
||||
access(&dst_volume_dir)
|
||||
cached_access(&dst_volume_dir)
|
||||
.await
|
||||
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
|
||||
}
|
||||
@@ -12546,7 +12523,7 @@ impl DiskAPI for LocalDisk {
|
||||
if !origvolume.is_empty() {
|
||||
let origvolume_dir = self.io_get_bucket_path(origvolume)?;
|
||||
if !skip_access_checks(origvolume) {
|
||||
access(origvolume_dir)
|
||||
cached_access(origvolume_dir)
|
||||
.await
|
||||
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
|
||||
}
|
||||
@@ -13176,7 +13153,7 @@ impl DiskAPI for LocalDisk {
|
||||
if !org_volume.is_empty() {
|
||||
let org_volume_path = self.io_get_bucket_path(org_volume)?;
|
||||
if !skip_access_checks(org_volume) {
|
||||
access(&org_volume_path)
|
||||
cached_access(&org_volume_path)
|
||||
.await
|
||||
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ use super::{
|
||||
LocalDisk,
|
||||
multipart_source_state::{
|
||||
LoadedSource, SourceHead, SourceState, SourceStore, decode, encode, load_source, raw_sha, read_optional,
|
||||
read_optional_in_pinned,
|
||||
},
|
||||
};
|
||||
use crate::disk::{
|
||||
@@ -127,20 +128,6 @@ pub(super) fn validate_retired_head(
|
||||
}
|
||||
|
||||
impl SourceStore {
|
||||
pub async fn root_missing(&self) -> Result<bool> {
|
||||
let source = self.source.clone();
|
||||
let base = self.base.clone();
|
||||
let root = self.publication_root.clone();
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
match os::pin_multipart_source_directory(&source, &base, &root) {
|
||||
Ok(_) => Ok(false),
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => Ok(true),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
fn verify_abort_cohort(&self, certificate: &MultipartAbortCertificate) -> Result<()> {
|
||||
certificate.verify_with_authority(&self.authority)?;
|
||||
if certificate.disk_ids != self.disk_ids
|
||||
@@ -265,7 +252,7 @@ impl SourceStore {
|
||||
let certificate = self.read_abort_certificate(&loaded).await?;
|
||||
Ok(MultipartSourceInspection {
|
||||
disk_id: self.disk_id,
|
||||
root_missing: self.root_missing().await?,
|
||||
root_missing: loaded.root_missing,
|
||||
abort_round: loaded
|
||||
.state
|
||||
.as_ref()
|
||||
@@ -620,12 +607,17 @@ pub(super) struct OrdinaryMultipartDeleteCheck {
|
||||
}
|
||||
impl OrdinaryMultipartDeleteCheck {
|
||||
pub(super) fn validate(&self) -> Result<()> {
|
||||
if let Some(raw) = read_optional(&self.source.join(STATE), &self.base, &self.root)? {
|
||||
let pinned = match os::pin_multipart_source_directory(&self.source, &self.base, &self.root) {
|
||||
Ok(pinned) => pinned,
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
if let Some(raw) = read_optional_in_pinned(&pinned, &self.source, STATE, &self.base, &self.root)? {
|
||||
let state: SourceState = decode(&raw)?;
|
||||
if state.abort.is_some() || state.phase == MultipartSourcePhase::Aborting {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
} else if os::multipart_source_has_native_controls(&self.source, &self.base, &self.root)? {
|
||||
} else if os::multipart_source_has_native_controls_in_pinned(&self.source, &pinned)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -170,7 +170,7 @@ impl LocalDisk {
|
||||
recovery: loaded.state.as_ref().and_then(|state| state.recovery.clone()),
|
||||
negative_certificate,
|
||||
ready_certificate,
|
||||
root_missing: store.root_missing().await?,
|
||||
root_missing: loaded.root_missing,
|
||||
abort_round: loaded
|
||||
.state
|
||||
.as_ref()
|
||||
|
||||
@@ -106,6 +106,7 @@ impl SourceHead {
|
||||
}
|
||||
|
||||
pub(super) struct LoadedSource {
|
||||
pub root_missing: bool,
|
||||
pub state: Option<SourceState>,
|
||||
pub head: Option<SourceHead>,
|
||||
pub original: Option<Bytes>,
|
||||
@@ -224,11 +225,31 @@ pub(super) fn read_optional(path: &Path, base: &Path, root: &os::PublicationRoot
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
}
|
||||
pub(super) fn read_optional_in_pinned(
|
||||
pinned: &os::ExistingBaseDirectoryGuard,
|
||||
parent: &Path,
|
||||
name: &str,
|
||||
base: &Path,
|
||||
root: &os::PublicationRoot,
|
||||
) -> Result<Option<Bytes>> {
|
||||
match os::read_multipart_source_metadata_in_pinned(pinned, parent, std::ffi::OsStr::new(name), base, root) {
|
||||
Ok(bytes) => Ok(Some(Bytes::from(bytes))),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
}
|
||||
/// Canonical source mutations are admitted under SourceRW at the syscall,
|
||||
/// not on the earlier async caller that may have timed out.
|
||||
pub(super) fn validate_source_mutation(source: &Path, base: &Path, root: &os::PublicationRoot, initial: bool) -> Result<()> {
|
||||
let state = read_optional(&source.join(STATE), base, root)?;
|
||||
let head = read_optional(&source.join(STORAGE_FORMAT_FILE), base, root)?;
|
||||
let pinned = match os::pin_multipart_source_directory(source, base, root) {
|
||||
Ok(pinned) => pinned,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
return if initial { Ok(()) } else { Err(DiskError::FileNotFound) };
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let state = read_optional_in_pinned(&pinned, source, STATE, base, root)?;
|
||||
let head = read_optional_in_pinned(&pinned, source, STORAGE_FORMAT_FILE, base, root)?;
|
||||
let Some(head) = head else {
|
||||
return if initial && state.is_none() {
|
||||
Ok(())
|
||||
@@ -348,6 +369,9 @@ impl SourceStore {
|
||||
.map_err(DiskError::from)?
|
||||
}
|
||||
pub async fn persist(&self, state: &SourceState) -> Result<()> {
|
||||
self.persist_with_source_sync(state, false).await
|
||||
}
|
||||
async fn persist_with_source_sync(&self, state: &SourceState, sync_source_before: bool) -> Result<()> {
|
||||
state.validate(self.disk_id, &self.path)?;
|
||||
let raw = encode(state)?;
|
||||
let source = self.source.clone();
|
||||
@@ -365,6 +389,9 @@ impl SourceStore {
|
||||
});
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
let _pin = os::pin_multipart_source_directory(&source, &base, &root)?;
|
||||
if sync_source_before {
|
||||
os::fsync_dir_std(&source)?;
|
||||
}
|
||||
if let Some(path) = terminal {
|
||||
os::replace_multipart_source_metadata(&path, &raw, &base, &root)?;
|
||||
}
|
||||
@@ -373,6 +400,7 @@ impl SourceStore {
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub async fn original(&self, state: &SourceState) -> Result<Bytes> {
|
||||
let path = original_path(&self.source, state.backup_owner);
|
||||
let base = self.base.clone();
|
||||
@@ -490,7 +518,7 @@ impl SourceStore {
|
||||
if encode(actual)? != encode(state)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let raw = self.original(state).await?;
|
||||
let raw = loaded.original.as_ref().ok_or(DiskError::FileCorrupt)?;
|
||||
let allowed = match loaded.head.as_ref() {
|
||||
None => {
|
||||
state.phase == MultipartSourcePhase::Recovering
|
||||
@@ -551,7 +579,7 @@ impl SourceStore {
|
||||
if !allowed {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let mut info = file_info(&raw, RUSTFS_META_MULTIPART_BUCKET, &self.path, state.lineage.original_version_id)?;
|
||||
let mut info = file_info(raw, RUSTFS_META_MULTIPART_BUCKET, &self.path, state.lineage.original_version_id)?;
|
||||
strip_native_controls(&mut info.metadata);
|
||||
mark_native_source(&mut info);
|
||||
set_source_status(
|
||||
@@ -772,13 +800,32 @@ pub(super) fn load_source(
|
||||
physical_slot: usize,
|
||||
disk_ids: &[Uuid],
|
||||
) -> Result<LoadedSource> {
|
||||
let state_raw = read_optional(&source.join(STATE), base, root)?;
|
||||
let pinned = match os::pin_multipart_source_directory(source, base, root) {
|
||||
Ok(pinned) => pinned,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
return Ok(LoadedSource {
|
||||
root_missing: true,
|
||||
state: None,
|
||||
head: None,
|
||||
original: None,
|
||||
info: None,
|
||||
snapshot: None,
|
||||
head_matches_state: false,
|
||||
head_matches_activation: false,
|
||||
});
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let state_raw = read_optional_in_pinned(&pinned, source, STATE, base, root)?;
|
||||
let state: Option<SourceState> = state_raw.as_deref().map(decode).transpose()?;
|
||||
if state.is_none() && os::multipart_source_has_native_controls(source, base, root)? {
|
||||
if state.is_none() && os::multipart_source_has_native_controls_in_pinned(source, &pinned)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let head_path = source.join(STORAGE_FORMAT_FILE);
|
||||
let head = if let Some(state) = state.as_ref() {
|
||||
let head = if let Some(state) = state
|
||||
.as_ref()
|
||||
.filter(|state| state.prepared_length.is_some_and(|len| len > 64 * 1024))
|
||||
{
|
||||
let head_path = source.join(STORAGE_FORMAT_FILE);
|
||||
let maximum = state
|
||||
.prepared_length
|
||||
.unwrap_or(0)
|
||||
@@ -789,7 +836,8 @@ pub(super) fn load_source(
|
||||
let raw = if is_proxy {
|
||||
None
|
||||
} else {
|
||||
let raw = read_optional(&head_path, base, root)?.ok_or(DiskError::FileCorrupt)?;
|
||||
let raw = read_optional_in_pinned(&pinned, source, STORAGE_FORMAT_FILE, base, root)?
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
if raw_sha(&raw) != sha256 {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
@@ -801,10 +849,17 @@ pub(super) fn load_source(
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
} else {
|
||||
read_optional(&head_path, base, root)?.map(|raw| SourceHead {
|
||||
sha256: raw_sha(&raw),
|
||||
length: raw.len() as u64,
|
||||
raw: Some(raw),
|
||||
read_optional_in_pinned(&pinned, source, STORAGE_FORMAT_FILE, base, root)?.map(|raw| {
|
||||
let sha256 = raw_sha(&raw);
|
||||
let length = raw.len() as u64;
|
||||
let is_proxy = state
|
||||
.as_ref()
|
||||
.is_some_and(|state| Some(sha256) == state.prepared_sha256 && Some(length) == state.prepared_length);
|
||||
SourceHead {
|
||||
sha256,
|
||||
length,
|
||||
raw: (!is_proxy).then_some(raw),
|
||||
}
|
||||
})
|
||||
};
|
||||
let mut original = None;
|
||||
@@ -950,6 +1005,7 @@ pub(super) fn load_source(
|
||||
head_matches_state = true;
|
||||
}
|
||||
Ok(LoadedSource {
|
||||
root_missing: false,
|
||||
state,
|
||||
head,
|
||||
original,
|
||||
@@ -1587,10 +1643,8 @@ impl NativeMultipartPublication {
|
||||
}
|
||||
}
|
||||
self.store.sync_target_with_lease(&self.publication, target_lease).await?;
|
||||
let source = self.store.source.clone();
|
||||
os::run_blocking_namespace_operation(self.store.lease.clone(), move || os::fsync_dir_std(&source)).await?;
|
||||
state.advance(MultipartSourcePhase::ConsumedUnsealed)?;
|
||||
self.store.persist(&state).await?;
|
||||
self.store.persist_with_source_sync(&state, true).await?;
|
||||
*self.state.lock() = state;
|
||||
let loaded = self.store.load().await?;
|
||||
let snapshot = loaded.snapshot.ok_or(DiskError::FileCorrupt)?;
|
||||
|
||||
@@ -411,15 +411,23 @@ impl LocalDisk {
|
||||
}
|
||||
let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
|
||||
if incoming.data.is_none() && !incoming.is_remote() {
|
||||
let source_data_dir = source_object.join(source_dir.to_string());
|
||||
let pinned_source_data = (!incoming.parts.is_empty())
|
||||
.then(|| os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root))
|
||||
.transpose()?;
|
||||
let mut names = Vec::with_capacity(incoming.parts.len().saturating_mul(3));
|
||||
for part in &incoming.parts {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let path = source_object
|
||||
.join(source_dir.to_string())
|
||||
.join(format!("part.{}.meta", part.number));
|
||||
let bytes = os::read_multipart_source_metadata(&path, &base, &publication_root)?;
|
||||
let meta_name = format!("part.{}.meta", part.number);
|
||||
let bytes = os::read_multipart_source_metadata_in_pinned(
|
||||
pinned_source_data.as_ref().expect("pinned source data directory"),
|
||||
&source_data_dir,
|
||||
std::ffi::OsStr::new(&meta_name),
|
||||
&base,
|
||||
&publication_root,
|
||||
)?;
|
||||
let actual = rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?;
|
||||
let normalized = i64::try_from(actual.size)
|
||||
.map_err(std::io::Error::other)?
|
||||
@@ -441,13 +449,13 @@ impl LocalDisk {
|
||||
// Preserve the native completed-MPU directory format. This
|
||||
// immutable, validated generation record is distinct from
|
||||
// the upload's mutable top-level xl.meta.
|
||||
names.push(format!("part.{}.meta", part.number));
|
||||
names.push(meta_name);
|
||||
if let Some(integrity) = &part.integrity {
|
||||
names.push(integrity.file_name());
|
||||
}
|
||||
}
|
||||
os::hardlink_multipart_source_files(
|
||||
&source_object.join(source_dir.to_string()),
|
||||
&source_data_dir,
|
||||
&private_object.join(final_dir.to_string()),
|
||||
&names,
|
||||
&base,
|
||||
|
||||
+135
-64
@@ -73,6 +73,10 @@ pub fn check_path_length(path_name: &str) -> Result<()> {
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
|
||||
if path_name.len() <= 255 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Check each path segment length is > 255 on all Unix
|
||||
// platforms, look for this value as NAME_MAX in
|
||||
// /usr/include/linux/limits.h
|
||||
@@ -3368,6 +3372,29 @@ fn open_existing_directory_below_base_std(
|
||||
use rustix::fs::{Mode, OFlags, openat};
|
||||
let relative = existing_base_directory_suffix(directory, base)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use rustix::fs::{ResolveFlags, openat2};
|
||||
use std::os::fd::AsFd;
|
||||
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
||||
let owned_base;
|
||||
let base_fd = if base == publication_root.io_path.as_path() || base == publication_root.path.as_path() {
|
||||
publication_root.directory.as_fd()
|
||||
} else {
|
||||
owned_base = std::os::fd::OwnedFd::from(publication_root.open_directory(base)?);
|
||||
owned_base.as_fd()
|
||||
};
|
||||
let target = if relative.as_os_str().is_empty() {
|
||||
Path::new(".")
|
||||
} else {
|
||||
relative
|
||||
};
|
||||
match openat2(base_fd, target, flags, Mode::empty(), ResolveFlags::BENEATH | ResolveFlags::NO_SYMLINKS) {
|
||||
Ok(leaf) => return Ok(vec![leaf]),
|
||||
Err(rustix::io::Errno::NOSYS) => {}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
let base = publication_root.open_directory(base)?.into();
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let base = {
|
||||
@@ -4921,12 +4948,16 @@ impl LinuxCommitDirectories {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
// A pinned directory may have been unlinked or replaced. Do not report a
|
||||
// successful publication into a detached object tree.
|
||||
for (path, directory) in [
|
||||
(&self.source_path, &self.source),
|
||||
(&self.destination_path, self.prepare_destination()?),
|
||||
] {
|
||||
let current = self.root.open_directory(path)?.metadata()?;
|
||||
if (current.dev(), current.ino()) != (directory.device, directory.inode) {
|
||||
let current = self.root.open_directory(&self.source_path)?.metadata()?;
|
||||
if (current.dev(), current.ino()) != (self.source.device, self.source.inode) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "commit directory identity changed"));
|
||||
}
|
||||
let destination = self.prepare_destination()?;
|
||||
if self.source_path != self.destination_path
|
||||
|| (self.source.device, self.source.inode) != (destination.device, destination.inode)
|
||||
{
|
||||
let current = self.root.open_directory(&self.destination_path)?.metadata()?;
|
||||
if (current.dev(), current.ino()) != (destination.device, destination.inode) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "commit directory identity changed"));
|
||||
}
|
||||
}
|
||||
@@ -5219,12 +5250,20 @@ pub(crate) fn prepare_rename_commit_guard(
|
||||
existing_base_directory_suffix(destination_parent, destination_base)?;
|
||||
let source = LinuxCommitDirectory::new(publication_root.open_directory(source_parent)?)?;
|
||||
let destination = std::sync::OnceLock::new();
|
||||
match publication_root.open_directory(destination_parent) {
|
||||
Ok(directory) => {
|
||||
let _ = destination.set(LinuxCommitDirectory::new(directory)?);
|
||||
if source_parent == destination_parent {
|
||||
let _ = destination.set(LinuxCommitDirectory {
|
||||
file: Arc::clone(&source.file),
|
||||
device: source.device,
|
||||
inode: source.inode,
|
||||
});
|
||||
} else {
|
||||
match publication_root.open_directory(destination_parent) {
|
||||
Ok(directory) => {
|
||||
let _ = destination.set(LinuxCommitDirectory::new(directory)?);
|
||||
}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
Ok(RenameCommitGuard {
|
||||
directories: Arc::new(LinuxCommitDirectories {
|
||||
@@ -6084,12 +6123,20 @@ fn existing_base_directory_suffix<'a>(dir_path: &'a Path, base_dir: &Path) -> io
|
||||
|
||||
/// Missing state never downgrades surviving native control records to legacy.
|
||||
/// Inspect only this pinned upload root; parts and proof trees are not traversed.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn multipart_source_has_native_controls(source: &Path, base: &Path, root: &PublicationRoot) -> io::Result<bool> {
|
||||
let _pin = match pin_multipart_source_directory(source, base, root) {
|
||||
let pin = match pin_multipart_source_directory(source, base, root) {
|
||||
Ok(pin) => pin,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
multipart_source_has_native_controls_in_pinned(source, &pin)
|
||||
}
|
||||
|
||||
pub(crate) fn multipart_source_has_native_controls_in_pinned(
|
||||
source: &Path,
|
||||
_pin: &ExistingBaseDirectoryGuard,
|
||||
) -> io::Result<bool> {
|
||||
let mut entries = std::fs::read_dir(source)?;
|
||||
for _ in 0..READ_DIR_PROBE_RAW_LIMIT {
|
||||
let Some(entry) = entries.next() else {
|
||||
@@ -6393,15 +6440,10 @@ pub(crate) fn sync_multipart_owned_record_parents(
|
||||
if !directory.starts_with(source) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "native record barrier escaped source"));
|
||||
}
|
||||
let pinned = pin_multipart_source_directory(directory, base, publication_root)?;
|
||||
let mut pinned = pin_multipart_source_directory(directory, base, publication_root)?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let opened = std::fs::File::from(
|
||||
pinned
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("empty native directory guard"))?
|
||||
.try_clone()?,
|
||||
);
|
||||
let opened = std::fs::File::from(pinned.pop().ok_or_else(|| io::Error::other("empty native directory guard"))?);
|
||||
fsync_directory_handle_std(directory, Some(&opened))?;
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
@@ -6513,6 +6555,16 @@ pub(crate) fn read_multipart_source_metadata(
|
||||
read_multipart_metadata(path, base, publication_root, Some(16 * 1024 * 1024))
|
||||
}
|
||||
|
||||
pub(crate) fn read_multipart_source_metadata_in_pinned(
|
||||
pinned: &ExistingBaseDirectoryGuard,
|
||||
parent: &Path,
|
||||
name: &std::ffi::OsStr,
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
read_multipart_metadata_in_pinned(pinned, parent, name, base, publication_root, Some(16 * 1024 * 1024))
|
||||
}
|
||||
|
||||
/// Target FI histories have no source-control size limit. Keep the same
|
||||
/// pinned-parent and no-follow checks when inspecting every referenced DD.
|
||||
pub(crate) fn read_multipart_reclaim_target_metadata(
|
||||
@@ -6523,22 +6575,71 @@ pub(crate) fn read_multipart_reclaim_target_metadata(
|
||||
read_multipart_metadata(path, base, publication_root, None)
|
||||
}
|
||||
|
||||
fn read_multipart_metadata_bytes(mut reader: impl std::io::Read, maximum: Option<u64>) -> io::Result<Vec<u8>> {
|
||||
let mut bytes = Vec::new();
|
||||
match maximum {
|
||||
Some(maximum) => {
|
||||
fn read_multipart_metadata_bytes(
|
||||
mut reader: impl std::io::Read,
|
||||
size_hint: Option<u64>,
|
||||
maximum: Option<u64>,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
if let (Some(len), Some(maximum)) = (size_hint, maximum)
|
||||
&& len > maximum
|
||||
{
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart metadata exceeds its size limit"));
|
||||
}
|
||||
let cap = size_hint
|
||||
.and_then(|len| usize::try_from(len).ok())
|
||||
.map_or(0, |len| len.saturating_add(1));
|
||||
let mut bytes = Vec::with_capacity(cap);
|
||||
match (size_hint, maximum) {
|
||||
(Some(len), _) => {
|
||||
std::io::Read::read_to_end(&mut reader.take(len), &mut bytes)?;
|
||||
}
|
||||
(None, Some(maximum)) => {
|
||||
std::io::Read::read_to_end(&mut reader.take(maximum.saturating_add(1)), &mut bytes)?;
|
||||
if bytes.len() as u64 > maximum {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart metadata exceeds its size limit"));
|
||||
}
|
||||
}
|
||||
None => {
|
||||
(None, None) => {
|
||||
std::io::Read::read_to_end(&mut reader, &mut bytes)?;
|
||||
}
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn read_multipart_metadata_in_pinned(
|
||||
pinned: &ExistingBaseDirectoryGuard,
|
||||
parent: &Path,
|
||||
name: &std::ffi::OsStr,
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
maximum: Option<u64>,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{Mode, OFlags, openat};
|
||||
let _ = (parent, base, publication_root);
|
||||
let directory = pinned
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("missing multipart source directory"))?;
|
||||
let mut file = std::fs::File::from(openat(
|
||||
directory,
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
let metadata = file.metadata()?;
|
||||
if !metadata.is_file() {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart metadata is not a regular file"));
|
||||
}
|
||||
read_multipart_metadata_bytes(&mut file, Some(metadata.len()), maximum)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = pinned;
|
||||
read_multipart_metadata(&parent.join(name), base, publication_root, maximum)
|
||||
}
|
||||
}
|
||||
|
||||
fn read_multipart_metadata(
|
||||
path: &Path,
|
||||
base: &Path,
|
||||
@@ -6547,27 +6648,14 @@ fn read_multipart_metadata(
|
||||
) -> io::Result<Vec<u8>> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{Mode, OFlags, openat};
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| io::Error::other("multipart metadata has no parent"))?;
|
||||
let guards = open_existing_directory_below_base_std(parent, base, publication_root)?;
|
||||
let directory = guards
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("missing multipart source directory"))?;
|
||||
let name = path
|
||||
.file_name()
|
||||
.ok_or_else(|| io::Error::other("multipart metadata has no name"))?;
|
||||
let mut file = std::fs::File::from(openat(
|
||||
directory,
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
if !file.metadata()?.is_file() {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart metadata is not a regular file"));
|
||||
}
|
||||
read_multipart_metadata_bytes(&mut file, maximum)
|
||||
let guards = open_existing_directory_below_base_std(parent, base, publication_root)?;
|
||||
read_multipart_metadata_in_pinned(&guards, parent, name, base, publication_root, maximum)
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
@@ -6598,7 +6686,7 @@ fn read_multipart_metadata(
|
||||
if windows_file_attribute_tag(&file)?.FileAttributes & (FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT) != 0 {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart metadata is not a regular file"));
|
||||
}
|
||||
read_multipart_metadata_bytes(file.as_file_mut(), maximum)
|
||||
read_multipart_metadata_bytes(file.as_file_mut(), None, maximum)
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
{
|
||||
@@ -7234,33 +7322,16 @@ pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Pat
|
||||
/// This function will not create directories under base_dir.
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
if !base_dir.as_ref().to_string_lossy().is_empty() && base_dir.as_ref().starts_with(dir_path.as_ref()) {
|
||||
let dir_path = dir_path.as_ref();
|
||||
let base_dir = base_dir.as_ref();
|
||||
if !base_dir.as_os_str().is_empty() && base_dir.starts_with(dir_path) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Err(e) = super::fs::mkdir(dir_path.as_ref()).await {
|
||||
if e.kind() == io::ErrorKind::AlreadyExists {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if e.kind() != io::ErrorKind::NotFound {
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
if let Some(parent) = dir_path.as_ref().parent() {
|
||||
// Fall back to creating the missing parent chain only when the direct mkdir proves it is required.
|
||||
if let Err(parent_err) = super::fs::make_dir_all(parent).await
|
||||
&& parent_err.kind() != io::ErrorKind::AlreadyExists
|
||||
{
|
||||
return Err(parent_err);
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(retry_err) = super::fs::mkdir(dir_path.as_ref()).await
|
||||
&& retry_err.kind() != io::ErrorKind::AlreadyExists
|
||||
{
|
||||
return Err(retry_err);
|
||||
}
|
||||
if let Err(err) = super::fs::make_dir_all(dir_path).await
|
||||
&& err.kind() != io::ErrorKind::AlreadyExists
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -5232,14 +5232,10 @@ impl SetDisks {
|
||||
let disks = self.get_disks_internal().await;
|
||||
let transaction_path = part_transaction_path(dst_object);
|
||||
let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
|
||||
let rollback_path = format!("{transaction_path}/{PART_TRANSACTION_ROLLBACK}");
|
||||
let current_meta_path = format!("{dst_object}.meta");
|
||||
|
||||
let reads = disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let transaction_meta_path = transaction_meta_path.clone();
|
||||
let rollback_path = rollback_path.clone();
|
||||
let current_meta_path = current_meta_path.clone();
|
||||
async move {
|
||||
let Some(disk) = disk else {
|
||||
return PartTransactionObservation {
|
||||
@@ -5249,44 +5245,20 @@ impl SetDisks {
|
||||
err: Some(DiskError::DiskNotFound),
|
||||
};
|
||||
};
|
||||
let transaction_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &transaction_meta_path).await {
|
||||
Ok(meta) => Some(meta),
|
||||
Err(DiskError::FileNotFound) => None,
|
||||
Err(err) => {
|
||||
return PartTransactionObservation {
|
||||
transaction_meta: None,
|
||||
current_meta: None,
|
||||
rollback: false,
|
||||
err: Some(err),
|
||||
};
|
||||
}
|
||||
};
|
||||
let rollback = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &rollback_path).await {
|
||||
Ok(_) => true,
|
||||
Err(DiskError::FileNotFound) => false,
|
||||
Err(err) => {
|
||||
return PartTransactionObservation {
|
||||
transaction_meta,
|
||||
current_meta: None,
|
||||
rollback: false,
|
||||
err: Some(err),
|
||||
};
|
||||
}
|
||||
};
|
||||
let current_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, ¤t_meta_path).await {
|
||||
Ok(meta) => Some(meta),
|
||||
Err(DiskError::FileNotFound | DiskError::DiskNotFound) => None,
|
||||
Err(_) => None,
|
||||
let (transaction_meta, err) = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &transaction_meta_path).await {
|
||||
Ok(meta) => (Some(meta), None),
|
||||
Err(DiskError::FileNotFound) => (None, None),
|
||||
Err(err) => (None, Some(err)),
|
||||
};
|
||||
PartTransactionObservation {
|
||||
transaction_meta,
|
||||
current_meta,
|
||||
rollback,
|
||||
err: None,
|
||||
current_meta: None,
|
||||
rollback: false,
|
||||
err,
|
||||
}
|
||||
}
|
||||
});
|
||||
let observations = join_all(reads).await;
|
||||
let mut observations = join_all(reads).await;
|
||||
let read_errs = observations
|
||||
.iter()
|
||||
.map(|observation| observation.err.clone())
|
||||
@@ -5303,6 +5275,42 @@ impl SetDisks {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let rollback_path = format!("{transaction_path}/{PART_TRANSACTION_ROLLBACK}");
|
||||
let current_meta_path = format!("{dst_object}.meta");
|
||||
let details = disks.iter().zip(observations.iter_mut()).map(|(disk, observation)| {
|
||||
let disk = disk.clone();
|
||||
let rollback_path = rollback_path.clone();
|
||||
let current_meta_path = current_meta_path.clone();
|
||||
async move {
|
||||
if observation.err.is_some() {
|
||||
return;
|
||||
}
|
||||
let Some(disk) = disk else {
|
||||
observation.err = Some(DiskError::DiskNotFound);
|
||||
return;
|
||||
};
|
||||
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &rollback_path).await {
|
||||
Ok(_) => observation.rollback = true,
|
||||
Err(DiskError::FileNotFound) => {}
|
||||
Err(err) => {
|
||||
observation.err = Some(err);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if let Ok(meta) = disk.read_all(RUSTFS_META_MULTIPART_BUCKET, ¤t_meta_path).await {
|
||||
observation.current_meta = Some(meta);
|
||||
}
|
||||
}
|
||||
});
|
||||
join_all(details).await;
|
||||
let read_errs = observations
|
||||
.iter()
|
||||
.map(|observation| observation.err.clone())
|
||||
.collect::<Vec<_>>();
|
||||
if let Some(err) = reduce_write_quorum_errs(&read_errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let mut current_counts: HashMap<Bytes, usize> = HashMap::new();
|
||||
let mut transaction_meta_values = HashSet::new();
|
||||
for observation in observations.iter().filter(|observation| observation.err.is_none()) {
|
||||
|
||||
@@ -36,13 +36,12 @@ use crate::disk::{
|
||||
self, BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, Disk, DiskAPI,
|
||||
};
|
||||
use crate::set_disk::{
|
||||
DiskError, DiskStore, FileInfo, HashAlgorithm, ReadOptions, SetDisks, build_inline_bitrot_readers_from_refs,
|
||||
can_try_inline_data_shards_direct, codec_streaming_rollout_applies, coding,
|
||||
collect_inline_data_shard_fileinfos_from_observations, file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request,
|
||||
inline_erasure_shard_file_offset, inline_erasure_shard_size, is_get_metadata_data_read_early_stop_enabled,
|
||||
is_get_metadata_early_stop_bounded_fanout_enabled, is_get_metadata_early_stop_enabled,
|
||||
is_get_metadata_non_inline_data_read_early_stop_enabled, is_version_early_stop_enabled, object_fits_single_block,
|
||||
try_read_inline_data_shards_direct,
|
||||
DiskError, DiskStore, FileInfo, HashAlgorithm, ReadOptions, SetDisks, can_try_inline_data_shards_direct,
|
||||
codec_streaming_rollout_applies, collect_inline_data_shard_fileinfos_from_observations, file_info_is_valid_for_metadata,
|
||||
get_metadata_slowtail_fault_request, inline_erasure_shard_file_offset, inline_erasure_shard_size,
|
||||
is_get_metadata_data_read_early_stop_enabled, is_get_metadata_early_stop_bounded_fanout_enabled,
|
||||
is_get_metadata_early_stop_enabled, is_get_metadata_non_inline_data_read_early_stop_enabled, is_version_early_stop_enabled,
|
||||
object_fits_single_block,
|
||||
};
|
||||
use futures::future::join_all;
|
||||
use metrics::counter;
|
||||
@@ -125,6 +124,7 @@ fn metadata_metrics_path(bucket: &str) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn metadata_distribution_key(bucket: &str, object: &str) -> String {
|
||||
[bucket, object].join("/")
|
||||
}
|
||||
@@ -449,36 +449,16 @@ pub(in crate::set_disk) fn bounded_metadata_fanout_order(
|
||||
total_disks: usize,
|
||||
default_parity_count: usize,
|
||||
) -> Vec<usize> {
|
||||
let fallback_order = || (0..total_disks).collect::<Vec<_>>();
|
||||
if default_parity_count == 0 || default_parity_count >= total_disks {
|
||||
return fallback_order();
|
||||
return (0..total_disks).collect();
|
||||
}
|
||||
|
||||
let data_blocks = total_disks - default_parity_count;
|
||||
let distribution_key = metadata_distribution_key(bucket, object);
|
||||
let distribution = FileInfo::new(&distribution_key, data_blocks, default_parity_count)
|
||||
.erasure
|
||||
.distribution;
|
||||
if distribution.len() != total_disks {
|
||||
return fallback_order();
|
||||
}
|
||||
|
||||
let start = FileInfo::distribution_start(bucket, object, total_disks);
|
||||
let offset = (start + 1) % total_disks;
|
||||
let mut order = Vec::with_capacity(total_disks);
|
||||
for block_index in 1..=data_blocks {
|
||||
let Some(disk_index) = distribution
|
||||
.iter()
|
||||
.position(|distributed_block| *distributed_block == block_index)
|
||||
else {
|
||||
return fallback_order();
|
||||
};
|
||||
order.push(disk_index);
|
||||
}
|
||||
order.extend(
|
||||
distribution
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(disk_index, block_index)| (*block_index > data_blocks).then_some(disk_index)),
|
||||
);
|
||||
order.extend((0..data_blocks).map(|i| (total_disks + i - offset) % total_disks));
|
||||
order.extend((0..total_disks).filter(|&disk_index| (start + disk_index + 1) % total_disks >= data_blocks));
|
||||
order
|
||||
}
|
||||
|
||||
@@ -597,6 +577,9 @@ impl MetadataFanoutDiagnostics {
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn record(&self, path: &'static str) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
rustfs_io_metrics::record_get_object_metadata_fanout_duration(path, self.fanout_duration.as_secs_f64());
|
||||
if let Some(latency) = self.first_response_latency() {
|
||||
rustfs_io_metrics::record_get_object_first_metadata_response_latency(path, latency.as_secs_f64());
|
||||
@@ -620,6 +603,9 @@ impl MetadataFanoutDiagnostics {
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn record_quorum_candidate_latency(&self, path: &'static str, read_quorum: usize) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
if let Some(latency) = self.quorum_candidate_latency(read_quorum) {
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(path, "reached");
|
||||
rustfs_io_metrics::record_get_object_quorum_reached_latency(path, latency.as_secs_f64());
|
||||
@@ -629,15 +615,14 @@ impl MetadataFanoutDiagnostics {
|
||||
|
||||
#[cfg(test)]
|
||||
pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
candidate: &FileInfo,
|
||||
parts_metadata: &[FileInfo],
|
||||
disks: &[Option<DiskStore>],
|
||||
) -> Option<&'static str> {
|
||||
verify_inline_body_from_observations(
|
||||
bucket,
|
||||
object,
|
||||
_object,
|
||||
candidate,
|
||||
parts_metadata
|
||||
.iter()
|
||||
@@ -645,13 +630,11 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
|
||||
.map(|(index, metadata)| (index, Some(metadata))),
|
||||
disks,
|
||||
)
|
||||
.await
|
||||
.err()
|
||||
}
|
||||
|
||||
async fn verify_inline_body_from_observations<'a>(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
fn verify_inline_body_from_observations<'a>(
|
||||
_object: &str,
|
||||
candidate: &FileInfo,
|
||||
observations: impl IntoIterator<Item = (usize, Option<&'a FileInfo>)>,
|
||||
disks: &[Option<DiskStore>],
|
||||
@@ -660,21 +643,18 @@ async fn verify_inline_body_from_observations<'a>(
|
||||
return Err(reason);
|
||||
}
|
||||
|
||||
let Ok(erasure) = coding::Erasure::try_new_with_options(
|
||||
candidate.erasure.data_blocks,
|
||||
candidate.erasure.parity_blocks,
|
||||
candidate.erasure.block_size,
|
||||
candidate.uses_legacy_checksum,
|
||||
) else {
|
||||
let data_shards = candidate.erasure.data_blocks;
|
||||
if data_shards == 0
|
||||
|| candidate.erasure.block_size == 0
|
||||
|| (candidate.erasure.parity_blocks > 0
|
||||
&& candidate.uses_legacy_checksum
|
||||
&& !reed_solomon_simd::ReedSolomonEncoder::supports(data_shards, candidate.erasure.parity_blocks))
|
||||
{
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
|
||||
};
|
||||
let data_files =
|
||||
match collect_inline_data_shard_fileinfos_from_observations(observations, candidate, erasure.data_shards, |index| {
|
||||
disks.get(index).is_some_and(Option::is_some)
|
||||
}) {
|
||||
Ok(data_files) => data_files,
|
||||
Err(reason) => return Err(reason),
|
||||
};
|
||||
}
|
||||
let data_files = collect_inline_data_shard_fileinfos_from_observations(observations, candidate, data_shards, |index| {
|
||||
disks.get(index).is_some_and(Option::is_some)
|
||||
})?;
|
||||
|
||||
let Some(part) = candidate.parts.first() else {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
|
||||
@@ -693,22 +673,51 @@ async fn verify_inline_body_from_observations<'a>(
|
||||
object_size,
|
||||
object_size,
|
||||
candidate.erasure.block_size,
|
||||
erasure.data_shards,
|
||||
data_shards,
|
||||
candidate.uses_legacy_checksum,
|
||||
);
|
||||
let shard_size = inline_erasure_shard_size(candidate.erasure.block_size, erasure.data_shards, candidate.uses_legacy_checksum);
|
||||
let Ok(mut readers) =
|
||||
build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
|
||||
else {
|
||||
let shard_size = inline_erasure_shard_size(candidate.erasure.block_size, data_shards, candidate.uses_legacy_checksum);
|
||||
if object_size == 0 || read_length == 0 || read_length > shard_size || data_files.len() < data_shards {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||
}
|
||||
let shards_needed = object_size.div_ceil(read_length);
|
||||
if shards_needed > data_shards {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||
}
|
||||
let Some(encoded_capacity) = read_length.checked_mul(shards_needed) else {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
crate::set_disk::disk_call_counters::record(object, crate::set_disk::disk_call_counters::KIND_BODY_DECODE, 0);
|
||||
match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await {
|
||||
Some(body) if body.len() == object_size => Ok(body),
|
||||
_ => Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
|
||||
{
|
||||
for file in &data_files {
|
||||
if file.data.is_some() {
|
||||
crate::set_disk::disk_call_counters::record_shard_open(_object, true);
|
||||
}
|
||||
}
|
||||
crate::set_disk::disk_call_counters::record(_object, crate::set_disk::disk_call_counters::KIND_BODY_DECODE, 0);
|
||||
}
|
||||
|
||||
let hash_size = checksum_algo.size();
|
||||
let need = hash_size + read_length;
|
||||
let mut body = Vec::with_capacity(encoded_capacity);
|
||||
for file in data_files.iter().take(shards_needed) {
|
||||
let Some(raw) = file.data.as_deref() else {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||
};
|
||||
let Some(block) = raw.get(..need) else {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||
};
|
||||
let (hash, data) = block.split_at(hash_size);
|
||||
if hash_size > 0 && checksum_algo.hash_encode(data).as_ref() != hash {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||
}
|
||||
body.extend_from_slice(data);
|
||||
if body.len() >= object_size {
|
||||
body.truncate(object_size);
|
||||
return Ok(bytes::Bytes::from(body));
|
||||
}
|
||||
}
|
||||
Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY)
|
||||
}
|
||||
|
||||
fn data_read_early_stop_inline_candidate_miss_reason(candidate: &FileInfo) -> Option<&'static str> {
|
||||
@@ -786,16 +795,18 @@ fn data_read_inline_missing_shards_are_pending(
|
||||
fanout_order: &[usize],
|
||||
scheduled_fanout_len: usize,
|
||||
) -> bool {
|
||||
let Ok(erasure) = coding::Erasure::try_new_with_options(
|
||||
candidate.erasure.data_blocks,
|
||||
candidate.erasure.parity_blocks,
|
||||
candidate.erasure.block_size,
|
||||
candidate.uses_legacy_checksum,
|
||||
) else {
|
||||
let data_shards = candidate.erasure.data_blocks;
|
||||
if data_shards == 0
|
||||
|| candidate.erasure.block_size == 0
|
||||
|| !candidate.has_valid_erasure_geometry()
|
||||
|| (candidate.erasure.parity_blocks > 0
|
||||
&& candidate.uses_legacy_checksum
|
||||
&& !reed_solomon_simd::ReedSolomonEncoder::supports(data_shards, candidate.erasure.parity_blocks))
|
||||
{
|
||||
return false;
|
||||
};
|
||||
}
|
||||
let distribution = &candidate.erasure.distribution;
|
||||
let mut data_shards_seen_or_pending = vec![false; erasure.data_shards];
|
||||
let mut data_shards_seen_or_pending = vec![false; data_shards];
|
||||
let mut missing_pending_data_shards = 0usize;
|
||||
|
||||
for slot in slots {
|
||||
@@ -803,7 +814,7 @@ fn data_read_inline_missing_shards_are_pending(
|
||||
let Some(&block_index) = distribution.get(disk_index) else {
|
||||
return false;
|
||||
};
|
||||
if block_index == 0 || block_index > erasure.data_shards {
|
||||
if block_index == 0 || block_index > data_shards {
|
||||
continue;
|
||||
}
|
||||
if !disks.get(disk_index).is_some_and(Option::is_some) {
|
||||
@@ -1170,7 +1181,8 @@ impl SetDisks {
|
||||
default_parity_count: usize,
|
||||
allow_coalescing: bool,
|
||||
) -> disk::error::Result<MetadataReadResult> {
|
||||
let fanout_start = Instant::now();
|
||||
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
let fanout_start = stage_metrics_enabled.then(Instant::now);
|
||||
let mut slots = MetadataReadResult::pending(disks.len());
|
||||
let mut task_ids = vec![None; disks.len()];
|
||||
let mut observations = Vec::with_capacity(disks.len());
|
||||
@@ -1220,7 +1232,7 @@ impl SetDisks {
|
||||
let diagnostic_context = diagnostic_context;
|
||||
let task = join_set.spawn(async move {
|
||||
let read = async move {
|
||||
let response_start = Instant::now();
|
||||
let response_start = stage_metrics_enabled.then(Instant::now);
|
||||
let result = if let Some(disk) = disk {
|
||||
#[cfg(test)]
|
||||
let _fanout_task_guard = rename_fanout_barrier::task_guard(&object);
|
||||
@@ -1243,7 +1255,7 @@ impl SetDisks {
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
};
|
||||
(index, result, response_start.elapsed())
|
||||
(index, result, response_start.map_or(Duration::ZERO, |s| s.elapsed()))
|
||||
};
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
return crate::cluster::rpc::runtime_sources::with_read_version_diagnostic_context(
|
||||
@@ -1348,7 +1360,10 @@ impl SetDisks {
|
||||
},
|
||||
Err(join_error) => {
|
||||
let err = DiskError::Unexpected;
|
||||
observations.push(MetadataFanoutObservation::from_error(&err, fanout_start.elapsed()));
|
||||
observations.push(MetadataFanoutObservation::from_error(
|
||||
&err,
|
||||
fanout_start.map_or(Duration::ZERO, |s| s.elapsed()),
|
||||
));
|
||||
if let Some(index) = task_ids.iter().position(|id| *id == Some(join_error.id())) {
|
||||
slots[index].result = Some(Err(err));
|
||||
accumulator.observe(&slots[index]);
|
||||
@@ -1384,14 +1399,11 @@ impl SetDisks {
|
||||
accumulator.candidate_has_read_reserve()
|
||||
}
|
||||
Some(candidate) => match verify_inline_body_from_observations(
|
||||
bucket.as_ref(),
|
||||
object.as_ref(),
|
||||
candidate,
|
||||
slots.iter().map(|slot| (slot.disk_index, slot.file_info())),
|
||||
disks,
|
||||
)
|
||||
.await
|
||||
{
|
||||
) {
|
||||
Ok(body) => {
|
||||
verified_inline_body = Some(body);
|
||||
true
|
||||
@@ -1449,7 +1461,8 @@ impl SetDisks {
|
||||
scheduled_count.saturating_sub(cancelled_count),
|
||||
cancelled_count,
|
||||
);
|
||||
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
|
||||
let diagnostics =
|
||||
MetadataFanoutDiagnostics::new(fanout_start.map_or(Duration::ZERO, |s| s.elapsed()), observations);
|
||||
return Ok(MetadataReadResult {
|
||||
slots,
|
||||
diagnostics,
|
||||
@@ -1532,7 +1545,7 @@ impl SetDisks {
|
||||
rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, final_miss_reason);
|
||||
rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0);
|
||||
rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0);
|
||||
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
|
||||
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.map_or(Duration::ZERO, |s| s.elapsed()), observations);
|
||||
Ok(MetadataReadResult {
|
||||
slots,
|
||||
diagnostics,
|
||||
|
||||
@@ -275,6 +275,22 @@ impl SetDisks {
|
||||
|
||||
pub(super) fn common_parity(parities: &[i32], default_parity_count: i32) -> i32 {
|
||||
let n = parities.len() as i32;
|
||||
let mut valid_parities = parities.iter().copied().filter(|&p| p != -1);
|
||||
let Some(first_parity) = valid_parities.next() else {
|
||||
return -1;
|
||||
};
|
||||
let mut same_count = 1i32;
|
||||
if valid_parities.all(|p| {
|
||||
same_count += 1;
|
||||
p == first_parity
|
||||
}) {
|
||||
let read_quorum = if default_parity_count > 0 && first_parity == 0 {
|
||||
n / 2 + 1
|
||||
} else {
|
||||
n - first_parity
|
||||
};
|
||||
return if same_count >= read_quorum { first_parity } else { -1 };
|
||||
}
|
||||
|
||||
let mut occ_map: HashMap<i32, i32> = HashMap::new();
|
||||
for &p in parities {
|
||||
@@ -333,6 +349,19 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
pub(super) fn common_time_and_occurrence(times: &[Option<OffsetDateTime>]) -> (Option<OffsetDateTime>, usize) {
|
||||
let mut valid_times = times.iter().flatten().copied();
|
||||
let Some(first_time) = valid_times.next() else {
|
||||
return (None, 0);
|
||||
};
|
||||
let mut count = 1usize;
|
||||
if valid_times.all(|t| {
|
||||
count += 1;
|
||||
t == first_time
|
||||
}) && first_time.unix_timestamp_nanos() != 0
|
||||
{
|
||||
return (Some(first_time), count);
|
||||
}
|
||||
|
||||
let mut time_occurrence_map = HashMap::new();
|
||||
|
||||
// Ignore the uuid sentinel and count the rest.
|
||||
@@ -599,6 +628,25 @@ impl SetDisks {
|
||||
return write_quorum;
|
||||
}
|
||||
|
||||
if let Some((first, _)) = parts_metadata
|
||||
.iter()
|
||||
.zip(errs.iter())
|
||||
.find(|(meta, err)| err.is_none() && file_info_is_valid_for_metadata(meta))
|
||||
{
|
||||
let same_count = parts_metadata
|
||||
.iter()
|
||||
.zip(errs.iter())
|
||||
.filter(|(meta, err)| {
|
||||
err.is_none()
|
||||
&& file_info_is_valid_for_metadata(meta)
|
||||
&& super::core::metadata_quorum::metadata_early_stop_candidate_matches(first, meta)
|
||||
})
|
||||
.count();
|
||||
if same_count >= write_quorum {
|
||||
return write_quorum;
|
||||
}
|
||||
}
|
||||
|
||||
let mut identity_counts = HashMap::with_capacity(usable_metadata);
|
||||
for (meta, err) in parts_metadata.iter().zip(errs.iter()) {
|
||||
if err.is_some() || !file_info_is_valid_for_metadata(meta) {
|
||||
@@ -982,6 +1030,33 @@ impl SetDisks {
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
let matches_time_or_etag = |meta: &FileInfo| {
|
||||
mod_time == &meta.mod_time
|
||||
|| (mod_time.is_none() && etag.as_deref().is_some_and(|e| meta.get_etag().as_deref() == Some(e)))
|
||||
};
|
||||
|
||||
if let Some(first) = metas
|
||||
.iter()
|
||||
.find(|meta| file_info_is_valid_for_metadata(meta) && matches_time_or_etag(meta))
|
||||
{
|
||||
let mut total_valid = 0usize;
|
||||
let mut same_count = 0usize;
|
||||
for meta in metas {
|
||||
if !file_info_is_valid_for_metadata(meta) || !matches_time_or_etag(meta) {
|
||||
continue;
|
||||
}
|
||||
total_valid += 1;
|
||||
if super::core::metadata_quorum::metadata_early_stop_candidate_matches(first, meta) {
|
||||
same_count += 1;
|
||||
}
|
||||
}
|
||||
if same_count >= quorum && same_count > total_valid - same_count {
|
||||
let mut fi = first.clone();
|
||||
fi.is_latest = fi.successor_mod_time.is_none();
|
||||
return Ok(fi);
|
||||
}
|
||||
}
|
||||
|
||||
let mut meta_hashes = vec![None; metas.len()];
|
||||
|
||||
for (i, meta) in metas.iter().enumerate() {
|
||||
|
||||
+107
-148
@@ -129,7 +129,7 @@ use rustfs_filemeta::{
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::{
|
||||
DriveState, HealAdmissionResult, HealChannelPriority, HealItemType, HealOpts, HealRequestSource, HealScanMode,
|
||||
send_heal_replacement_disk, send_heal_request_with_admission,
|
||||
send_heal_request_with_admission,
|
||||
};
|
||||
use rustfs_io_metrics::{
|
||||
record_object_lock_diag_acquire_duration, record_object_lock_diag_enabled, record_object_lock_diag_hold_duration,
|
||||
@@ -160,7 +160,7 @@ use rustfs_utils::http::{
|
||||
use rustfs_utils::{
|
||||
HashAlgorithm,
|
||||
crypto::hex,
|
||||
path::{SLASH_SEPARATOR, encode_dir_object, encode_dir_object_ref, has_suffix, object_name_disk_identity, path_join_buf},
|
||||
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, object_name_disk_identity, path_join_buf},
|
||||
};
|
||||
use sha2::Sha256;
|
||||
use std::borrow::Cow;
|
||||
@@ -934,12 +934,20 @@ pub(in crate::set_disk) struct GetObjectFileInfo {
|
||||
}
|
||||
|
||||
struct OwnedGetObjectFileInfo {
|
||||
fi: FileInfo,
|
||||
parts_metadata: Vec<FileInfo>,
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
payload: OwnedGetObjectFileInfoPayload,
|
||||
verified_inline_body: Option<Bytes>,
|
||||
}
|
||||
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
enum OwnedGetObjectFileInfoPayload {
|
||||
Direct {
|
||||
fi: FileInfo,
|
||||
parts_metadata: Vec<FileInfo>,
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
},
|
||||
Published(Arc<GetObjectMetadataCacheEntry>),
|
||||
}
|
||||
|
||||
enum LateMetadataFanout {
|
||||
CapturedDisks(Vec<Option<DiskStore>>),
|
||||
RefreshCurrentSet,
|
||||
@@ -951,9 +959,11 @@ impl GetObjectFileInfo {
|
||||
fn owned(fi: FileInfo, parts_metadata: Vec<FileInfo>, online_disks: Vec<Option<DiskStore>>) -> Self {
|
||||
Self {
|
||||
owned: Some(OwnedGetObjectFileInfo {
|
||||
fi,
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
payload: OwnedGetObjectFileInfoPayload::Direct {
|
||||
fi,
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
},
|
||||
verified_inline_body: None,
|
||||
}),
|
||||
shared: None,
|
||||
@@ -962,15 +972,21 @@ impl GetObjectFileInfo {
|
||||
}
|
||||
}
|
||||
|
||||
fn owned_with_late_metadata_fanout(
|
||||
fi: FileInfo,
|
||||
parts_metadata: Vec<FileInfo>,
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
late_metadata_fanout_disks: Vec<Option<DiskStore>>,
|
||||
) -> Self {
|
||||
let mut snapshot = Self::owned(fi, parts_metadata, online_disks);
|
||||
snapshot.late_metadata_fanout = Some(LateMetadataFanout::CapturedDisks(late_metadata_fanout_disks));
|
||||
snapshot
|
||||
fn owned_published(entry: Arc<GetObjectMetadataCacheEntry>) -> Self {
|
||||
let condition_stopped = entry.capability == GetObjectMetadataCacheCapability::ConditionOnly;
|
||||
Self {
|
||||
owned: Some(OwnedGetObjectFileInfo {
|
||||
payload: OwnedGetObjectFileInfoPayload::Published(entry),
|
||||
verified_inline_body: None,
|
||||
}),
|
||||
shared: None,
|
||||
condition_stopped,
|
||||
late_metadata_fanout: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn set_late_metadata_fanout_disks(&mut self, late_metadata_fanout_disks: Vec<Option<DiskStore>>) {
|
||||
self.late_metadata_fanout = Some(LateMetadataFanout::CapturedDisks(late_metadata_fanout_disks));
|
||||
}
|
||||
|
||||
fn shared(entry: Arc<GetObjectMetadataCacheEntry>) -> Self {
|
||||
@@ -989,28 +1005,31 @@ impl GetObjectFileInfo {
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn fi(&self) -> &FileInfo {
|
||||
fn as_parts(&self) -> (&FileInfo, &[FileInfo], &[Option<DiskStore>]) {
|
||||
match (&self.owned, &self.shared) {
|
||||
(Some(snapshot), None) => &snapshot.fi,
|
||||
(None, Some(entry)) => &entry.fi,
|
||||
(Some(snapshot), None) => match &snapshot.payload {
|
||||
OwnedGetObjectFileInfoPayload::Direct {
|
||||
fi,
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
} => (fi, parts_metadata, online_disks),
|
||||
OwnedGetObjectFileInfoPayload::Published(entry) => (&entry.fi, &entry.parts_metadata, &entry.online_disks),
|
||||
},
|
||||
(None, Some(entry)) => (&entry.fi, &entry.parts_metadata, &entry.online_disks),
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
}
|
||||
}
|
||||
|
||||
fn fi(&self) -> &FileInfo {
|
||||
self.as_parts().0
|
||||
}
|
||||
|
||||
fn parts_metadata(&self) -> &[FileInfo] {
|
||||
match (&self.owned, &self.shared) {
|
||||
(Some(snapshot), None) => &snapshot.parts_metadata,
|
||||
(None, Some(entry)) => &entry.parts_metadata,
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
}
|
||||
self.as_parts().1
|
||||
}
|
||||
|
||||
fn online_disks(&self) -> &[Option<DiskStore>] {
|
||||
match (&self.owned, &self.shared) {
|
||||
(Some(snapshot), None) => &snapshot.online_disks,
|
||||
(None, Some(entry)) => &entry.online_disks,
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
}
|
||||
self.as_parts().2
|
||||
}
|
||||
|
||||
fn has_late_metadata_fanout(&self) -> bool {
|
||||
@@ -1024,17 +1043,28 @@ impl GetObjectFileInfo {
|
||||
}
|
||||
|
||||
fn take_verified_inline_body(&mut self) -> Option<Bytes> {
|
||||
self.owned.as_mut().and_then(|owned| owned.verified_inline_body.take())
|
||||
match (&mut self.owned, &self.shared) {
|
||||
(Some(owned), None) => owned.verified_inline_body.take().or_else(|| match &owned.payload {
|
||||
OwnedGetObjectFileInfoPayload::Direct { .. } => None,
|
||||
OwnedGetObjectFileInfoPayload::Published(entry) => entry.verified_inline_body.clone(),
|
||||
}),
|
||||
(None, Some(entry)) => entry.verified_inline_body.clone(),
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
}
|
||||
}
|
||||
|
||||
fn into_payload(self) -> (OwnedGetObjectFileInfoPayload, Option<LateMetadataFanout>) {
|
||||
match (self.owned, self.shared) {
|
||||
(Some(snapshot), None) => (snapshot.payload, self.late_metadata_fanout),
|
||||
(None, Some(entry)) => (OwnedGetObjectFileInfoPayload::Published(entry), self.late_metadata_fanout),
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
}
|
||||
}
|
||||
|
||||
fn into_object_info(self, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
|
||||
match (self.owned, self.shared) {
|
||||
(Some(snapshot), None) => ObjectInfo::from_owned_file_info(snapshot.fi, bucket, object, versioned),
|
||||
(None, Some(entry)) => match Arc::try_unwrap(entry) {
|
||||
Ok(entry) => ObjectInfo::from_owned_file_info(entry.fi, bucket, object, versioned),
|
||||
Err(entry) => ObjectInfo::from_file_info(&entry.fi, bucket, object, versioned),
|
||||
},
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
match self.into_payload().0 {
|
||||
OwnedGetObjectFileInfoPayload::Direct { fi, .. } => ObjectInfo::from_owned_file_info(fi, bucket, object, versioned),
|
||||
OwnedGetObjectFileInfoPayload::Published(entry) => ObjectInfo::from_file_info(&entry.fi, bucket, object, versioned),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1044,24 +1074,17 @@ impl GetObjectFileInfo {
|
||||
}
|
||||
|
||||
fn into_owned_with_late_metadata_fanout(self) -> OwnedGetObjectFileInfoParts {
|
||||
let Self {
|
||||
owned,
|
||||
shared,
|
||||
late_metadata_fanout,
|
||||
..
|
||||
} = self;
|
||||
match (owned, shared) {
|
||||
(Some(snapshot), None) => (snapshot.fi, snapshot.parts_metadata, snapshot.online_disks, late_metadata_fanout),
|
||||
(None, Some(entry)) => match Arc::try_unwrap(entry) {
|
||||
Ok(entry) => (entry.fi, entry.parts_metadata, entry.online_disks, late_metadata_fanout),
|
||||
Err(entry) => (
|
||||
entry.fi.clone(),
|
||||
entry.parts_metadata.clone(),
|
||||
entry.online_disks.clone(),
|
||||
late_metadata_fanout,
|
||||
),
|
||||
},
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
let (payload, late_metadata_fanout) = self.into_payload();
|
||||
match payload {
|
||||
OwnedGetObjectFileInfoPayload::Direct {
|
||||
fi,
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
} => (fi, parts_metadata, online_disks, late_metadata_fanout),
|
||||
OwnedGetObjectFileInfoPayload::Published(entry) => {
|
||||
let entry = Arc::unwrap_or_clone(entry);
|
||||
(entry.fi, entry.parts_metadata, entry.online_disks, late_metadata_fanout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1446,6 +1469,7 @@ mod prepared_get_object_metadata_tests {
|
||||
fi,
|
||||
online_disks: vec![None],
|
||||
read_quorum: 0,
|
||||
verified_inline_body: None,
|
||||
});
|
||||
let snapshot = GetObjectFileInfo::shared(Arc::clone(&cached));
|
||||
|
||||
@@ -2762,7 +2786,9 @@ impl SetDisks {
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PreparedGetObjectMetadata> {
|
||||
let snapshot = self.get_object_fileinfo(bucket, object, opts, true, false).await?;
|
||||
let snapshot = self
|
||||
.get_object_fileinfo_for_object_read(bucket, object, opts, true, false)
|
||||
.await?;
|
||||
let object_info = build_get_object_info(snapshot.fi(), bucket, object, opts.versioned || opts.version_suspended);
|
||||
Ok(PreparedGetObjectMetadata {
|
||||
snapshot,
|
||||
@@ -4901,7 +4927,7 @@ enum GetObjectMetadataCacheCapability {
|
||||
ConditionOnly,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[derive(Clone, Debug)]
|
||||
struct GetObjectMetadataCacheEntry {
|
||||
#[allow(dead_code)] // Kept for debugging; moka handles TTL internally
|
||||
created_at: Instant,
|
||||
@@ -4910,6 +4936,7 @@ struct GetObjectMetadataCacheEntry {
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
read_quorum: usize,
|
||||
capability: GetObjectMetadataCacheCapability,
|
||||
verified_inline_body: Option<Bytes>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -5220,40 +5247,6 @@ impl SetDisks {
|
||||
.with_attempt(attempt))
|
||||
}
|
||||
|
||||
pub(crate) async fn read_listing_metadata_after_namespace_barrier(
|
||||
&self,
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<(MetaCacheEntries, usize)> {
|
||||
// This is only called after the lock-free LIST fast path observes an
|
||||
// unresolved metadata generation. Waiting on the ordinary object read
|
||||
// lock establishes a publication boundary with concurrent overwrites.
|
||||
let stored_object = encode_dir_object_ref(object);
|
||||
let _guard = self
|
||||
.acquire_read_lock_diag("list_object_reconcile", bucket, stored_object.as_ref())
|
||||
.await?;
|
||||
let (raw_entries, errors) = Self::read_all_raw_file_info(disks, bucket, stored_object.as_ref(), false).await;
|
||||
let confirmed_absent = errors
|
||||
.iter()
|
||||
.flatten()
|
||||
.filter(|err| DiskError::is_err_object_not_found(err) || DiskError::is_err_version_not_found(err))
|
||||
.count();
|
||||
let entries = raw_entries
|
||||
.into_iter()
|
||||
.map(|raw| {
|
||||
raw.filter(|raw| !raw.buf.is_empty()).map(|raw| MetaCacheEntry {
|
||||
name: object.to_owned(),
|
||||
metadata: raw.buf,
|
||||
cached: None,
|
||||
reusable: false,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok((MetaCacheEntries(entries), confirmed_absent))
|
||||
}
|
||||
|
||||
async fn acquire_write_lock_diag(&self, op: &'static str, bucket: &str, object: &str) -> Result<ObjectLockDiagGuard> {
|
||||
crate::hp_guard!("SetDisks::acquire_write_lock");
|
||||
let diag_enabled = is_object_lock_diag_enabled();
|
||||
@@ -8814,17 +8807,31 @@ mod tests {
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
runtime.block_on(async {
|
||||
let healthy: Arc<dyn LockClient> =
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let failing: Arc<dyn LockClient> = Arc::new(FailingClient);
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
let set = make_test_set_disks_with_ctx(vec![healthy, failing], ctx).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
// A real acquire can time out before the failing client is scheduled.
|
||||
let err = resolve_put_object_commit_lock_acquire_result(
|
||||
&set,
|
||||
"put_object_commit",
|
||||
"bucket",
|
||||
"object",
|
||||
Err(LockError::QuorumNotReached {
|
||||
required: 2,
|
||||
achieved: 1,
|
||||
}),
|
||||
)
|
||||
.expect_err("a quorum failure must remain a lock error");
|
||||
assert!(
|
||||
set.acquire_write_lock_diag("put_object_commit", "bucket", "object")
|
||||
.await
|
||||
.is_err(),
|
||||
"one healthy locker must not satisfy the PUT commit write quorum"
|
||||
matches!(
|
||||
err,
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
required: 2,
|
||||
achieved: 1,
|
||||
..
|
||||
}
|
||||
),
|
||||
"expected the typed quorum error, got: {err:?}"
|
||||
);
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
@@ -16436,54 +16443,6 @@ mod tests {
|
||||
assert_eq!(err, StorageError::PreconditionFailed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn listing_metadata_reconcile_encodes_directory_object_and_waits_for_writer() {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
let bucket = "bucket-list-reconcile-lock";
|
||||
let object = "directory/";
|
||||
let stored_object = encode_dir_object(object);
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(b"directory marker".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, &stored_object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("directory object should be written under its encoded key");
|
||||
let disks = set_disks.disks.read().await.clone();
|
||||
let namespace_lock = set_disks
|
||||
.new_ns_lock(bucket, &stored_object)
|
||||
.await
|
||||
.expect("namespace lock should be created");
|
||||
let writer_guard = namespace_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("writer lock should be acquired");
|
||||
let read = set_disks.read_listing_metadata_after_namespace_barrier(&disks, bucket, object);
|
||||
tokio::pin!(read);
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(50), &mut read)
|
||||
.await
|
||||
.is_err(),
|
||||
"LIST reconciliation must wait for the publishing writer"
|
||||
);
|
||||
drop(writer_guard);
|
||||
|
||||
let (entries, confirmed_absent) = tokio::time::timeout(std::time::Duration::from_secs(30), read)
|
||||
.await
|
||||
.expect("LIST reconciliation should resume after writer release")
|
||||
.expect("LIST reconciliation should read after the namespace barrier");
|
||||
assert_eq!(entries.0.len(), disks.len());
|
||||
assert!(
|
||||
entries.0.iter().all(|entry| entry
|
||||
.as_ref()
|
||||
.is_some_and(|entry| entry.name == object && !entry.metadata.is_empty())),
|
||||
"LIST reconciliation must read encoded metadata and retain the logical key"
|
||||
);
|
||||
assert_eq!(confirmed_absent, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repeated_body_write_keeps_etag_but_changes_data_dir_generation() {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
|
||||
@@ -21,17 +21,22 @@
|
||||
|
||||
use super::super::{
|
||||
Arc, DiskError, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, Endpoint, Error, FormatV3, HealChannelPriority, LockResult,
|
||||
NamespaceLock, NamespaceLockWrapper, ObjectKey, Result, SetDisks, StorageError, debug, disk, info, load_format_erasure,
|
||||
send_heal_replacement_disk, warn,
|
||||
NamespaceLock, NamespaceLockWrapper, ObjectKey, Result, SetDisks, StorageError, debug, disk, load_format_erasure, warn,
|
||||
};
|
||||
use crate::disk::health_state::DriveMembershipSnapshot;
|
||||
use crate::disk::{DiskAPI, new_disk};
|
||||
use crate::layout::disks_layout::MAX_ERASURE_SET_DRIVE_COUNT;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use futures::{StreamExt, stream, stream::FuturesUnordered};
|
||||
use rand::prelude::SliceRandom;
|
||||
use rustfs_heal_contracts::heal_channel::{create_auto_replacement_disks_request, send_heal_request};
|
||||
use rustfs_utils::path::object_name_disk_identity;
|
||||
#[cfg(test)]
|
||||
use uuid::Uuid;
|
||||
|
||||
// Bound batching when another reconnect is still pending.
|
||||
const REPLACEMENT_BATCH_WAIT: std::time::Duration = std::time::Duration::from_secs(1);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
|
||||
type Error = Error;
|
||||
@@ -326,38 +331,83 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
async fn connect_disks_with_active_monitoring(&self, active_monitoring: bool) {
|
||||
let rl = self.disks.read().await;
|
||||
let disks = self.get_disks_internal().await;
|
||||
// Reconnect drives concurrently without holding the inventory lock.
|
||||
let mut reconnects = stream::iter(disks.into_iter().enumerate())
|
||||
.map(|(i, opdisk)| async move {
|
||||
if let Some(disk) = opdisk {
|
||||
if !active_monitoring
|
||||
&& disk.get_disk_location().set_idx.is_some()
|
||||
&& let disk::Disk::Local(local) = disk.as_ref()
|
||||
&& local.can_skip_background_probe().await
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if disk.is_online().await && disk.get_disk_location().set_idx.is_some() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let disks = rl.clone();
|
||||
|
||||
// Explicitly release the lock
|
||||
drop(rl);
|
||||
|
||||
for (i, opdisk) in disks.iter().enumerate() {
|
||||
if let Some(disk) = opdisk {
|
||||
if !active_monitoring
|
||||
&& disk.get_disk_location().set_idx.is_some()
|
||||
&& let disk::Disk::Local(local) = disk.as_ref()
|
||||
&& local.can_skip_background_probe().await
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if disk.is_online().await && disk.get_disk_location().set_idx.is_some() {
|
||||
info!("Disk {:?} is online", disk.to_string());
|
||||
continue;
|
||||
let _ = disk.close().await;
|
||||
}
|
||||
|
||||
let _ = disk.close().await;
|
||||
let endpoint = self.set_endpoints.get(i)?;
|
||||
self.reconnect_disk(endpoint).await.then(|| endpoint.to_string())
|
||||
})
|
||||
.buffer_unordered(MAX_ERASURE_SET_DRIVE_COUNT);
|
||||
let mut submissions = FuturesUnordered::new();
|
||||
let mut batch = Vec::new();
|
||||
let mut deadline = None;
|
||||
let mut reconnects_done = false;
|
||||
loop {
|
||||
if reconnects_done && submissions.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
if let Some(endpoint) = self.set_endpoints.get(i) {
|
||||
info!("will renew disk, opdisk: {:?}", opdisk);
|
||||
self.renew_disk(endpoint).await;
|
||||
tokio::select! {
|
||||
result = reconnects.next(), if !reconnects_done => match result {
|
||||
Some(Some(endpoint)) => {
|
||||
batch.push(endpoint);
|
||||
deadline.get_or_insert_with(|| tokio::time::Instant::now() + REPLACEMENT_BATCH_WAIT);
|
||||
}
|
||||
Some(None) => {}
|
||||
None => {
|
||||
reconnects_done = true;
|
||||
if !batch.is_empty() {
|
||||
submissions.push(self.submit_replacement_heal(std::mem::take(&mut batch)));
|
||||
}
|
||||
deadline = None;
|
||||
}
|
||||
},
|
||||
_ = tokio::time::sleep_until(deadline.unwrap_or_else(tokio::time::Instant::now)), if deadline.is_some() => {
|
||||
submissions.push(self.submit_replacement_heal(std::mem::take(&mut batch)));
|
||||
deadline = None;
|
||||
}
|
||||
_ = submissions.next(), if !submissions.is_empty() => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn submit_replacement_heal(&self, replacements: Vec<String>) {
|
||||
if replacements.is_empty() {
|
||||
return;
|
||||
}
|
||||
// Queue admission deduplicates by set, so publish ready targets together.
|
||||
let request = create_auto_replacement_disks_request(
|
||||
self.pool_index,
|
||||
self.set_index,
|
||||
replacements,
|
||||
Some(HealChannelPriority::Normal),
|
||||
);
|
||||
let _ = send_heal_request(request).await;
|
||||
}
|
||||
|
||||
pub async fn renew_disk(&self, ep: &Endpoint) {
|
||||
if self.reconnect_disk(ep).await {
|
||||
self.submit_replacement_heal(vec![ep.to_string()]).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Return whether this reconnect attached a verified unformatted target.
|
||||
async fn reconnect_disk(&self, ep: &Endpoint) -> bool {
|
||||
debug!("renew_disk: start {:?}", ep);
|
||||
|
||||
let previous_disk = {
|
||||
@@ -377,28 +427,17 @@ impl SetDisks {
|
||||
Err(e) => {
|
||||
warn!("renew_disk: connect_endpoint err {:?}", &e);
|
||||
if !matches!(e, DiskError::UnformattedDisk | DiskError::Io(_)) {
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
let attached = match self.attach_unformatted_replacement_disk(ep).await {
|
||||
Ok(attached) => attached,
|
||||
Err(err) => {
|
||||
warn!(endpoint = %ep, error = ?err, "renew_disk: unformatted replacement probe failed");
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
if !attached {
|
||||
return;
|
||||
}
|
||||
|
||||
info!("renew_disk attached unformatted replacement and will trigger heal_disk, {:?}", ep);
|
||||
let (Ok(pool_index), Ok(set_index)) = (usize::try_from(ep.pool_idx), usize::try_from(ep.set_idx)) else {
|
||||
warn!("renew_disk: replacement target has invalid pool or set index, {:?}", ep);
|
||||
return;
|
||||
};
|
||||
let _ =
|
||||
send_heal_replacement_disk(pool_index, set_index, ep.to_string(), Some(HealChannelPriority::Normal)).await;
|
||||
return;
|
||||
return attached;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -406,7 +445,7 @@ impl SetDisks {
|
||||
Ok(res) => res,
|
||||
Err(e) => {
|
||||
warn!("renew_disk: find_disk_index err {:?}", e);
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -424,7 +463,7 @@ impl SetDisks {
|
||||
expected_set_index = self.set_index,
|
||||
"renew_disk rejected a drive whose endpoint and format do not identify the same topology slot"
|
||||
);
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
|
||||
@@ -444,7 +483,7 @@ impl SetDisks {
|
||||
new_disk.enable_health_check();
|
||||
*slot = Some(new_disk);
|
||||
}
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
new_disk.enable_health_check();
|
||||
@@ -466,6 +505,7 @@ impl SetDisks {
|
||||
|
||||
let mut disk_lock = self.disks.write().await;
|
||||
disk_lock[disk_idx] = Some(new_disk);
|
||||
false
|
||||
}
|
||||
|
||||
/// Attach a replacement target only after proving that the exact local slot
|
||||
@@ -727,10 +767,6 @@ mod tests {
|
||||
let format = FormatV3::new(1, 1);
|
||||
let (_dir, endpoint, disk) = make_formatted_local_disk(0, &format).await;
|
||||
assert!(disk.is_online().await, "initialize the expected disk identity");
|
||||
let disk::Disk::Local(wrapper) = disk.as_ref() else {
|
||||
panic!("fixture must use a local disk");
|
||||
};
|
||||
let local = wrapper.get_disk();
|
||||
let set = SetDisks::new(
|
||||
"quiet-monitor".to_owned(),
|
||||
Arc::new(RwLock::new(vec![Some(disk.clone())])),
|
||||
@@ -749,13 +785,136 @@ mod tests {
|
||||
(false, RuntimeDriveHealthState::Offline),
|
||||
(false, RuntimeDriveHealthState::Returning),
|
||||
] {
|
||||
let disk = set.get_disks_internal().await[0].clone().expect("current drive");
|
||||
let disk::Disk::Local(wrapper) = disk.as_ref() else {
|
||||
panic!("fixture must use a local disk");
|
||||
};
|
||||
let local = wrapper.get_disk();
|
||||
disk.force_runtime_state_for_test(state);
|
||||
local.format_info.write().await.last_check = None;
|
||||
set.connect_disks_with_active_monitoring(active).await;
|
||||
let current = set.get_disks_internal().await[0].clone().expect("probed or renewed drive");
|
||||
let disk::Disk::Local(wrapper) = current.as_ref() else {
|
||||
panic!("fixture must use a local disk");
|
||||
};
|
||||
assert!(
|
||||
local.format_info.read().await.last_check.is_some(),
|
||||
wrapper.get_disk().format_info.read().await.last_check.is_some(),
|
||||
"probe required for active={active}, state={state:?}"
|
||||
);
|
||||
current
|
||||
.disk_info(&DiskInfoOptions::default())
|
||||
.await
|
||||
.expect("the probed or renewed drive must admit real I/O");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_reconnect_restores_ec4_io_without_all_disks_returning() {
|
||||
use crate::object_api::{ObjectOptions, PutObjReader};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::object::ObjectIO;
|
||||
use disk::health_state::RuntimeDriveHealthState;
|
||||
use http::HeaderMap;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
for active_monitoring in [true, false] {
|
||||
let format = FormatV3::new(1, 16);
|
||||
let mut dirs = Vec::new();
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for index in 0..16 {
|
||||
let (dir, endpoint, disk) = make_formatted_local_disk(index, &format).await;
|
||||
assert!(disk.is_online().await, "initialize the original drive identity");
|
||||
dirs.push(dir);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
let set = SetDisks::new(
|
||||
"partial-reconnect".to_owned(),
|
||||
Arc::new(RwLock::new(disks.clone())),
|
||||
16,
|
||||
4,
|
||||
0,
|
||||
0,
|
||||
endpoints,
|
||||
format,
|
||||
Vec::new(),
|
||||
)
|
||||
.await;
|
||||
let bucket = "partial-reconnect";
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket before the outage");
|
||||
let payload = vec![b'r'; 1_300_000];
|
||||
set.put_object(bucket, "before", &mut PutObjReader::from_vec(payload.clone()), &opts)
|
||||
.await
|
||||
.expect("write the original object");
|
||||
|
||||
let mut saved_formats = Vec::new();
|
||||
for (dir, disk) in dirs.iter().zip(&disks).take(8) {
|
||||
let disk = disk.as_ref().expect("original disk");
|
||||
disk.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
|
||||
let path = dir.path().join(disk::RUSTFS_META_BUCKET).join(disk::FORMAT_CONFIG_FILE);
|
||||
saved_formats.push(tokio::fs::read(&path).await.expect("save original format"));
|
||||
// A corrupt format makes reconnect fail without admitting a blank replacement.
|
||||
tokio::fs::write(&path, b"invalid-format")
|
||||
.await
|
||||
.expect("make the original drive unavailable");
|
||||
let disk::Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("fixture must use local drives");
|
||||
};
|
||||
local.get_disk().format_info.write().await.last_check = None;
|
||||
}
|
||||
set.connect_disks_with_active_monitoring(active_monitoring).await;
|
||||
let unavailable = set.get_disks_internal().await;
|
||||
for (original, retained) in disks.iter().zip(&unavailable).take(8) {
|
||||
assert!(
|
||||
Arc::ptr_eq(original.as_ref().expect("original"), retained.as_ref().expect("retained")),
|
||||
"failed reconnect must leave the old handle to exercise subsequent recovery"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
set.put_object(bucket, "during", &mut PutObjReader::from_vec(payload.clone()), &opts)
|
||||
.await
|
||||
.is_err(),
|
||||
"eight available drives must not satisfy a 16-drive EC:4 write"
|
||||
);
|
||||
|
||||
for (dir, saved) in dirs.iter().zip(&saved_formats).take(4) {
|
||||
let path = dir.path().join(disk::RUSTFS_META_BUCKET).join(disk::FORMAT_CONFIG_FILE);
|
||||
tokio::fs::write(path, saved)
|
||||
.await
|
||||
.expect("restore four original drive identities");
|
||||
}
|
||||
set.connect_disks_with_active_monitoring(active_monitoring).await;
|
||||
assert_eq!(
|
||||
set.drive_membership_snapshot().await.online.len(),
|
||||
12,
|
||||
"the four restored drives must be usable while four others remain unavailable"
|
||||
);
|
||||
set.put_object(bucket, "after", &mut PutObjReader::from_vec(payload.clone()), &opts)
|
||||
.await
|
||||
.expect("write at the restored EC:4 quorum");
|
||||
for object in ["before", "after"] {
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("read at the restored EC:4 quorum");
|
||||
let mut actual = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("read every reconstructed byte");
|
||||
assert_eq!(actual, payload, "the complete object must survive partial recovery");
|
||||
}
|
||||
for disk in set.get_disks_internal().await.into_iter().flatten() {
|
||||
disk.close().await.expect("stop the recovered drive monitors");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -985,6 +1144,162 @@ mod tests {
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconnect_sweep_collects_all_blank_targets_and_excludes_original_disks() {
|
||||
use rustfs_heal_contracts::heal_channel::{HealAdmissionResult, HealChannelCommand, init_heal_channel};
|
||||
|
||||
const CHILD: &str = "RUSTFS_TEST_GROUPED_RECONNECT_CHILD";
|
||||
if std::env::var_os(CHILD).is_none() {
|
||||
// The heal sender is process-global; keep this receiver isolated.
|
||||
let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"set_disk::ops::locking::tests::reconnect_sweep_collects_all_blank_targets_and_excludes_original_disks",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD, "1")
|
||||
.output()
|
||||
.expect("isolated reconnect test should run");
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
assert!(output.status.success(), "isolated reconnect test failed: {stdout} {stderr}");
|
||||
assert!(stdout.contains("1 passed"), "child must execute the reconnect test: {stdout}");
|
||||
return;
|
||||
}
|
||||
let mut heal_rx = init_heal_channel().expect("isolated heal receiver");
|
||||
let disk_count = 4;
|
||||
let format = FormatV3::new(1, disk_count);
|
||||
let mut directories = Vec::new();
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for disk_idx in 0..2 {
|
||||
let (directory, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
|
||||
directories.push(directory);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
for disk_idx in 2..disk_count {
|
||||
let directory = tempfile::tempdir().expect("replacement directory");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(directory.path().to_str().expect("UTF-8 replacement path")).expect("replacement endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_idx);
|
||||
directories.push(directory);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(None);
|
||||
}
|
||||
let mut ready_targets = endpoints[2..].iter().map(ToString::to_string).collect::<Vec<_>>();
|
||||
ready_targets.sort_unstable();
|
||||
// Disk 1 returns with its format and data; disks 2 and 3 are blank.
|
||||
disks[1]
|
||||
.take()
|
||||
.expect("original disk handle")
|
||||
.close()
|
||||
.await
|
||||
.expect("close original disk");
|
||||
let retained_data = directories[1].path().join("retained-data");
|
||||
tokio::fs::write(&retained_data, b"original data")
|
||||
.await
|
||||
.expect("write original data");
|
||||
let set = Arc::new(
|
||||
SetDisks::new(
|
||||
"reconnect-sweep".to_string(),
|
||||
Arc::new(RwLock::new(disks)),
|
||||
disk_count,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
endpoints.clone(),
|
||||
format,
|
||||
Vec::new(),
|
||||
)
|
||||
.await,
|
||||
);
|
||||
|
||||
// Block disk 0's format check, then make it a late blank replacement.
|
||||
let slow = set.get_disks_internal().await[0].clone().expect("original disk");
|
||||
assert!(slow.is_online().await, "initialize original disk identity");
|
||||
let disk::Disk::Local(wrapper) = slow.as_ref() else {
|
||||
panic!("fixture must use a local disk");
|
||||
};
|
||||
let slow_local = wrapper.get_disk();
|
||||
let mut slow_format_guard = slow_local.format_info.write().await;
|
||||
slow_format_guard.last_check = None;
|
||||
std::fs::remove_file(
|
||||
directories[0]
|
||||
.path()
|
||||
.join(disk::RUSTFS_META_BUCKET)
|
||||
.join(disk::FORMAT_CONFIG_FILE),
|
||||
)
|
||||
.expect("make the blocked disk a blank replacement");
|
||||
let sweep = tokio::spawn({
|
||||
let set = set.clone();
|
||||
async move { set.connect_disks_with_active_monitoring(true).await }
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
loop {
|
||||
let attached = set.get_disks_internal().await;
|
||||
if attached[1..].iter().all(Option::is_some) {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("other drives must reconnect while the first drive is blocked");
|
||||
let command = tokio::time::timeout(std::time::Duration::from_secs(10), heal_rx.recv())
|
||||
.await
|
||||
.expect("ready targets must be submitted while a drive is blocked")
|
||||
.expect("heal receiver should remain open");
|
||||
let HealChannelCommand::Start { request, response_tx } = command else {
|
||||
panic!("expected a replacement heal request");
|
||||
};
|
||||
assert_eq!(request.heal_endpoints, ready_targets, "ready blank disks must share one request");
|
||||
assert!(!sweep.is_finished(), "the blocked drive must still be pending");
|
||||
drop(slow_format_guard);
|
||||
// Do not acknowledge the first request yet: a delayed admission
|
||||
// response must not stop the late drive reconnecting and submitting.
|
||||
let late_command = tokio::time::timeout(std::time::Duration::from_secs(10), heal_rx.recv())
|
||||
.await
|
||||
.expect("late target must be submitted while the first response is pending")
|
||||
.expect("heal receiver should remain open");
|
||||
let HealChannelCommand::Start {
|
||||
request: late_request,
|
||||
response_tx: late_response,
|
||||
} = late_command
|
||||
else {
|
||||
panic!("expected a late replacement heal request");
|
||||
};
|
||||
assert_eq!(late_request.heal_endpoints, [endpoints[0].to_string()]);
|
||||
response_tx
|
||||
.send(Ok(HealAdmissionResult::Accepted))
|
||||
.expect("acknowledge ready targets");
|
||||
late_response
|
||||
.send(Ok(HealAdmissionResult::Accepted))
|
||||
.expect("acknowledge late target");
|
||||
sweep.await.expect("reconnect sweep should finish");
|
||||
assert!(heal_rx.try_recv().is_err(), "no duplicate request should be sent");
|
||||
let attached = set.get_disks_internal().await;
|
||||
for (index, disk) in attached.iter().enumerate() {
|
||||
let disk = disk.as_ref().expect("each verified endpoint should be attached");
|
||||
let format = load_format_erasure(disk, false).await;
|
||||
if index == 1 {
|
||||
assert!(format.is_ok(), "original disk must retain its format");
|
||||
} else {
|
||||
assert_eq!(format.expect_err("replacement must remain unformatted"), DiskError::UnformattedDisk);
|
||||
assert!(!disk.health_check_enabled_for_test(), "format must precede replacement health checks");
|
||||
}
|
||||
disk.close().await.expect("close test disk");
|
||||
}
|
||||
assert_eq!(tokio::fs::read(&retained_data).await.expect("read original data"), b"original data");
|
||||
let local_map = runtime_sources::local_disk_map_handle();
|
||||
let mut local_map = local_map.write().await;
|
||||
for endpoint in endpoints {
|
||||
local_map.remove(&endpoint.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renew_disk_attaches_only_a_verified_local_unformatted_replacement() {
|
||||
let disk_count = 4;
|
||||
|
||||
@@ -5617,7 +5617,10 @@ mod tests {
|
||||
.await
|
||||
.expect("abort task should join after completion releases")
|
||||
.expect_err("the committed upload should no longer exist");
|
||||
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
|
||||
assert!(
|
||||
matches!(abort_err, StorageError::InvalidUploadID(..)),
|
||||
"the completed upload must be absent after the tail drains, got: {abort_err:?}"
|
||||
);
|
||||
|
||||
for (disk_index, disk) in disk_stores.iter().enumerate() {
|
||||
let fi = disk
|
||||
@@ -9553,10 +9556,12 @@ mod tests {
|
||||
).expect("actual completion identity"));
|
||||
}
|
||||
}
|
||||
if published == 3 && tasks.running() == 1 { break; }
|
||||
// The fourth native syscall is held by release_rx even if
|
||||
// its deadline has already retired the async wrapper.
|
||||
if published == 3 { break; }
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}).await.expect("three real native publication legs must return before the fourth deadline");
|
||||
}).await.expect("three real native publication legs must succeed while the fourth syscall remains paused");
|
||||
after_publication.as_ref().expect("publication barrier").wait_until_paused().await;
|
||||
// Quorum has been validated and the parent released its object
|
||||
// guard. The successful physical tail cannot retain a clone.
|
||||
|
||||
@@ -2999,6 +2999,44 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
let object_size = usize::try_from(fi.size)
|
||||
.map_err(|_| to_object_err(Error::other("inline fast path object size is invalid"), vec![bucket, object]))?;
|
||||
|
||||
if can_try_inline_data_shards_direct(object_size, fi.erasure.block_size)
|
||||
&& let Some(body) = verified_inline_body.filter(|body| body.len() == object_size)
|
||||
{
|
||||
if let Some(inline_prepare_stage_start) = inline_prepare_stage_start.take() {
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
GET_OBJECT_PATH_INLINE_DIRECT,
|
||||
GET_STAGE_INLINE_PREPARE,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
inline_prepare_stage_start.elapsed().as_secs_f64(),
|
||||
);
|
||||
}
|
||||
if rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
GET_OBJECT_PATH_INLINE_DIRECT,
|
||||
GET_STAGE_READER_SETUP,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
0.0,
|
||||
);
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
GET_OBJECT_PATH_INLINE_DIRECT,
|
||||
GET_STAGE_DECODE,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
0.0,
|
||||
);
|
||||
}
|
||||
record_get_object_reader_path_observation(GET_OBJECT_PATH_INLINE_DIRECT, object_class, size_bucket);
|
||||
let reader = GetObjectReader {
|
||||
stream: Box::new(Cursor::new(body.clone())),
|
||||
object_info,
|
||||
buffered_body: Some(body),
|
||||
body_source: GetObjectBodySource::Unprobed,
|
||||
};
|
||||
return Ok(reader);
|
||||
}
|
||||
|
||||
let checksum_info = fi.erasure.get_checksum_info(fi.parts[0].number);
|
||||
let checksum_algo =
|
||||
if fi.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S {
|
||||
@@ -3012,42 +3050,6 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
disks.get(index).is_some_and(Option::is_some)
|
||||
})
|
||||
{
|
||||
if let Some(body) = verified_inline_body.filter(|body| body.len() == object_size) {
|
||||
if let Some(inline_prepare_stage_start) = inline_prepare_stage_start.take() {
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
GET_OBJECT_PATH_INLINE_DIRECT,
|
||||
GET_STAGE_INLINE_PREPARE,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
inline_prepare_stage_start.elapsed().as_secs_f64(),
|
||||
);
|
||||
}
|
||||
if rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
GET_OBJECT_PATH_INLINE_DIRECT,
|
||||
GET_STAGE_READER_SETUP,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
0.0,
|
||||
);
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
GET_OBJECT_PATH_INLINE_DIRECT,
|
||||
GET_STAGE_DECODE,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
0.0,
|
||||
);
|
||||
}
|
||||
record_get_object_reader_path_observation(GET_OBJECT_PATH_INLINE_DIRECT, object_class, size_bucket);
|
||||
let reader = GetObjectReader {
|
||||
stream: Box::new(Cursor::new(body.clone())),
|
||||
object_info,
|
||||
buffered_body: Some(body),
|
||||
body_source: GetObjectBodySource::Unprobed,
|
||||
};
|
||||
return Ok(reader);
|
||||
}
|
||||
|
||||
let read_length = inline_erasure_shard_file_offset(
|
||||
0,
|
||||
object_size,
|
||||
@@ -11087,7 +11089,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
// shared lock and permit metadata caching can early-stop once write
|
||||
// quorum agrees on the object identity.
|
||||
let allow_early_stop = !opts.no_lock || opts.metadata_cache_safe;
|
||||
let read = self.get_object_fileinfo(bucket, object, opts, true, allow_early_stop);
|
||||
let read = self.get_object_fileinfo_for_object_read(bucket, object, opts, true, allow_early_stop);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ObjectInfoRead,
|
||||
@@ -16904,6 +16906,7 @@ mod transition_commit_failure_tests {
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
read_quorum: 2,
|
||||
verified_inline_body: None,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
@@ -22842,6 +22845,116 @@ mod put_object_tmp_cleanup_tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Per-drive view of an object: every version (including free versions)
|
||||
/// recorded in its xl.meta, and the data directories next to it. A missing
|
||||
/// xl.meta and one without versions both hold no version.
|
||||
async fn per_drive_object_state(
|
||||
dirs: &[TempDir],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Vec<(Vec<(Option<Uuid>, Option<Uuid>, Option<OffsetDateTime>)>, Vec<String>)> {
|
||||
let mut drives = Vec::with_capacity(dirs.len());
|
||||
for dir in dirs {
|
||||
let object_dir = dir.path().join(bucket).join(object);
|
||||
let versions = match tokio::fs::read(object_dir.join(STORAGE_FORMAT_FILE)).await {
|
||||
Ok(encoded) => {
|
||||
let all = FileMeta::load(&encoded)
|
||||
.expect("xl.meta should decode")
|
||||
.get_all_file_info_versions(bucket, object, true)
|
||||
.expect("xl.meta versions should be readable");
|
||||
all.versions
|
||||
.iter()
|
||||
.chain(all.free_versions.iter())
|
||||
.map(|version| (version.version_id, version.data_dir, version.mod_time))
|
||||
.collect()
|
||||
}
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Vec::new(),
|
||||
Err(err) => panic!("xl.meta at {object_dir:?} should be readable: {err}"),
|
||||
};
|
||||
let mut data_dirs = Vec::new();
|
||||
if let Ok(mut entries) = tokio::fs::read_dir(&object_dir).await {
|
||||
while let Some(entry) = entries.next_entry().await.expect("object directory entry should be readable") {
|
||||
if entry
|
||||
.file_type()
|
||||
.await
|
||||
.expect("object entry type should be readable")
|
||||
.is_dir()
|
||||
{
|
||||
data_dirs.push(entry.file_name().to_string_lossy().into_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
data_dirs.sort();
|
||||
drives.push((versions, data_dirs));
|
||||
}
|
||||
drives
|
||||
}
|
||||
|
||||
/// Stage F lists any version present on half of the drives, so a PUT that
|
||||
/// misses write quorum must not leave its version behind once its rollback
|
||||
/// completes: no drive may keep the new xl.meta entry or its data directory.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn failed_put_rollback_leaves_no_new_version_on_any_drive() {
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
|
||||
for size in [4096, TEST_OBJECT_SIZE] {
|
||||
// (versioned, seed an earlier version first)
|
||||
for (versioned, seeded) in [(false, false), (false, true), (true, true)] {
|
||||
let case = format!("{write_completion:?} size={size} versioned={versioned} seeded={seeded}");
|
||||
let (dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-rollback-no-residue";
|
||||
let object = "rolled-back-object";
|
||||
make_completion_test_bucket(&disks, bucket).await;
|
||||
if seeded {
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![b'0'; size]),
|
||||
&ObjectOptions {
|
||||
versioned,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("{case}: seed version should commit on every drive: {err}"));
|
||||
}
|
||||
wait_for_tmp_workspace_to_drain(&dirs, "seed PUT must leave no staging").await;
|
||||
let before = per_drive_object_state(&dirs, bucket, object).await;
|
||||
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let _fault = rename_fault_injection::fail_rename_on(object, &[0, 1]);
|
||||
let err = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![b'1'; size]),
|
||||
&ObjectOptions {
|
||||
versioned,
|
||||
write_completion,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("two renamed drives cannot satisfy write quorum three");
|
||||
assert!(
|
||||
matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
|
||||
"{case}: original quorum error expected: {err}"
|
||||
);
|
||||
assert_eq!(tasks.running(), 0, "{case}: rename and undo tasks must be reaped before return");
|
||||
// An incomplete rollback retains staging for recovery, so a
|
||||
// drained workspace shows the rollback completed.
|
||||
wait_for_tmp_workspace_to_drain(&dirs, "a completed rollback must release staging").await;
|
||||
let after = per_drive_object_state(&dirs, bucket, object).await;
|
||||
assert_eq!(after, before, "{case}: no drive may keep the failed PUT's version or data");
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn put_incomplete_rollback_preserves_staging_and_old_version_backup() {
|
||||
|
||||
@@ -264,6 +264,66 @@ enum MetadataReadPurpose {
|
||||
GetObjectReader,
|
||||
}
|
||||
|
||||
/// How a metadata read-quorum failure is reported when the replies prove the
|
||||
/// object or version dangling. Only the object read entry points
|
||||
/// (`get_object_info`, `get_object_reader` and prepared GET metadata) report
|
||||
/// it as not found; read-before-write callers of `get_object_fileinfo*` keep
|
||||
/// the quorum error so fences and delete recovery never treat a torn write as
|
||||
/// an absent key.
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum DanglingRead {
|
||||
KeepQuorumError,
|
||||
ReportNotFound,
|
||||
}
|
||||
|
||||
/// Read-only form of MinIO's GET-path dangling check: once
|
||||
/// `totalResp == setDriveCount`, `getObjectFileInfo` calls `deleteIfDangling`
|
||||
/// and a dangling verdict replaces the read-quorum error with
|
||||
/// `errFileNotFound` or `errFileVersionNotFound`.
|
||||
///
|
||||
/// Each deliberate difference keeps the quorum error instead:
|
||||
/// - nothing is deleted; heal owns cleanup and its recent-write grace window;
|
||||
/// - MinIO counts an offline drive as a response and lets `isObjectDangling`
|
||||
/// reject it, while here every slot must hold valid metadata or a not-found
|
||||
/// reply, so an offline drive, a timeout, an unanswered slot, or a corrupt or
|
||||
/// unusable xl.meta stays uncertain;
|
||||
/// - the valid replies must be one metadata generation and not a remote
|
||||
/// (transitioned) version, the candidates heal's `dangling_delete_safety`
|
||||
/// also refuses to delete.
|
||||
fn dangling_read_not_found(
|
||||
bucket: &str,
|
||||
version_id: &str,
|
||||
parts_metadata: &[FileInfo],
|
||||
errs: &[Option<DiskError>],
|
||||
fanout_complete: bool,
|
||||
) -> Option<DiskError> {
|
||||
if !fanout_complete || crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
return None;
|
||||
}
|
||||
let mut generation = None;
|
||||
for (metadata, err) in parts_metadata.iter().zip(errs) {
|
||||
match err {
|
||||
Some(DiskError::FileNotFound | DiskError::FileVersionNotFound) => {}
|
||||
Some(_) => return None,
|
||||
None => {
|
||||
if !super::file_info_is_valid_for_metadata(metadata) || metadata.is_remote() {
|
||||
return None;
|
||||
}
|
||||
let identity = SetDisks::file_info_quorum_hash(metadata);
|
||||
if *generation.get_or_insert(identity) != identity {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let (_, dangling) = super::is_object_dangling(parts_metadata, errs, &std::collections::HashMap::new());
|
||||
dangling.then_some(if version_id.is_empty() {
|
||||
DiskError::FileNotFound
|
||||
} else {
|
||||
DiskError::FileVersionNotFound
|
||||
})
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
/// A startup quorum can leave remote slots unregistered even after their
|
||||
/// peers become ready. Refresh those slots before taking the GET metadata
|
||||
@@ -427,6 +487,7 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn cache_get_object_fileinfo(
|
||||
&self,
|
||||
identity: (&str, &str),
|
||||
@@ -437,8 +498,32 @@ impl SetDisks {
|
||||
read_quorum: usize,
|
||||
) {
|
||||
let (fi, capability) = selected;
|
||||
let _ = self
|
||||
.cache_get_object_fileinfo_owned(
|
||||
identity,
|
||||
generation,
|
||||
(fi.clone(), capability, None),
|
||||
parts_metadata.to_vec(),
|
||||
online_disks.to_vec(),
|
||||
read_quorum,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn cache_get_object_fileinfo_owned(
|
||||
&self,
|
||||
identity: (&str, &str),
|
||||
generation: Option<GetObjectMetadataCacheGeneration>,
|
||||
selected: (FileInfo, GetObjectMetadataCacheCapability, Option<Bytes>),
|
||||
parts_metadata: Vec<FileInfo>,
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
read_quorum: usize,
|
||||
) -> GetObjectFileInfo {
|
||||
let (fi, capability, verified_inline_body) = selected;
|
||||
if fi.deleted || !fi.has_valid_erasure_geometry() {
|
||||
return;
|
||||
let mut snapshot = GetObjectFileInfo::owned(fi, parts_metadata, online_disks);
|
||||
snapshot.set_verified_inline_body(verified_inline_body);
|
||||
return snapshot;
|
||||
}
|
||||
let (bucket, object) = identity;
|
||||
|
||||
@@ -449,20 +534,24 @@ impl SetDisks {
|
||||
GET_METADATA_CACHE_DECISION_REJECT,
|
||||
GET_METADATA_CACHE_REASON_STALE_PUBLICATION,
|
||||
);
|
||||
return;
|
||||
let mut snapshot = GetObjectFileInfo::owned(fi, parts_metadata, online_disks);
|
||||
snapshot.set_verified_inline_body(verified_inline_body);
|
||||
return snapshot;
|
||||
};
|
||||
|
||||
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
|
||||
let entry = Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
fi: fi.clone(),
|
||||
parts_metadata: parts_metadata.to_vec(),
|
||||
online_disks: online_disks.to_vec(),
|
||||
fi,
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
read_quorum,
|
||||
capability,
|
||||
verified_inline_body,
|
||||
});
|
||||
self.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || {})
|
||||
self.insert_get_object_metadata_cache_entry_after_insert(key, generation, Arc::clone(&entry), || {})
|
||||
.await;
|
||||
GetObjectFileInfo::owned_published(entry)
|
||||
}
|
||||
|
||||
async fn insert_get_object_metadata_cache_entry_after_insert(
|
||||
@@ -583,6 +672,31 @@ impl SetDisks {
|
||||
read_data,
|
||||
caller_allows_early_stop,
|
||||
MetadataReadPurpose::FullLayout,
|
||||
DanglingRead::KeepQuorumError,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// `get_object_fileinfo` for `get_object_info` and prepared GET metadata:
|
||||
/// a provably dangling object or version reads as not found.
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
pub(super) async fn get_object_fileinfo_for_object_read(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
read_data: bool,
|
||||
caller_allows_early_stop: bool,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
self.get_object_fileinfo_gated_inner(
|
||||
bucket,
|
||||
object,
|
||||
opts,
|
||||
read_data,
|
||||
caller_allows_early_stop,
|
||||
MetadataReadPurpose::FullLayout,
|
||||
DanglingRead::ReportNotFound,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -604,6 +718,7 @@ impl SetDisks {
|
||||
read_data,
|
||||
caller_allows_early_stop,
|
||||
MetadataReadPurpose::GetObjectReader,
|
||||
DanglingRead::ReportNotFound,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -621,8 +736,16 @@ impl SetDisks {
|
||||
read_data: bool,
|
||||
allow_early_stop: bool,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, allow_early_stop, MetadataReadPurpose::FullLayout)
|
||||
.await
|
||||
self.get_object_fileinfo_gated_inner(
|
||||
bucket,
|
||||
object,
|
||||
opts,
|
||||
read_data,
|
||||
allow_early_stop,
|
||||
MetadataReadPurpose::FullLayout,
|
||||
DanglingRead::KeepQuorumError,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -634,6 +757,7 @@ impl SetDisks {
|
||||
read_data: bool,
|
||||
allow_early_stop: bool,
|
||||
purpose: MetadataReadPurpose,
|
||||
dangling_read: DanglingRead,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
let vid = opts.version_id.clone().unwrap_or_default();
|
||||
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
@@ -793,7 +917,17 @@ impl SetDisks {
|
||||
let _min_disks = self.set_drive_count - self.default_parity_count;
|
||||
|
||||
let metadata_resolve_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
// Like MinIO, check for a dangling object after either metadata quorum
|
||||
// reduction fails; only a read-quorum failure is reclassified.
|
||||
let dangling_not_found = |err: &DiskError| {
|
||||
if dangling_read == DanglingRead::ReportNotFound && *err == DiskError::ErasureReadQuorum {
|
||||
dangling_read_not_found(bucket, &vid, &parts_metadata, &errs, metadata_fanout_complete)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)
|
||||
.map_err(|err| dangling_not_found(&err).unwrap_or(err))
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object, &vid]))
|
||||
{
|
||||
Ok(v) => v,
|
||||
@@ -813,6 +947,15 @@ impl SetDisks {
|
||||
let write_quorum = usize::try_from(write_quorum)
|
||||
.map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
if let Some(not_found) = dangling_not_found(&err) {
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "not_found");
|
||||
record_get_stage_duration_if_enabled(
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
GET_STAGE_METADATA_RESOLVE,
|
||||
metadata_resolve_stage_start,
|
||||
);
|
||||
return Err(to_object_err(not_found.into(), vec![bucket, object, &vid]));
|
||||
}
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "not_reached");
|
||||
error!("reduce_read_quorum_errs: {:?}, bucket: {}, object: {}", &err, bucket, object);
|
||||
record_get_stage_duration_if_enabled(
|
||||
@@ -845,7 +988,11 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
metadata_fanout_diagnostics.record_quorum_candidate_latency(metadata_metrics_path, fileinfo_selection_quorum);
|
||||
if !opts.suppress_read_repair && errs.iter().any(|err| err.is_some()) {
|
||||
let needs_late_metadata_fanout = !metadata_fanout_complete
|
||||
&& allow_early_stop
|
||||
&& non_inline_data_read_early_stop_allowed(read_data, bucket, object)
|
||||
&& late_materialization_candidate_is_safe(&fi);
|
||||
let mut snapshot = if !opts.suppress_read_repair && errs.iter().any(|err| err.is_some()) {
|
||||
let version_id = resolved_read_repair_version_id(&fi, opts.version_id.as_deref());
|
||||
submit_read_repair_heal(
|
||||
&fi.volume,
|
||||
@@ -857,6 +1004,9 @@ impl SetDisks {
|
||||
"metadata_read_error",
|
||||
)
|
||||
.await;
|
||||
let mut snapshot = GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks);
|
||||
snapshot.set_verified_inline_body(verified_inline_body);
|
||||
snapshot
|
||||
} else if use_metadata_cache
|
||||
&& (metadata_fanout_complete
|
||||
|| condition_stopped
|
||||
@@ -869,11 +1019,11 @@ impl SetDisks {
|
||||
{
|
||||
#[cfg(test)]
|
||||
metadata_cache_tests::wait_before_metadata_cache_publish(bucket, object).await;
|
||||
self.cache_get_object_fileinfo(
|
||||
self.cache_get_object_fileinfo_owned(
|
||||
(bucket, object),
|
||||
metadata_cache_generation,
|
||||
(
|
||||
&fi,
|
||||
fi,
|
||||
if metadata_fanout_complete {
|
||||
GetObjectMetadataCacheCapability::Full
|
||||
} else if condition_stopped {
|
||||
@@ -881,29 +1031,23 @@ impl SetDisks {
|
||||
} else {
|
||||
GetObjectMetadataCacheCapability::QuorumSelected
|
||||
},
|
||||
verified_inline_body,
|
||||
),
|
||||
&parts_metadata,
|
||||
&op_online_disks,
|
||||
parts_metadata,
|
||||
op_online_disks,
|
||||
read_quorum,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_METADATA_RESOLVE, metadata_resolve_stage_start);
|
||||
// debug!("get_object_fileinfo pick fi {:?}", &fi);
|
||||
|
||||
// let online_disks: Vec<Option<DiskStore>> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect();
|
||||
|
||||
let mut snapshot = if !metadata_fanout_complete
|
||||
&& allow_early_stop
|
||||
&& non_inline_data_read_early_stop_allowed(read_data, bucket, object)
|
||||
&& late_materialization_candidate_is_safe(&fi)
|
||||
{
|
||||
GetObjectFileInfo::owned_with_late_metadata_fanout(fi, parts_metadata, op_online_disks, disks)
|
||||
.await
|
||||
} else {
|
||||
GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks)
|
||||
let mut snapshot = GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks);
|
||||
snapshot.set_verified_inline_body(verified_inline_body);
|
||||
snapshot
|
||||
};
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_METADATA_RESOLVE, metadata_resolve_stage_start);
|
||||
if needs_late_metadata_fanout {
|
||||
snapshot.set_late_metadata_fanout_disks(disks);
|
||||
}
|
||||
snapshot.condition_stopped = condition_stopped;
|
||||
snapshot.set_verified_inline_body(verified_inline_body);
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
@@ -4030,6 +4174,7 @@ mod metadata_cache_tests {
|
||||
parts_metadata: vec![fi],
|
||||
online_disks: vec![None],
|
||||
read_quorum: 1,
|
||||
verified_inline_body: None,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
@@ -4691,6 +4836,7 @@ mod metadata_cache_tests {
|
||||
online_disks: Vec::new(),
|
||||
read_quorum: 0,
|
||||
capability,
|
||||
verified_inline_body: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -5183,6 +5329,7 @@ mod metadata_cache_tests {
|
||||
parts_metadata: vec![fi],
|
||||
online_disks: vec![None],
|
||||
read_quorum: 1,
|
||||
verified_inline_body: None,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
@@ -5301,6 +5448,7 @@ mod metadata_cache_tests {
|
||||
parts_metadata: vec![fi],
|
||||
online_disks: Vec::new(),
|
||||
read_quorum: 0,
|
||||
verified_inline_body: None,
|
||||
});
|
||||
|
||||
set.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || {
|
||||
@@ -5436,6 +5584,7 @@ mod metadata_cache_tests {
|
||||
fi,
|
||||
online_disks: Vec::new(),
|
||||
read_quorum: 0,
|
||||
verified_inline_body: None,
|
||||
})
|
||||
};
|
||||
|
||||
@@ -9591,3 +9740,344 @@ mod tests {
|
||||
assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod dangling_read_tests {
|
||||
//! GET and HEAD report a metadata read-quorum failure as not found only when
|
||||
//! every drive answered and the surviving copies prove the object or version
|
||||
//! dangling; any uncertainty keeps the read-quorum error (503 SlowDownRead).
|
||||
use super::*;
|
||||
use crate::disk::{DeleteOptions, STORAGE_FORMAT_FILE};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::object::ObjectOperations;
|
||||
use http::HeaderMap;
|
||||
use std::path::PathBuf;
|
||||
use tempfile::TempDir;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
/// EC 4+2: read quorum is four, so three surviving copies are below it
|
||||
/// while still reaching the half-set probe.
|
||||
const DRIVES: usize = 6;
|
||||
const PARITY: usize = 2;
|
||||
|
||||
async fn dangling_test_set(bucket: &str) -> (Vec<TempDir>, Arc<SetDisks>) {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
ctx.update_erasure_type(crate::layout::endpoints::SetupType::Erasure).await;
|
||||
let (dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(DRIVES, PARITY, ctx).await;
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("test bucket should be created on every drive");
|
||||
(dirs, set)
|
||||
}
|
||||
|
||||
async fn put(set: &SetDisks, bucket: &str, object: &str, body: &[u8], versioned: bool) -> ObjectInfo {
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(body.to_vec()),
|
||||
&ObjectOptions {
|
||||
versioned,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed object should commit on every drive")
|
||||
}
|
||||
|
||||
fn object_dir(dirs: &[TempDir], disk: usize, bucket: &str, object: &str) -> PathBuf {
|
||||
dirs[disk].path().join(bucket).join(object)
|
||||
}
|
||||
|
||||
async fn remove_object_from(dirs: &[TempDir], bucket: &str, object: &str, disks: &[usize]) {
|
||||
for &disk in disks {
|
||||
tokio::fs::remove_dir_all(object_dir(dirs, disk, bucket, object))
|
||||
.await
|
||||
.expect("object directory should be removable to model a drive that never received the write");
|
||||
}
|
||||
}
|
||||
|
||||
fn read_opts(version_id: Option<&str>) -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
version_id: version_id.map(str::to_owned),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn head(set: &SetDisks, bucket: &str, object: &str, version_id: Option<&str>) -> Result<ObjectInfo> {
|
||||
set.get_object_info(bucket, object, &read_opts(version_id)).await
|
||||
}
|
||||
|
||||
async fn get(set: &SetDisks, bucket: &str, object: &str, version_id: Option<&str>) -> Result<Vec<u8>> {
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts(version_id))
|
||||
.await?;
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("an opened object should stream its body");
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
fn is_read_quorum_error(err: &StorageError) -> bool {
|
||||
matches!(err, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))
|
||||
}
|
||||
|
||||
async fn assert_get_and_head_keep_read_quorum_error(set: &SetDisks, bucket: &str, object: &str, case: &str) {
|
||||
let Err(err) = head(set, bucket, object, None).await else {
|
||||
panic!("{case}: HEAD must not serve an object below read quorum");
|
||||
};
|
||||
assert!(is_read_quorum_error(&err), "{case}: HEAD must keep the read-quorum error: {err:?}");
|
||||
let Err(err) = get(set, bucket, object, None).await else {
|
||||
panic!("{case}: GET must not serve an object below read quorum");
|
||||
};
|
||||
assert!(is_read_quorum_error(&err), "{case}: GET must keep the read-quorum error: {err:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_object_reads_as_not_found_on_get_and_head() {
|
||||
let bucket = "dangling-read-unversioned";
|
||||
let object = "torn/object";
|
||||
let (dirs, set) = dangling_test_set(bucket).await;
|
||||
put(&set, bucket, object, b"torn write payload", false).await;
|
||||
// Three copies stay below read quorum four; the other drives answer FileNotFound.
|
||||
remove_object_from(&dirs, bucket, object, &[3, 4, 5]).await;
|
||||
|
||||
let is_not_found = |err: &StorageError| matches!(err, StorageError::ObjectNotFound(b, o) if b == bucket && o == object);
|
||||
let Err(err) = head(&set, bucket, object, None).await else {
|
||||
panic!("HEAD must not serve a dangling object");
|
||||
};
|
||||
assert!(is_not_found(&err), "HEAD of a dangling object must be NoSuchKey: {err:?}");
|
||||
let Err(err) = get(&set, bucket, object, None).await else {
|
||||
panic!("GET must not serve a dangling object");
|
||||
};
|
||||
assert!(is_not_found(&err), "GET of a dangling object must be NoSuchKey: {err:?}");
|
||||
let Err(err) = set
|
||||
.prepare_get_object_metadata(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
else {
|
||||
panic!("prepared GET metadata must not resolve a dangling object");
|
||||
};
|
||||
assert!(is_not_found(&err), "prepared GET metadata must be NoSuchKey: {err:?}");
|
||||
|
||||
// Read-before-write callers keep the quorum error so fences and delete
|
||||
// recovery never treat a torn write as an absent key.
|
||||
let Err(err) = set
|
||||
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
|
||||
.await
|
||||
else {
|
||||
panic!("internal metadata reads must not resolve a dangling object");
|
||||
};
|
||||
assert!(is_read_quorum_error(&err), "internal metadata reads keep the quorum error: {err:?}");
|
||||
|
||||
// The classification is read-only; heal owns dangling cleanup.
|
||||
for disk in 0..3 {
|
||||
assert!(
|
||||
object_dir(&dirs, disk, bucket, object).join(STORAGE_FORMAT_FILE).exists(),
|
||||
"GET/HEAD must not delete the surviving copy on drive {disk}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_classification_requires_every_drive_to_answer() {
|
||||
let bucket = "dangling-read-offline";
|
||||
let object = "torn/object";
|
||||
let (dirs, set) = dangling_test_set(bucket).await;
|
||||
let payload = b"offline drive may hold the fourth copy";
|
||||
put(&set, bucket, object, payload, false).await;
|
||||
remove_object_from(&dirs, bucket, object, &[3, 4]).await;
|
||||
let offline = set.disks.write().await[5].take().expect("drive five should start online");
|
||||
|
||||
assert_get_and_head_keep_read_quorum_error(&set, bucket, object, "one drive offline").await;
|
||||
|
||||
// The offline drive held a copy: once it answers, read quorum is met.
|
||||
set.disks.write().await[5] = Some(offline);
|
||||
let body = get(&set, bucket, object, None)
|
||||
.await
|
||||
.expect("four copies satisfy read quorum once the drive returns");
|
||||
assert_eq!(body, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_version_reads_as_no_such_version_while_other_versions_stay_readable() {
|
||||
let bucket = "dangling-read-versioned";
|
||||
let object = "torn/versioned";
|
||||
let (_dirs, set) = dangling_test_set(bucket).await;
|
||||
let v1 = put(&set, bucket, object, b"version one", true)
|
||||
.await
|
||||
.version_id
|
||||
.expect("versioned PUT should assign a version id");
|
||||
let v2 = put(&set, bucket, object, b"version two", true)
|
||||
.await
|
||||
.version_id
|
||||
.expect("versioned PUT should assign a version id");
|
||||
let disks = set.disks.read().await.clone();
|
||||
for disk in disks.iter().skip(3) {
|
||||
let disk = disk.as_ref().expect("every drive should be online");
|
||||
let mut remove = FileInfo {
|
||||
volume: bucket.to_string(),
|
||||
name: object.to_string(),
|
||||
version_id: Some(v2),
|
||||
..Default::default()
|
||||
};
|
||||
remove.set_skip_tier_free_version();
|
||||
disk.delete_version(bucket, object, remove, false, DeleteOptions::default())
|
||||
.await
|
||||
.expect("dropping version two should model a drive that never received it");
|
||||
assert!(matches!(
|
||||
disk.read_version("", bucket, object, &v2.to_string(), &ReadOptions::default())
|
||||
.await,
|
||||
Err(DiskError::FileVersionNotFound | DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
let v1 = v1.to_string();
|
||||
let v2 = v2.to_string();
|
||||
let is_no_such_version =
|
||||
|err: &StorageError| matches!(err, StorageError::VersionNotFound(b, o, v) if b == bucket && o == object && v == &v2);
|
||||
let Err(err) = head(&set, bucket, object, Some(&v2)).await else {
|
||||
panic!("HEAD must not serve a dangling version");
|
||||
};
|
||||
assert!(is_no_such_version(&err), "HEAD of a dangling version must be NoSuchVersion: {err:?}");
|
||||
let Err(err) = get(&set, bucket, object, Some(&v2)).await else {
|
||||
panic!("GET must not serve a dangling version");
|
||||
};
|
||||
assert!(is_no_such_version(&err), "GET of a dangling version must be NoSuchVersion: {err:?}");
|
||||
|
||||
assert_eq!(
|
||||
get(&set, bucket, object, Some(&v1))
|
||||
.await
|
||||
.expect("the committed version must stay readable"),
|
||||
b"version one"
|
||||
);
|
||||
head(&set, bucket, object, Some(&v1))
|
||||
.await
|
||||
.expect("HEAD of the committed version must succeed");
|
||||
// The latest read sees two generations (version two on three drives,
|
||||
// version one on the rest), which is not proof of absence.
|
||||
let Err(err) = head(&set, bucket, object, None).await else {
|
||||
panic!("a split latest version must not resolve without quorum");
|
||||
};
|
||||
assert!(is_read_quorum_error(&err), "a split latest version keeps the quorum error: {err:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_classification_keeps_quorum_error_for_corrupt_metadata() {
|
||||
let bucket = "dangling-read-corrupt";
|
||||
let object = "torn/object";
|
||||
let (dirs, set) = dangling_test_set(bucket).await;
|
||||
put(&set, bucket, object, b"corrupt copies are present, not absent", false).await;
|
||||
for disk in 3..DRIVES {
|
||||
tokio::fs::write(object_dir(&dirs, disk, bucket, object).join(STORAGE_FORMAT_FILE), b"corrupt metadata")
|
||||
.await
|
||||
.expect("xl.meta should be corruptible for the test");
|
||||
}
|
||||
|
||||
assert_get_and_head_keep_read_quorum_error(&set, bucket, object, "corrupt xl.meta").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_classification_keeps_quorum_error_for_two_generations() {
|
||||
let bucket = "dangling-read-two-generations";
|
||||
let object = "torn/object";
|
||||
let (dirs, set) = dangling_test_set(bucket).await;
|
||||
put(&set, bucket, object, b"first generation", false).await;
|
||||
let mut first_generation = Vec::new();
|
||||
for disk in 0..2 {
|
||||
let path = object_dir(&dirs, disk, bucket, object).join(STORAGE_FORMAT_FILE);
|
||||
let bytes = tokio::fs::read(&path)
|
||||
.await
|
||||
.expect("first generation xl.meta should be readable");
|
||||
first_generation.push((path, bytes));
|
||||
}
|
||||
put(&set, bucket, object, b"second generation", false).await;
|
||||
// First generation on two drives, second on one, three drives absent:
|
||||
// each generation is below read quorum and neither proves the other absent.
|
||||
for (path, bytes) in first_generation {
|
||||
tokio::fs::write(&path, bytes)
|
||||
.await
|
||||
.expect("first generation xl.meta should be restorable");
|
||||
}
|
||||
remove_object_from(&dirs, bucket, object, &[3, 4, 5]).await;
|
||||
|
||||
assert_get_and_head_keep_read_quorum_error(&set, bucket, object, "two generations").await;
|
||||
}
|
||||
|
||||
/// Writes `fi` as the only version on the first `copies` drives; the
|
||||
/// remaining drives answer FileNotFound.
|
||||
async fn write_metadata_copies(set: &SetDisks, bucket: &str, object: &str, fi: &FileInfo, copies: usize) {
|
||||
let disks = set.disks.read().await.clone();
|
||||
for (index, disk) in disks.iter().take(copies).enumerate() {
|
||||
let mut copy = fi.clone();
|
||||
copy.erasure.index = index + 1;
|
||||
disk.as_ref()
|
||||
.expect("every drive should be online")
|
||||
.write_metadata(bucket, bucket, object, copy)
|
||||
.await
|
||||
.expect("hand-written metadata should be stored");
|
||||
}
|
||||
}
|
||||
|
||||
fn handwritten_version(bucket: &str, object: &str, data_blocks: usize, parity_blocks: usize) -> FileInfo {
|
||||
let mut fi = FileInfo::new(object, data_blocks, parity_blocks);
|
||||
fi.volume = bucket.to_string();
|
||||
fi.name = object.to_string();
|
||||
fi.size = 1;
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
fi.metadata.insert("etag".to_string(), "handwritten-etag".to_string());
|
||||
fi.add_object_part(1, "part-etag".to_string(), 1, None, 1, None, None);
|
||||
fi
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_zero_parity_version_reads_as_not_found() {
|
||||
// An EC:0 version on a default EC:2 set passes the common-parity check
|
||||
// with four of six copies (n/2 + 1), so its read fails at the later
|
||||
// read-quorum reduction, which must classify it as well.
|
||||
let bucket = "dangling-read-zero-parity";
|
||||
let object = "torn/zero-parity";
|
||||
let (_dirs, set) = dangling_test_set(bucket).await;
|
||||
write_metadata_copies(&set, bucket, object, &handwritten_version(bucket, object, DRIVES, 0), 4).await;
|
||||
|
||||
let is_not_found = |err: &StorageError| matches!(err, StorageError::ObjectNotFound(b, o) if b == bucket && o == object);
|
||||
let Err(err) = head(&set, bucket, object, None).await else {
|
||||
panic!("HEAD must not serve a zero-parity object missing two shards");
|
||||
};
|
||||
assert!(is_not_found(&err), "HEAD of a dangling zero-parity object must be NoSuchKey: {err:?}");
|
||||
let Err(err) = get(&set, bucket, object, None).await else {
|
||||
panic!("GET must not serve a zero-parity object missing two shards");
|
||||
};
|
||||
assert!(is_not_found(&err), "GET of a dangling zero-parity object must be NoSuchKey: {err:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_classification_keeps_quorum_error_for_remote_versions() {
|
||||
// Local metadata is the only pointer to tiered data, and heal never
|
||||
// deletes a remote candidate as dangling, so GET must not report it absent.
|
||||
let bucket = "dangling-read-remote";
|
||||
let object = "torn/transitioned";
|
||||
let (_dirs, set) = dangling_test_set(bucket).await;
|
||||
let mut fi = handwritten_version(bucket, object, DRIVES - PARITY, PARITY);
|
||||
fi.transition_status = crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string();
|
||||
fi.transition_tier = "WARM".to_string();
|
||||
fi.transitioned_objname = "remote/torn/transitioned".to_string();
|
||||
assert!(fi.is_remote(), "the fixture must be a remote version");
|
||||
write_metadata_copies(&set, bucket, object, &fi, 3).await;
|
||||
|
||||
assert_get_and_head_keep_read_quorum_error(&set, bucket, object, "remote version").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_classification_skips_the_meta_bucket() {
|
||||
// Like MinIO, internal objects are never classified: their readers
|
||||
// treat not-found as absent configuration.
|
||||
let object = "config/dangling-read.json";
|
||||
let (dirs, set) = dangling_test_set("dangling-read-meta").await;
|
||||
put(&set, RUSTFS_META_BUCKET, object, b"internal config", false).await;
|
||||
remove_object_from(&dirs, RUSTFS_META_BUCKET, object, &[3, 4, 5]).await;
|
||||
|
||||
assert_get_and_head_keep_read_quorum_error(&set, RUSTFS_META_BUCKET, object, "meta bucket").await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3221,6 +3221,9 @@ mod tests {
|
||||
#[serial_test::serial]
|
||||
async fn full_format_heal_lost_after_last_save_does_not_publish_or_renew_and_latches() {
|
||||
let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), store.pools[0].stop_endpoint_monitor_for_test())
|
||||
.await
|
||||
.expect("endpoint reconnection must stop before the format-loss fixture takes ownership");
|
||||
let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
|
||||
store.pools[0].disk_set[0].disks.write().await[3] = None;
|
||||
let barrier = HealFormatAfterSaveBarrier::install(&store.pools[0], 3);
|
||||
|
||||
@@ -5090,8 +5090,8 @@ mod tests {
|
||||
assert!(Arc::ptr_eq(disk, expected), "endpoint monitor must not replace an offline test handle");
|
||||
assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
|
||||
assert!(
|
||||
disk.is_online().await,
|
||||
"the valid disk identity must prevent endpoint renewal while the IO health gate remains offline"
|
||||
!disk.is_online().await,
|
||||
"the explicit drive fault must survive renewal despite a valid disk identity"
|
||||
);
|
||||
}
|
||||
OfflineTestDisks { disks }
|
||||
@@ -12018,7 +12018,9 @@ mod tests {
|
||||
Ok::<(), Error>(())
|
||||
}
|
||||
})
|
||||
.buffer_unordered(32)
|
||||
// Pagination, not write throughput, is the oracle. Drain each pair of
|
||||
// durable receipts before preparing the next to bound fixture IO.
|
||||
.buffer_unordered(1)
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.expect("more than one receipt page should persist");
|
||||
|
||||
+1192
-1932
File diff suppressed because it is too large
Load Diff
@@ -14,14 +14,14 @@
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::disk::STORAGE_FORMAT_FILE;
|
||||
use crate::disk::{DiskAPI, STORAGE_FORMAT_FILE, WalkDirOptions};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo};
|
||||
use rustfs_filemeta::{FileInfo, MetacacheReader, ObjectPartInfo};
|
||||
use std::collections::BTreeMap;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncWrite, DuplexStream};
|
||||
use tokio::io::{AsyncWrite, DuplexStream, duplex};
|
||||
use tokio::sync::Notify;
|
||||
use tokio::time::timeout;
|
||||
|
||||
@@ -468,6 +468,50 @@ async fn list_objects_shared_corpus_offline_corrupt_and_minority_stale() {
|
||||
fixture.assert_walk("", None, None, 3).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn list_objects_shared_corpus_quorum_delete_leaves_no_resurrectable_stale_copy() {
|
||||
let mut fixture = Fixture::new("list-corpus-stale-copies", namespace(&shared_keys()), true).await;
|
||||
// A delete that reached write quorum (three of four drives) leaves one
|
||||
// stale copy, the most any quorum delete can leave on this set.
|
||||
let deleted = ["a0", "z"];
|
||||
for key in deleted {
|
||||
let physical_key = rustfs_utils::path::encode_dir_object(key);
|
||||
for dir in fixture.dirs.iter().take(3) {
|
||||
tokio::fs::remove_dir_all(dir.path().join(&fixture.bucket).join(&physical_key))
|
||||
.await
|
||||
.expect("quorum delete fixture should remove the object directory");
|
||||
}
|
||||
}
|
||||
fixture.objects.retain(|object| !deleted.contains(&object.key.as_str()));
|
||||
|
||||
temp_env::async_with_vars([(ENV_API_LIST_OBJECTS_QUORUM, Some("strict"))], async {
|
||||
for max in [1, 3, 1000] {
|
||||
fixture.assert_walk("", None, None, max).await;
|
||||
fixture.assert_walk("", Some("/"), None, max).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
// A sampling mode promises only that no committed key goes missing.
|
||||
temp_env::async_with_vars([(ENV_API_LIST_OBJECTS_QUORUM, Some("optimal"))], async {
|
||||
let mut listed = Vec::new();
|
||||
let mut marker = None;
|
||||
for _ in 0..=fixture.objects.len() + deleted.len() {
|
||||
let page = fixture.page("", marker.clone(), None, 3).await;
|
||||
listed.extend(page.objects.into_iter().map(|object| object.name));
|
||||
if !page.is_truncated {
|
||||
break;
|
||||
}
|
||||
marker = page.next_marker;
|
||||
}
|
||||
for object in &fixture.objects {
|
||||
assert!(listed.contains(&object.key), "optimal LIST lost committed key {}", object.key);
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_objects_shared_corpus_versions_delete_markers_and_free_versions() {
|
||||
let mut objects = vec![
|
||||
|
||||
@@ -446,7 +446,6 @@ fn bench_metacache_resolve(c: &mut Criterion) {
|
||||
requested_versions: 1,
|
||||
bucket: "bucket".to_string(),
|
||||
strict: false,
|
||||
write_quorum_slack: 0,
|
||||
candidates: Vec::new(),
|
||||
});
|
||||
black_box(resolved);
|
||||
@@ -463,24 +462,6 @@ fn bench_metacache_resolve(c: &mut Criterion) {
|
||||
requested_versions: 1,
|
||||
bucket: "bucket".to_string(),
|
||||
strict: false,
|
||||
write_quorum_slack: 0,
|
||||
candidates: Vec::new(),
|
||||
});
|
||||
black_box(resolved);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
group.bench_function("8_disks_write_quorum/1000_objects", |b| {
|
||||
b.iter(|| {
|
||||
for entries in &sets_8_agree {
|
||||
let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams {
|
||||
dir_quorum: 4,
|
||||
obj_quorum: 4,
|
||||
requested_versions: 1,
|
||||
bucket: "bucket".to_string(),
|
||||
strict: true,
|
||||
write_quorum_slack: 0,
|
||||
candidates: Vec::new(),
|
||||
});
|
||||
black_box(resolved);
|
||||
@@ -491,13 +472,12 @@ fn bench_metacache_resolve(c: &mut Criterion) {
|
||||
group.bench_function("8_disks_disagree_merge/1000_objects", |b| {
|
||||
b.iter(|| {
|
||||
for entries in &sets_8_disagree {
|
||||
let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams {
|
||||
let resolved = entries.resolve(MetadataResolutionParams {
|
||||
dir_quorum: 4,
|
||||
obj_quorum: 4,
|
||||
requested_versions: 1,
|
||||
bucket: "bucket".to_string(),
|
||||
strict: true,
|
||||
write_quorum_slack: 0,
|
||||
candidates: Vec::new(),
|
||||
});
|
||||
black_box(resolved);
|
||||
@@ -508,13 +488,12 @@ fn bench_metacache_resolve(c: &mut Criterion) {
|
||||
group.bench_function("8_disks_agree_13_versions/1000_objects", |b| {
|
||||
b.iter(|| {
|
||||
for entries in &sets_8_agree_13v {
|
||||
let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams {
|
||||
let resolved = entries.resolve(MetadataResolutionParams {
|
||||
dir_quorum: 4,
|
||||
obj_quorum: 4,
|
||||
requested_versions: 1,
|
||||
bucket: "bucket".to_string(),
|
||||
strict: true,
|
||||
write_quorum_slack: 0,
|
||||
candidates: Vec::new(),
|
||||
});
|
||||
black_box(resolved);
|
||||
@@ -525,13 +504,12 @@ fn bench_metacache_resolve(c: &mut Criterion) {
|
||||
group.bench_function("8_disks_disagree_13_versions/1000_objects", |b| {
|
||||
b.iter(|| {
|
||||
for entries in &sets_8_disagree_13v {
|
||||
let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams {
|
||||
let resolved = entries.resolve(MetadataResolutionParams {
|
||||
dir_quorum: 4,
|
||||
obj_quorum: 4,
|
||||
requested_versions: 1,
|
||||
bucket: "bucket".to_string(),
|
||||
strict: true,
|
||||
write_quorum_slack: 0,
|
||||
candidates: Vec::new(),
|
||||
});
|
||||
black_box(resolved);
|
||||
@@ -596,191 +574,6 @@ fn bench_metacache_resolve(c: &mut Criterion) {
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn listing_history(versions: usize, cached: bool) -> MetaCacheEntry {
|
||||
let mut metadata = FileMeta::new();
|
||||
for index in 0..versions {
|
||||
let (data, parity) = if index + 1 == versions { (12, 4) } else { (8, 8) };
|
||||
let mut info = FileInfo::new("object", data, parity);
|
||||
info.volume = "bucket".to_owned();
|
||||
info.erasure.index = 1;
|
||||
info.data_dir = Some(Uuid::from_u128(0x1000 + u128::try_from(index).expect("data directory index")));
|
||||
info.versioned = true;
|
||||
info.version_id = Some(Uuid::from_u128(u128::try_from(index + 1).expect("version index")));
|
||||
info.mod_time = Some(
|
||||
OffsetDateTime::from_unix_timestamp(1_700_000_000 + i64::try_from(index).expect("timestamp")).expect("fixture time"),
|
||||
);
|
||||
info.size = 1;
|
||||
info.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
}];
|
||||
info.metadata.insert("etag".to_owned(), format!("etag-{index}"));
|
||||
info.metadata.insert("x-amz-meta-description".to_owned(), "m".repeat(1024));
|
||||
metadata.add_version(info).expect("valid version history");
|
||||
}
|
||||
MetaCacheEntry {
|
||||
name: "object".to_owned(),
|
||||
metadata: metadata.marshal_msg().expect("serialize history"),
|
||||
cached: cached.then_some(metadata),
|
||||
reusable: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn owned_agreed_latest_reference(
|
||||
mut entry: MetaCacheEntry,
|
||||
replicas: usize,
|
||||
params: &rustfs_filemeta::MetadataResolutionParams,
|
||||
enforce_write_quorum: bool,
|
||||
) -> rustfs_filemeta::MetaCacheEntryResolution {
|
||||
use rustfs_filemeta::MetaCacheEntryResolution::{NeedsSupplement, Rejected, Resolved};
|
||||
if !enforce_write_quorum {
|
||||
return Resolved(entry);
|
||||
}
|
||||
if entry.is_dir() {
|
||||
return if replicas >= params.dir_quorum {
|
||||
Resolved(entry)
|
||||
} else {
|
||||
Rejected
|
||||
};
|
||||
}
|
||||
if replicas < params.obj_quorum {
|
||||
return Rejected;
|
||||
}
|
||||
let metadata = match entry.xl_meta_ref() {
|
||||
Ok(metadata) => metadata,
|
||||
Err(_) if params.obj_quorum <= 1 => return Resolved(entry),
|
||||
Err(_) => return Rejected,
|
||||
};
|
||||
if params.requested_versions == 1
|
||||
&& let Some(first) = metadata.versions.first()
|
||||
&& !first.header.free_version()
|
||||
&& replicas >= first.listing_quorum(params.obj_quorum, params.write_quorum_slack)
|
||||
{
|
||||
return Resolved(entry);
|
||||
}
|
||||
// These fixtures cover the fast prefix, never older-version reconciliation.
|
||||
NeedsSupplement(entry)
|
||||
}
|
||||
|
||||
fn bench_listing_borrowed_metadata(c: &mut Criterion) {
|
||||
use rustfs_filemeta::{FileMetaVersion, MetaCacheEntries, MetadataResolutionParams};
|
||||
let mut group = c.benchmark_group("listing_borrowed_metadata");
|
||||
group
|
||||
.sample_size(20)
|
||||
.warm_up_time(Duration::from_millis(150))
|
||||
.measurement_time(Duration::from_millis(700));
|
||||
for versions in [2, 16, 256] {
|
||||
for cached in [false, true] {
|
||||
let entry = listing_history(versions, cached);
|
||||
let params = MetadataResolutionParams {
|
||||
dir_quorum: 8,
|
||||
obj_quorum: 8,
|
||||
requested_versions: 1,
|
||||
write_quorum_slack: 8,
|
||||
bucket: "bucket".to_owned(),
|
||||
..Default::default()
|
||||
};
|
||||
let slots = vec![Some(&entry); 8];
|
||||
let owned = MetaCacheEntries(vec![Some(entry.clone()); 8])
|
||||
.resolve_with_write_quorum(params.clone())
|
||||
.expect("previous committed version");
|
||||
let borrowed =
|
||||
MetaCacheEntries::resolve_borrowed_with_write_quorum(&slots, params.clone()).expect("borrowed previous version");
|
||||
assert_eq!(owned.metadata, borrowed.metadata);
|
||||
let id = format!("{versions}_versions/{}", if cached { "cached" } else { "cold" });
|
||||
group.throughput(Throughput::Bytes(u64::try_from(entry.metadata.len()).expect("metadata size")));
|
||||
// ABBA order permits an explicit baseline-drift check for each fixture.
|
||||
for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] {
|
||||
group.bench_function(BenchmarkId::new("replicas", format!("{id}/{phase}")), |b| {
|
||||
b.iter(|| {
|
||||
if phase.starts_with("owned") {
|
||||
black_box(
|
||||
MetaCacheEntries(vec![Some(black_box(&entry).clone()); 8])
|
||||
.resolve_with_write_quorum(params.clone()),
|
||||
)
|
||||
} else {
|
||||
let slots = vec![Some(black_box(&entry)); 8];
|
||||
black_box(MetaCacheEntries::resolve_borrowed_with_write_quorum(&slots, params.clone()))
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
let mut agreed_metadata = entry
|
||||
.cached
|
||||
.clone()
|
||||
.unwrap_or_else(|| FileMeta::load(&entry.metadata).expect("agreed fixture"));
|
||||
let mut latest = FileMetaVersion::try_from(agreed_metadata.versions[0].meta.as_slice()).expect("latest body");
|
||||
let object = latest.object.as_mut().expect("latest object");
|
||||
object.erasure_m = 8;
|
||||
object.erasure_n = 8;
|
||||
agreed_metadata.versions[0] = latest.try_into().expect("balanced latest header");
|
||||
let agreed = MetaCacheEntry {
|
||||
name: entry.name.clone(),
|
||||
metadata: agreed_metadata.marshal_msg().expect("agreed history"),
|
||||
cached: cached.then_some(agreed_metadata),
|
||||
reusable: false,
|
||||
};
|
||||
for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] {
|
||||
group.bench_function(BenchmarkId::new("agreed", format!("{id}/{phase}")), |b| {
|
||||
b.iter_batched(
|
||||
|| agreed.clone(),
|
||||
|candidate| {
|
||||
let resolved = if phase.starts_with("owned") {
|
||||
owned_agreed_latest_reference(candidate, 8, ¶ms, true)
|
||||
} else {
|
||||
rustfs_filemeta::resolve_agreed_entry(candidate, 8, ¶ms, true)
|
||||
};
|
||||
let rustfs_filemeta::MetaCacheEntryResolution::Resolved(candidate) = resolved else {
|
||||
panic!("balanced latest must use agreed fast prefix");
|
||||
};
|
||||
black_box(candidate)
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
);
|
||||
});
|
||||
}
|
||||
let replicas: Vec<_> = (1..=8)
|
||||
.map(|disk| {
|
||||
let mut metadata = entry
|
||||
.cached
|
||||
.clone()
|
||||
.unwrap_or_else(|| FileMeta::load(&entry.metadata).expect("decode fixture"));
|
||||
for version in &mut metadata.versions {
|
||||
let mut payload = FileMetaVersion::try_from(version.meta.as_slice()).expect("version body");
|
||||
payload.object.as_mut().expect("object body").erasure_index = disk;
|
||||
version.meta = payload.marshal_msg().expect("per-disk body");
|
||||
}
|
||||
MetaCacheEntry {
|
||||
name: entry.name.clone(),
|
||||
metadata: metadata.marshal_msg().expect("per-disk metadata"),
|
||||
cached: cached.then_some(metadata),
|
||||
reusable: false,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
assert!(replicas[0].headers_agree(&replicas, false));
|
||||
for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] {
|
||||
group.bench_function(BenchmarkId::new("headers", format!("{id}/{phase}")), |b| {
|
||||
b.iter(|| {
|
||||
let agreed = if phase.starts_with("owned") {
|
||||
replicas
|
||||
.iter()
|
||||
.all(|candidate| replicas[0].matches(Some(black_box(candidate)), false).1)
|
||||
} else {
|
||||
replicas[0].headers_agree(black_box(&replicas), false)
|
||||
};
|
||||
black_box(agreed)
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
bench_create_real_xlmeta,
|
||||
@@ -797,8 +590,7 @@ criterion_group!(
|
||||
bench_list_page_to_fileinfo,
|
||||
bench_delete_inline_versions,
|
||||
bench_metacache_stream_page,
|
||||
bench_metacache_resolve,
|
||||
bench_listing_borrowed_metadata
|
||||
bench_metacache_resolve
|
||||
);
|
||||
|
||||
criterion_main!(benches);
|
||||
|
||||
@@ -764,6 +764,17 @@ pub(crate) fn is_valid_distribution(distribution: &[usize], n: usize) -> bool {
|
||||
}
|
||||
|
||||
impl FileInfo {
|
||||
pub fn distribution_start(bucket: &str, object: &str, cardinality: usize) -> usize {
|
||||
if cardinality == 0 {
|
||||
return 0;
|
||||
}
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(bucket.as_bytes());
|
||||
hasher.update(b"/");
|
||||
hasher.update(object.as_bytes());
|
||||
(hasher.finalize() as u32 as usize) % cardinality
|
||||
}
|
||||
|
||||
pub fn new(object: &str, data_blocks: usize, parity_blocks: usize) -> Self {
|
||||
let indices = {
|
||||
let cardinality = data_blocks + parity_blocks;
|
||||
|
||||
@@ -115,6 +115,7 @@ pub(crate) fn data_key_for_version(version_id: Option<Uuid>) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn legacy_data_key_for_version(version_id: Option<Uuid>) -> Option<String> {
|
||||
if version_id.is_none() || version_id == Some(Uuid::nil()) {
|
||||
Some(Uuid::nil().to_string())
|
||||
@@ -154,7 +155,6 @@ pub fn is_skip_meta_key(key: &str) -> bool {
|
||||
}
|
||||
|
||||
mod codec;
|
||||
pub(crate) use codec::{BorrowedFileMeta, reconcile_headers};
|
||||
mod inline_data;
|
||||
pub(crate) mod msgp_decode;
|
||||
mod validation;
|
||||
@@ -348,19 +348,21 @@ impl FileMeta {
|
||||
self.versions.sort_by(cmp_shallow_versions_for_order);
|
||||
}
|
||||
|
||||
fn find_inline_data_slice(inline_data: &[u8], version_id: Option<Uuid>) -> Result<Option<&[u8]>> {
|
||||
if let Some(vid) = version_id.filter(|v| !v.is_nil()) {
|
||||
let mut buf = Uuid::encode_buffer();
|
||||
let key = vid.hyphenated().encode_lower(&mut buf);
|
||||
InlineData::find_slice_in_raw(inline_data, key)
|
||||
} else {
|
||||
if let Some(data) = InlineData::find_slice_in_raw(inline_data, NULL_VERSION_ID)? {
|
||||
return Ok(Some(data));
|
||||
}
|
||||
InlineData::find_slice_in_raw(inline_data, "00000000-0000-0000-0000-000000000000")
|
||||
}
|
||||
}
|
||||
|
||||
fn find_inline_data_in_slice(inline_data: &[u8], version_id: Option<Uuid>) -> Result<Option<Vec<u8>>> {
|
||||
let key = data_key_for_version(version_id);
|
||||
if let Some(data) = InlineData::find_in_slice(inline_data, key.as_str())? {
|
||||
return Ok(Some(data));
|
||||
}
|
||||
|
||||
if let Some(legacy_key) = legacy_data_key_for_version(version_id)
|
||||
&& legacy_key != key
|
||||
{
|
||||
return InlineData::find_in_slice(inline_data, legacy_key.as_str());
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
Ok(Self::find_inline_data_slice(inline_data, version_id)?.map(<[u8]>::to_vec))
|
||||
}
|
||||
|
||||
fn find_inline_data_for_version(&self, version_id: Option<Uuid>) -> Result<Option<Vec<u8>>> {
|
||||
|
||||
@@ -622,18 +622,6 @@ pub struct FileMetaShallowVersion {
|
||||
pub meta: Vec<u8>, // FileMetaVersion.marshal_msg
|
||||
}
|
||||
|
||||
fn write_quorum_from_erasure(data_blocks: usize, parity_blocks: usize) -> Option<usize> {
|
||||
if data_blocks == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(if data_blocks == parity_blocks {
|
||||
data_blocks.saturating_add(1)
|
||||
} else {
|
||||
data_blocks
|
||||
})
|
||||
}
|
||||
|
||||
impl FileMetaShallowVersion {
|
||||
/// Parse version meta with legacy format compatibility.
|
||||
/// Use this instead of `FileMetaVersion::default()` + `unmarshal_msg()` to handle old-version xl.meta.
|
||||
@@ -641,60 +629,11 @@ impl FileMetaShallowVersion {
|
||||
FileMetaVersion::try_from(self.meta.as_slice())
|
||||
}
|
||||
|
||||
pub fn write_quorum(&self, fallback_quorum: usize) -> usize {
|
||||
write_quorum_from_header_and_meta(&self.header, self.meta.as_slice(), fallback_quorum)
|
||||
}
|
||||
|
||||
/// Quorum for listing this version without accepting an unreadable payload.
|
||||
pub fn listing_quorum(&self, fallback_quorum: usize, unreachable_disks: usize) -> usize {
|
||||
listing_quorum_from_header_and_meta(&self.header, &self.meta, fallback_quorum, unreachable_disks)
|
||||
}
|
||||
|
||||
pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfo> {
|
||||
FileMetaVersion::decode_fileinfo_from_slice(self.meta.as_slice(), volume, path, all_parts, true)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn write_quorum_from_header_and_meta(header: &FileMetaVersionHeader, meta: &[u8], fallback_quorum: usize) -> usize {
|
||||
let header_quorum = header.write_quorum(fallback_quorum);
|
||||
if header.has_ec() || !matches!(header.version_type, VersionType::Object | VersionType::Legacy) {
|
||||
return header_quorum;
|
||||
}
|
||||
|
||||
FileMetaVersion::try_from(meta)
|
||||
.ok()
|
||||
.and_then(|version| version.erasure_write_quorum())
|
||||
.unwrap_or(header_quorum)
|
||||
}
|
||||
|
||||
pub(crate) fn listing_quorum_from_header_and_meta(
|
||||
header: &FileMetaVersionHeader,
|
||||
meta: &[u8],
|
||||
fallback_quorum: usize,
|
||||
unreachable_disks: usize,
|
||||
) -> usize {
|
||||
let geometry = if header.version_type == VersionType::Object && header.has_ec() {
|
||||
Some((usize::from(header.ec_m), usize::from(header.ec_n)))
|
||||
} else if matches!(header.version_type, VersionType::Object | VersionType::Legacy) {
|
||||
FileMetaVersion::try_from(meta)
|
||||
.ok()
|
||||
.and_then(|version| version.erasure_geometry())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let Some((data_blocks, parity_blocks)) = geometry else {
|
||||
return fallback_quorum;
|
||||
};
|
||||
let Some(write_quorum) = write_quorum_from_erasure(data_blocks, parity_blocks) else {
|
||||
return fallback_quorum;
|
||||
};
|
||||
write_quorum
|
||||
.saturating_sub(unreachable_disks)
|
||||
.max(data_blocks)
|
||||
.max(fallback_quorum)
|
||||
}
|
||||
|
||||
impl TryFrom<FileMetaVersion> for FileMetaShallowVersion {
|
||||
type Error = Error;
|
||||
|
||||
@@ -737,26 +676,6 @@ pub(super) struct V2ObjectSliceSummary {
|
||||
}
|
||||
|
||||
impl FileMetaVersion {
|
||||
fn erasure_geometry(&self) -> Option<(usize, usize)> {
|
||||
Some(match self.version_type {
|
||||
VersionType::Object | VersionType::Legacy => {
|
||||
if let Some(object) = &self.object {
|
||||
(object.erasure_m, object.erasure_n)
|
||||
} else if let Some(object) = &self.legacy_object {
|
||||
(object.erasure.data_blocks, object.erasure.parity_blocks)
|
||||
} else {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
fn erasure_write_quorum(&self) -> Option<usize> {
|
||||
let (data_blocks, parity_blocks) = self.erasure_geometry()?;
|
||||
write_quorum_from_erasure(data_blocks, parity_blocks)
|
||||
}
|
||||
|
||||
/// Fast borrowed scanner for `V2Obj` slices. Returning `Ok` guarantees
|
||||
/// `FileMetaVersion::try_from` would succeed with the identical summary fields;
|
||||
/// any non-`V2Obj` or invalid input returns `Err` so callers fall back to `try_from`.
|
||||
@@ -1775,14 +1694,6 @@ impl FileMetaVersionHeader {
|
||||
self.ec_m > 0 && self.ec_n > 0
|
||||
}
|
||||
|
||||
pub fn write_quorum(&self, fallback_quorum: usize) -> usize {
|
||||
if self.version_type != VersionType::Object || !self.has_ec() {
|
||||
return fallback_quorum;
|
||||
}
|
||||
|
||||
write_quorum_from_erasure(usize::from(self.ec_m), usize::from(self.ec_n)).unwrap_or(fallback_quorum)
|
||||
}
|
||||
|
||||
pub fn matches_not_strict(&self, o: &FileMetaVersionHeader) -> bool {
|
||||
let mut ok = self.version_id == o.version_id && self.version_type == o.version_type && self.matches_ec(o);
|
||||
// Disk-loaded headers keep the null version as Some(nil), not None: all null
|
||||
@@ -4167,15 +4078,49 @@ pub fn merge_file_meta_versions(
|
||||
.iter()
|
||||
.map(|v| v.iter().map(|ver| (&ver.header, ver.meta.as_slice())).collect())
|
||||
.collect();
|
||||
merge_file_meta_borrowed_streams(quorum, strict, requested_versions, false, 0, streams)
|
||||
merge_file_meta_borrowed_streams(quorum, strict, requested_versions, streams)
|
||||
}
|
||||
|
||||
/// Picks the most common header among the tops that share `latest`'s version
|
||||
/// id; a tie keeps the header that sorts first (the newest).
|
||||
fn most_common_header_for_version_id<'a>(
|
||||
tops: &[(&'a FileMetaVersionHeader, &'a [u8])],
|
||||
mut latest: (&'a FileMetaVersionHeader, &'a [u8]),
|
||||
strict: bool,
|
||||
) -> ((&'a FileMetaVersionHeader, &'a [u8]), usize) {
|
||||
let version_id = latest.0.version_id;
|
||||
let comparable = |header: &FileMetaVersionHeader| {
|
||||
let mut header = header.clone();
|
||||
if !strict {
|
||||
header.signature = [0; 4];
|
||||
}
|
||||
header
|
||||
};
|
||||
let mut counts: HashMap<FileMetaVersionHeader, usize> = HashMap::new();
|
||||
for &top in tops {
|
||||
if top.0.version_id == version_id {
|
||||
*counts.entry(comparable(top.0)).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
let mut latest_count = 0;
|
||||
for (header, count) in &counts {
|
||||
if *count < latest_count || (*count == latest_count && latest.0.sorts_before(header)) {
|
||||
continue;
|
||||
}
|
||||
for &top in tops {
|
||||
if comparable(top.0) == *header {
|
||||
latest = top;
|
||||
}
|
||||
}
|
||||
latest_count = *count;
|
||||
}
|
||||
(latest, latest_count)
|
||||
}
|
||||
|
||||
pub(crate) fn merge_file_meta_borrowed_streams(
|
||||
mut quorum: usize,
|
||||
mut strict: bool,
|
||||
requested_versions: usize,
|
||||
enforce_write_quorum: bool,
|
||||
write_quorum_slack: usize,
|
||||
mut streams: Vec<Vec<(&FileMetaVersionHeader, &[u8])>>,
|
||||
) -> Vec<FileMetaShallowVersion> {
|
||||
let to_shallow = |(header, meta): (&FileMetaVersionHeader, &[u8])| FileMetaShallowVersion {
|
||||
@@ -4192,32 +4137,13 @@ pub(crate) fn merge_file_meta_borrowed_streams(
|
||||
}
|
||||
|
||||
if streams.len() == 1 {
|
||||
if !enforce_write_quorum {
|
||||
return streams[0].iter().copied().map(to_shallow).collect();
|
||||
}
|
||||
|
||||
let required_quorum = streams[0]
|
||||
.first()
|
||||
.map(|&(hdr, meta)| listing_quorum_from_header_and_meta(hdr, meta, quorum, write_quorum_slack))
|
||||
.unwrap_or(quorum);
|
||||
if streams.len() >= required_quorum {
|
||||
return streams[0].iter().copied().map(to_shallow).collect();
|
||||
}
|
||||
return Vec::new();
|
||||
return streams[0].iter().copied().map(to_shallow).collect();
|
||||
}
|
||||
|
||||
if quorum == 1 {
|
||||
strict = true;
|
||||
}
|
||||
|
||||
let required_quorum = |(hdr, meta): (&FileMetaVersionHeader, &[u8])| {
|
||||
if enforce_write_quorum {
|
||||
listing_quorum_from_header_and_meta(hdr, meta, quorum, write_quorum_slack)
|
||||
} else {
|
||||
quorum
|
||||
}
|
||||
};
|
||||
|
||||
let mut tops: Vec<(&FileMetaVersionHeader, &[u8])> = Vec::with_capacity(streams.len());
|
||||
|
||||
let mut n_versions = 0;
|
||||
@@ -4249,7 +4175,7 @@ pub(crate) fn merge_file_meta_borrowed_streams(
|
||||
let latest: (&FileMetaVersionHeader, &[u8]);
|
||||
if consistent {
|
||||
latest = tops[0];
|
||||
if tops.len() >= required_quorum(latest) {
|
||||
if tops.len() >= quorum {
|
||||
merged.push(to_shallow(latest));
|
||||
if !latest.0.free_version() {
|
||||
n_versions += 1;
|
||||
@@ -4258,6 +4184,7 @@ pub(crate) fn merge_file_meta_borrowed_streams(
|
||||
} else {
|
||||
let mut latest_ref = tops[0];
|
||||
let mut latest_count = 1;
|
||||
let mut settled_by_version_id = false;
|
||||
for &ver in &tops[1..] {
|
||||
if ver.0 == latest_ref.0 {
|
||||
latest_count += 1;
|
||||
@@ -4281,39 +4208,26 @@ pub(crate) fn merge_file_meta_borrowed_streams(
|
||||
}
|
||||
|
||||
if ver.0.version_id == latest_ref.0.version_id {
|
||||
let mut x: HashMap<FileMetaVersionHeader, usize> = HashMap::new();
|
||||
for &a in &tops {
|
||||
if a.0.version_id != ver.0.version_id {
|
||||
continue;
|
||||
}
|
||||
let mut hdr = a.0.clone();
|
||||
if !strict {
|
||||
hdr.signature = [0; 4];
|
||||
}
|
||||
*x.entry(hdr).or_insert(0) += 1;
|
||||
}
|
||||
latest_count = 0;
|
||||
for (k, v) in x.iter() {
|
||||
if *v < latest_count || (*v == latest_count && latest_ref.0.sorts_before(k)) {
|
||||
continue;
|
||||
}
|
||||
for &a in &tops {
|
||||
let mut hdr = a.0.clone();
|
||||
if !strict {
|
||||
hdr.signature = [0; 4];
|
||||
}
|
||||
if hdr == *k {
|
||||
latest_ref = a;
|
||||
}
|
||||
}
|
||||
|
||||
latest_count = *v;
|
||||
}
|
||||
(latest_ref, latest_count) = most_common_header_for_version_id(&tops, latest_ref, strict);
|
||||
settled_by_version_id = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Readers return their heads in any order. When every older header
|
||||
// sharing the newest header's version id (an interrupted null-version
|
||||
// overwrite) came first, the scan above never compared them, and the
|
||||
// stream trim below would then drop that older committed generation.
|
||||
if !settled_by_version_id
|
||||
&& tops.iter().any(|ver| {
|
||||
ver.0 != latest_ref.0
|
||||
&& ver.0.version_id == latest_ref.0.version_id
|
||||
&& (strict || !ver.0.matches_not_strict(latest_ref.0))
|
||||
})
|
||||
{
|
||||
(latest_ref, latest_count) = most_common_header_for_version_id(&tops, latest_ref, strict);
|
||||
}
|
||||
latest = latest_ref;
|
||||
if latest_count >= required_quorum(latest) {
|
||||
if latest_count >= quorum {
|
||||
if !latest.0.free_version() {
|
||||
n_versions += 1;
|
||||
}
|
||||
@@ -4505,14 +4419,14 @@ fn read_more_sync<R: std::io::Read>(
|
||||
}
|
||||
|
||||
let extra = read_size - has;
|
||||
if buf.capacity() >= read_size {
|
||||
// Extend the buffer if we have enough space.
|
||||
buf.resize(read_size, 0);
|
||||
} else {
|
||||
buf.extend(vec![0u8; extra]);
|
||||
buf.try_reserve(extra.saturating_add(1)).map_err(Error::other)?;
|
||||
std::io::Read::read_to_end(&mut std::io::Read::take(reader, extra as u64), buf)?;
|
||||
if buf.len() < read_size {
|
||||
return Err(Error::from(std::io::Error::new(
|
||||
std::io::ErrorKind::UnexpectedEof,
|
||||
"failed to fill whole buffer",
|
||||
)));
|
||||
}
|
||||
|
||||
reader.read_exact(&mut buf[has..])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -4534,8 +4448,8 @@ pub fn read_xl_meta_no_data_sync<R: std::io::Read>(reader: &mut R, size: usize)
|
||||
has_full = false;
|
||||
}
|
||||
|
||||
let mut buf = vec![0u8; initial];
|
||||
reader.read_exact(&mut buf)?;
|
||||
let mut buf = Vec::new();
|
||||
read_more_sync(reader, &mut buf, size, initial, false)?;
|
||||
|
||||
let (tmp_buf, major, minor) = FileMeta::check_xl2_v1(&buf)?;
|
||||
|
||||
@@ -5102,39 +5016,6 @@ mod tests {
|
||||
wr
|
||||
}
|
||||
|
||||
fn encode_legacy_v2_object_body(data_blocks: usize, parity_blocks: usize) -> Vec<u8> {
|
||||
let drive_count = data_blocks + parity_blocks;
|
||||
let payload = LegacyObjectVersionFixture {
|
||||
version_type: LegacyObjectVersionTypeFixture::Object,
|
||||
object: Some(LegacyObjectFixture {
|
||||
version_id: Some(sample_version_id().as_bytes().to_vec()),
|
||||
data_dir: Some(Uuid::from_u128(42).as_bytes().to_vec()),
|
||||
erasure_algorithm: "ReedSolomon".to_string(),
|
||||
erasure_m: data_blocks,
|
||||
erasure_n: parity_blocks,
|
||||
erasure_block_size: 1_048_576,
|
||||
erasure_index: 1,
|
||||
erasure_dist: (1..=drive_count)
|
||||
.map(|idx| u8::try_from(idx).expect("test drive index should fit u8"))
|
||||
.collect(),
|
||||
bitrot_checksum_algo: "HighwayHash".to_string(),
|
||||
part_numbers: vec![1],
|
||||
part_etags: vec!["etag-1".to_string()],
|
||||
part_sizes: vec![11],
|
||||
part_actual_sizes: vec![11],
|
||||
part_indices: vec![Vec::new()],
|
||||
size: 11,
|
||||
mod_time: Some(sample_mod_time()),
|
||||
meta_sys: HashMap::new(),
|
||||
meta_user: HashMap::from([("content-type".to_string(), "text/plain".to_string())]),
|
||||
}),
|
||||
delete_marker: None,
|
||||
write_version: 3,
|
||||
};
|
||||
|
||||
rmp_serde::to_vec_named(&payload).expect("legacy object payload should marshal")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_header_unmarshal_v1_uses_legacy_layout_defaults() {
|
||||
let expected = sample_header();
|
||||
@@ -5169,80 +5050,6 @@ mod tests {
|
||||
assert_eq!(decoded.ec_m, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shallow_version_write_quorum_uses_legacy_object_payload_when_header_lacks_ec() {
|
||||
let expected = sample_header();
|
||||
let encoded = encode_v2_header(&expected);
|
||||
let mut header = FileMetaVersionHeader::default();
|
||||
header.unmarshal_v(2, &encoded).expect("legacy v2 header should decode");
|
||||
|
||||
let version = FileMetaShallowVersion {
|
||||
header,
|
||||
meta: encode_legacy_v2_object_body(7, 1),
|
||||
};
|
||||
|
||||
assert_eq!(version.header.write_quorum(5), 5);
|
||||
assert_eq!(version.write_quorum(5), 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listing_quorum_uses_each_versions_data_floor() {
|
||||
for (data, parity, offline, fallback, expected) in
|
||||
[(8, 8, 0, 9, 9), (8, 8, 8, 8, 8), (12, 4, 8, 8, 12), (12, 4, 0, 9, 12)]
|
||||
{
|
||||
let version = FileMetaShallowVersion {
|
||||
header: FileMetaVersionHeader {
|
||||
version_type: VersionType::Object,
|
||||
ec_m: data,
|
||||
ec_n: parity,
|
||||
..Default::default()
|
||||
},
|
||||
// Modern headers must not decode the payload to decide quorum.
|
||||
meta: vec![0xff],
|
||||
};
|
||||
assert_eq!(version.listing_quorum(fallback, offline), expected);
|
||||
}
|
||||
let marker = FileMetaShallowVersion {
|
||||
header: FileMetaVersionHeader {
|
||||
version_type: VersionType::Delete,
|
||||
..Default::default()
|
||||
},
|
||||
meta: Vec::new(),
|
||||
};
|
||||
assert_eq!(marker.listing_quorum(9, 0), 9, "healthy marker needs majority metadata");
|
||||
assert_eq!(marker.listing_quorum(8, 8), 8, "degraded marker retains metadata floor");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listing_quorum_preserves_legacy_and_zero_parity_data_floor() {
|
||||
for (data, parity, fallback, expected) in [(12, 4, 8, 12), (8, 8, 8, 8), (4, 0, 2, 4)] {
|
||||
let version = FileMetaShallowVersion {
|
||||
header: FileMetaVersionHeader {
|
||||
version_type: VersionType::Object,
|
||||
..Default::default()
|
||||
},
|
||||
meta: encode_legacy_v2_object_body(data, parity),
|
||||
};
|
||||
assert_eq!(version.listing_quorum(fallback, 8), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shallow_version_write_quorum_uses_zero_parity_object_payload() {
|
||||
let expected = sample_header();
|
||||
let encoded = encode_v2_header(&expected);
|
||||
let mut header = FileMetaVersionHeader::default();
|
||||
header.unmarshal_v(2, &encoded).expect("legacy v2 header should decode");
|
||||
|
||||
let version = FileMetaShallowVersion {
|
||||
header,
|
||||
meta: encode_legacy_v2_object_body(4, 0),
|
||||
};
|
||||
|
||||
assert_eq!(version.header.write_quorum(2), 2);
|
||||
assert_eq!(version.write_quorum(2), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_header_unmarshal_v3_round_trips_current_layout() {
|
||||
let expected = sample_header();
|
||||
|
||||
@@ -230,7 +230,7 @@ impl InlineData {
|
||||
Self::find_in_slice(&self.0, key)
|
||||
}
|
||||
|
||||
pub(crate) fn find_in_slice(raw: &[u8], key: &str) -> Result<Option<Vec<u8>>> {
|
||||
pub(crate) fn find_slice_in_raw<'a>(raw: &'a [u8], key: &str) -> Result<Option<&'a [u8]>> {
|
||||
if raw.is_empty() || raw[0] == 0 || raw[0] > INLINE_DATA_VER {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -243,13 +243,17 @@ impl InlineData {
|
||||
fields_len -= 1;
|
||||
let (field, value) = Self::read_entry_slice(buf, &mut cur, false)?;
|
||||
if field == key {
|
||||
return Ok(Some(value.to_vec()));
|
||||
return Ok(Some(value));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub(crate) fn find_in_slice(raw: &[u8], key: &str) -> Result<Option<Vec<u8>>> {
|
||||
Ok(Self::find_slice_in_raw(raw, key)?.map(<[u8]>::to_vec))
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
Self::validate_slice(&self.0)
|
||||
}
|
||||
|
||||
+118
-745
File diff suppressed because it is too large
Load Diff
@@ -683,14 +683,17 @@ fn create_auto_heal_disk_request(set_disk_id: String, priority: Option<HealChann
|
||||
}
|
||||
}
|
||||
|
||||
fn create_auto_replacement_disk_request(
|
||||
/// Build one automatic replacement request for the verified targets of a set.
|
||||
pub fn create_auto_replacement_disks_request(
|
||||
pool_index: usize,
|
||||
set_index: usize,
|
||||
replacement_endpoint: String,
|
||||
mut replacement_endpoints: Vec<String>,
|
||||
priority: Option<HealChannelPriority>,
|
||||
) -> HealChannelRequest {
|
||||
replacement_endpoints.sort_unstable();
|
||||
replacement_endpoints.dedup();
|
||||
let mut request = create_auto_heal_disk_request(format!("pool_{pool_index}_set_{set_index}"), priority);
|
||||
request.heal_endpoints = vec![replacement_endpoint];
|
||||
request.heal_endpoints = replacement_endpoints;
|
||||
request.pool_index = Some(pool_index);
|
||||
request.set_index = Some(set_index);
|
||||
request
|
||||
@@ -711,10 +714,10 @@ pub async fn send_heal_replacement_disk(
|
||||
replacement_endpoint: String,
|
||||
priority: Option<HealChannelPriority>,
|
||||
) -> Result<(), String> {
|
||||
send_heal_request(create_auto_replacement_disk_request(
|
||||
send_heal_request(create_auto_replacement_disks_request(
|
||||
pool_index,
|
||||
set_index,
|
||||
replacement_endpoint,
|
||||
vec![replacement_endpoint],
|
||||
priority,
|
||||
))
|
||||
.await
|
||||
@@ -726,8 +729,12 @@ mod auto_heal_disk_request_tests {
|
||||
|
||||
#[test]
|
||||
fn replacement_disk_request_carries_its_exact_endpoint() {
|
||||
let request =
|
||||
create_auto_replacement_disk_request(2, 3, "http://node2:9000/drive3".to_string(), Some(HealChannelPriority::Normal));
|
||||
let request = create_auto_replacement_disks_request(
|
||||
2,
|
||||
3,
|
||||
vec!["http://node2:9000/drive3".to_string()],
|
||||
Some(HealChannelPriority::Normal),
|
||||
);
|
||||
|
||||
assert_eq!(request.disk.as_deref(), Some("pool_2_set_3"));
|
||||
assert_eq!(request.heal_endpoints, ["http://node2:9000/drive3"]);
|
||||
@@ -736,6 +743,23 @@ mod auto_heal_disk_request_tests {
|
||||
assert_eq!(request.source, HealRequestSource::AutoHeal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replacement_disk_request_keeps_every_distinct_target() {
|
||||
let request = create_auto_replacement_disks_request(
|
||||
2,
|
||||
3,
|
||||
vec!["drive-b".to_string(), "drive-a".to_string(), "drive-b".to_string()],
|
||||
Some(HealChannelPriority::Normal),
|
||||
);
|
||||
|
||||
assert_eq!(request.disk.as_deref(), Some("pool_2_set_3"));
|
||||
assert_eq!(request.heal_endpoints, ["drive-a", "drive-b"]);
|
||||
assert_eq!(request.pool_index, Some(2));
|
||||
assert_eq!(request.set_index, Some(3));
|
||||
assert_eq!(request.priority, HealChannelPriority::Normal);
|
||||
assert_eq!(request.source, HealRequestSource::AutoHeal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_auto_heal_disk_request_has_no_replacement_endpoint() {
|
||||
let request = create_auto_heal_disk_request("pool_2_set_3".to_string(), None);
|
||||
|
||||
@@ -825,6 +825,21 @@ fn admin_heal_options_compatible(left: &HealOptions, right: &HealOptions) -> boo
|
||||
&& left.no_lock == right.no_lock
|
||||
}
|
||||
|
||||
fn replacement_request_compatible(
|
||||
request: &HealRequest,
|
||||
owner_type: &HealType,
|
||||
owner_options: &HealOptions,
|
||||
owner_source: HealRequestSource,
|
||||
) -> bool {
|
||||
request.source == HealRequestSource::AutoHeal
|
||||
&& owner_source == HealRequestSource::AutoHeal
|
||||
&& matches!(request.heal_type, HealType::ErasureSet { .. })
|
||||
&& request.heal_type == *owner_type
|
||||
&& request.options.pool_index == owner_options.pool_index
|
||||
&& request.options.set_index == owner_options.set_index
|
||||
&& admin_heal_options_compatible(&request.options, owner_options)
|
||||
}
|
||||
|
||||
fn admin_overlap_rejection(
|
||||
request: &HealRequest,
|
||||
owner_type: &HealType,
|
||||
@@ -1276,7 +1291,9 @@ impl HealManager {
|
||||
request: HealRequest,
|
||||
config: &HealConfig,
|
||||
context: &'static str,
|
||||
replacement_targets_frozen: bool,
|
||||
) -> HealAdmissionDecision {
|
||||
let replacement_targets_frozen = replacement_targets_frozen || request.retry_attempts != 0;
|
||||
let queue_len = queue.len();
|
||||
publish_heal_queue_length(queue);
|
||||
let queue_capacity = config.queue_size;
|
||||
@@ -1284,6 +1301,25 @@ impl HealManager {
|
||||
let best_effort_reserve = best_effort_queue_reserve(queue_capacity);
|
||||
let best_effort_source_reserves = best_effort_source_reserves(queue_capacity);
|
||||
|
||||
// Scanner and retry admission bypass submit_heal_request's duplicate
|
||||
// checks. Keep their target ownership checks ahead of queue capacity.
|
||||
if !request.force_start
|
||||
&& request.source == HealRequestSource::AutoHeal
|
||||
&& !request.heal_endpoints.is_empty()
|
||||
&& let Some(owner_id) = queue
|
||||
.queued_request_id_for_dedup_key(&PriorityHealQueue::make_dedup_key(&request))
|
||||
.map(ToOwned::to_owned)
|
||||
{
|
||||
let mut admission = Self::duplicate_admission_for_request(&request, config);
|
||||
if admission == HealAdmissionResult::Merged
|
||||
&& !queue.merge_replacement_targets(&owner_id, &request, true, replacement_targets_frozen)
|
||||
{
|
||||
admission = HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning);
|
||||
}
|
||||
Self::record_admission_metric(request.source, admission, context);
|
||||
return HealAdmissionDecision::new(admission);
|
||||
}
|
||||
|
||||
if queue_len >= queue_capacity && !request.force_start {
|
||||
if Self::can_displace_queued_work(&request)
|
||||
&& queue.can_displace_lower_priority_where(request.priority, Self::queued_request_can_be_displaced)
|
||||
@@ -1443,7 +1479,7 @@ impl HealManager {
|
||||
let priority = request.priority;
|
||||
let source = request.source;
|
||||
|
||||
match queue.push(request) {
|
||||
match queue.push_with_replacement_scope(request, replacement_targets_frozen) {
|
||||
QueuePushOutcome::Accepted => {
|
||||
publish_heal_queue_length(queue);
|
||||
Self::record_admission_metric(source, HealAdmissionResult::Accepted, context);
|
||||
@@ -2260,7 +2296,7 @@ impl HealManager {
|
||||
// HS-06: under the minio_error overlap policy an exact duplicate
|
||||
// admin start reports the typed AlreadyRunning rejection instead
|
||||
// of the silent merge (MinIO's ErrHealAlreadyRunning).
|
||||
let admission = if (request.source == HealRequestSource::Admin
|
||||
let mut admission = if (request.source == HealRequestSource::Admin
|
||||
&& config.overlap_policy == HealOverlapPolicy::MinioError)
|
||||
|| (duplicate_state != "queued" && mrf_notice_target.as_ref().is_some_and(|target| target.kind.is_durable()))
|
||||
{
|
||||
@@ -2270,6 +2306,44 @@ impl HealManager {
|
||||
} else {
|
||||
Self::duplicate_admission_for_request(&request, &config)
|
||||
};
|
||||
if matches!(admission, HealAdmissionResult::Merged)
|
||||
&& request.source == HealRequestSource::AutoHeal
|
||||
&& !request.heal_endpoints.is_empty()
|
||||
{
|
||||
let (owner_recovered, incoming_recovered) = {
|
||||
// Match queue -> recovery-anchor ordering in the scanner.
|
||||
let anchors = self
|
||||
.replacement_recovery_anchors
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
(anchors.contains_key(&merged_task_id), anchors.contains_key(&request.id))
|
||||
};
|
||||
let incoming_frozen = incoming_recovered || request.retry_attempts != 0;
|
||||
let covered = if incoming_frozen && request.id != merged_task_id {
|
||||
false
|
||||
} else if duplicate_state == "queued" {
|
||||
queue.merge_replacement_targets(&merged_task_id, &request, !owner_recovered, incoming_frozen)
|
||||
} else {
|
||||
let owner = active_heals
|
||||
.get(&merged_task_id)
|
||||
.map(|task| (&task.heal_type, &task.options, task.source, &task.heal_endpoints))
|
||||
.or_else(|| {
|
||||
retrying_heals.get(&merged_task_id).map(|retry| {
|
||||
let owner = &retry.request;
|
||||
(&owner.heal_type, &owner.options, owner.source, &owner.heal_endpoints)
|
||||
})
|
||||
});
|
||||
owner.is_some_and(|(heal_type, options, source, endpoints)| {
|
||||
replacement_request_compatible(&request, heal_type, options, source)
|
||||
&& request.heal_endpoints.iter().all(|endpoint| endpoints.contains(endpoint))
|
||||
})
|
||||
};
|
||||
if !covered {
|
||||
// A frozen generation cannot certify newly arriving disks.
|
||||
// Leave them discoverable by the scanner for the next pass.
|
||||
admission = HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning);
|
||||
}
|
||||
}
|
||||
if matches!(admission, HealAdmissionResult::Merged)
|
||||
&& let Some(target) = mrf_notice_target
|
||||
{
|
||||
@@ -2426,7 +2500,14 @@ impl HealManager {
|
||||
let request_id = request.id.clone();
|
||||
let request_heal_type = request.heal_type.clone();
|
||||
let request_source = request.source;
|
||||
let admission_decision = Self::admit_request_to_queue(&mut queue, request, &config, "submit");
|
||||
let recovered_replacement = request.source == HealRequestSource::AutoHeal
|
||||
&& !request.heal_endpoints.is_empty()
|
||||
&& self
|
||||
.replacement_recovery_anchors
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.contains_key(&request.id);
|
||||
let admission_decision = Self::admit_request_to_queue(&mut queue, request, &config, "submit", recovered_replacement);
|
||||
let admission = admission_decision.result;
|
||||
if durable_handoff_required && !admission.is_admitted() {
|
||||
self.root_recovery
|
||||
|
||||
@@ -682,6 +682,7 @@ impl HealManager {
|
||||
HealPriority::Low,
|
||||
);
|
||||
let recovery_anchor = recovery.as_ref().map(|(_, _, _, anchor)| anchor.clone());
|
||||
let recovered_replacement = recovery.is_some();
|
||||
if let Some((task_id, _, _, _)) = recovery {
|
||||
req.id = task_id;
|
||||
}
|
||||
@@ -691,7 +692,8 @@ impl HealManager {
|
||||
let endpoint_count = req.heal_endpoints.len();
|
||||
let config = config.read().await;
|
||||
let mut queue = heal_queue.lock().await;
|
||||
let admission_decision = Self::admit_request_to_queue(&mut queue, req, &config, "auto_scan");
|
||||
let admission_decision =
|
||||
Self::admit_request_to_queue(&mut queue, req, &config, "auto_scan", recovered_replacement);
|
||||
let admission = admission_decision.result;
|
||||
let should_notify =
|
||||
matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable;
|
||||
|
||||
@@ -70,6 +70,8 @@ pub(super) struct PriorityQueueItem {
|
||||
pub(super) sequence: u64,
|
||||
pub(super) dedup_key: String,
|
||||
pub(super) request: HealRequest,
|
||||
/// Execution or durable recovery fixed these targets; grace waits can have zero retries.
|
||||
pub(super) replacement_targets_frozen: bool,
|
||||
}
|
||||
|
||||
impl Eq for PriorityQueueItem {}
|
||||
@@ -316,6 +318,10 @@ impl PriorityHealQueue {
|
||||
}
|
||||
|
||||
pub(super) fn push(&mut self, request: HealRequest) -> QueuePushOutcome {
|
||||
self.push_with_replacement_scope(request, false)
|
||||
}
|
||||
|
||||
pub(super) fn push_with_replacement_scope(&mut self, request: HealRequest, frozen: bool) -> QueuePushOutcome {
|
||||
let key = Self::make_dedup_key(&request);
|
||||
|
||||
// Check for duplicates unless the caller explicitly forces admission.
|
||||
@@ -340,6 +346,7 @@ impl PriorityHealQueue {
|
||||
sequence: self.sequence,
|
||||
dedup_key: key,
|
||||
request,
|
||||
replacement_targets_frozen: frozen,
|
||||
});
|
||||
if is_best_effort_source(source) {
|
||||
self.best_effort_queued += 1;
|
||||
@@ -680,6 +687,51 @@ impl PriorityHealQueue {
|
||||
self.heap.iter().map(|item| &item.request)
|
||||
}
|
||||
|
||||
pub(super) fn merge_replacement_targets(
|
||||
&mut self,
|
||||
owner_id: &str,
|
||||
request: &HealRequest,
|
||||
allow_expand: bool,
|
||||
incoming_frozen: bool,
|
||||
) -> bool {
|
||||
let Some(item) = self.heap.iter().find(|item| item.request.id == owner_id) else {
|
||||
return false;
|
||||
};
|
||||
let owner = &item.request;
|
||||
let same_id = request.id == owner.id;
|
||||
let eligible = replacement_request_compatible(request, &owner.heal_type, &owner.options, owner.source)
|
||||
&& (!incoming_frozen || same_id);
|
||||
let covered = eligible
|
||||
&& request
|
||||
.heal_endpoints
|
||||
.iter()
|
||||
.all(|endpoint| owner.heal_endpoints.contains(endpoint));
|
||||
let mark_frozen = incoming_frozen && same_id && !item.replacement_targets_frozen;
|
||||
let can_expand = eligible
|
||||
&& allow_expand
|
||||
&& !item.replacement_targets_frozen
|
||||
&& !incoming_frozen
|
||||
&& owner.retry_attempts == 0
|
||||
&& request.retry_attempts == 0
|
||||
&& !owner.heal_endpoints.is_empty();
|
||||
if !mark_frozen && (covered || !can_expand) {
|
||||
return covered;
|
||||
}
|
||||
|
||||
// Neither targets nor their frozen flag affect heap ordering.
|
||||
let mut items = std::mem::take(&mut self.heap).into_vec();
|
||||
if let Some(item) = items.iter_mut().find(|item| item.request.id == owner_id) {
|
||||
item.replacement_targets_frozen |= mark_frozen;
|
||||
if !covered && can_expand {
|
||||
item.request.heal_endpoints.extend(request.heal_endpoints.iter().cloned());
|
||||
item.request.heal_endpoints.sort_unstable();
|
||||
item.request.heal_endpoints.dedup();
|
||||
}
|
||||
}
|
||||
self.heap = BinaryHeap::from(items);
|
||||
covered || can_expand
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn contains_request_id(&self, request_id: &str) -> bool {
|
||||
self.heap.iter().any(|item| item.request.id == request_id)
|
||||
|
||||
@@ -582,7 +582,7 @@ impl HealManager {
|
||||
return;
|
||||
}
|
||||
let admission_decision =
|
||||
Self::admit_request_to_queue(&mut queue, retry_request.clone(), &retry_config, "retry");
|
||||
Self::admit_request_to_queue(&mut queue, retry_request.clone(), &retry_config, "retry", true);
|
||||
let admission = admission_decision.result;
|
||||
let should_notify = matches!(admission, HealAdmissionResult::Accepted)
|
||||
&& retry_config.event_driven_scheduler_enable;
|
||||
|
||||
@@ -2505,6 +2505,301 @@ fn test_heal_request_and_task_metric_labels_match() {
|
||||
assert_eq!(task.metric_set_label(), "pool_0_set_1");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_replacement_queued_targets_merge_without_changing_fifo_or_token() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let owner = automatic_replacement_request(&["replacement-a"]);
|
||||
let owner_id = owner.id.clone();
|
||||
let created_at = owner.created_at;
|
||||
let first = manager
|
||||
.submit_heal_request_with_receipt(owner)
|
||||
.await
|
||||
.expect("queue first target");
|
||||
assert_eq!(first.result, HealAdmissionResult::Accepted);
|
||||
|
||||
let other = HealRequest::object("bucket".to_string(), "unrelated".to_string(), None);
|
||||
let other_id = other.id.clone();
|
||||
manager.submit_heal_request(other).await.expect("queue unrelated work");
|
||||
let merged = manager
|
||||
.submit_heal_request_with_receipt(automatic_replacement_request(&["replacement-b", "replacement-a"]))
|
||||
.await
|
||||
.expect("merge all fresh targets");
|
||||
assert_eq!(merged.result, HealAdmissionResult::Merged);
|
||||
assert_eq!(merged.task_id, owner_id);
|
||||
|
||||
let mut queue = manager.heal_queue.lock().await;
|
||||
assert_eq!(queue.len(), 2);
|
||||
let owner = queue.pop_next().expect("original task remains first");
|
||||
assert_eq!(owner.id, owner_id);
|
||||
assert_eq!(owner.created_at, created_at);
|
||||
assert_eq!(owner.heal_endpoints, ["replacement-a", "replacement-b"]);
|
||||
assert_eq!(queue.pop_next().expect("unrelated task remains queued").id, other_id);
|
||||
}
|
||||
|
||||
fn automatic_replacement_request(targets: &[&str]) -> HealRequest {
|
||||
let mut request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: Vec::new(),
|
||||
set_disk_id: "pool_0_set_1".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
pool_index: Some(0),
|
||||
set_index: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
request.source = HealRequestSource::AutoHeal;
|
||||
request.heal_endpoints = targets.iter().map(|target| (*target).to_string()).collect();
|
||||
request
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_replacement_frozen_targets_reject_late_disks_but_cover_duplicates() {
|
||||
for state in ["active", "retrying", "grace", "recovered", "recovery_anchor"] {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None);
|
||||
let mut owner = automatic_replacement_request(&["replacement-a"]);
|
||||
let owner_id = owner.id.clone();
|
||||
if state == "grace" {
|
||||
let now = SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("test clock")
|
||||
.as_secs();
|
||||
let task = HealTask::from_request(owner, storage.clone());
|
||||
owner = retry_request_for_result_with_budget(
|
||||
&task,
|
||||
&Err(Error::DanglingDeleteDeferred {
|
||||
retry_not_before: now + 60,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("grace wait keeps the generation replayable")
|
||||
.0;
|
||||
assert_eq!(owner.retry_attempts, 0, "grace must not consume a failure retry");
|
||||
}
|
||||
match state {
|
||||
"active" => {
|
||||
manager
|
||||
.active_heals
|
||||
.lock()
|
||||
.await
|
||||
.insert(owner_id.clone(), Arc::new(HealTask::from_request(owner, storage.clone())));
|
||||
}
|
||||
"retrying" => {
|
||||
manager.retrying_heals.lock().await.insert(
|
||||
owner_id.clone(),
|
||||
RetryingHeal {
|
||||
request: owner,
|
||||
error: "target unavailable".to_string(),
|
||||
cancel_token: CancellationToken::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
"grace" | "recovered" => {
|
||||
let config = manager.config.read().await.clone();
|
||||
let mut queue = manager.heal_queue.lock().await;
|
||||
let context = if state == "grace" { "retry" } else { "auto_scan" };
|
||||
assert_eq!(
|
||||
HealManager::admit_request_to_queue(&mut queue, owner, &config, context, true).result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert!(
|
||||
queue
|
||||
.heap
|
||||
.iter()
|
||||
.any(|item| item.request.id == owner_id && item.replacement_targets_frozen),
|
||||
"zero retries cannot mean the scope is fresh"
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
manager.submit_heal_request(owner).await.expect("queue frozen generation");
|
||||
if state == "recovery_anchor" {
|
||||
manager
|
||||
.replacement_recovery_anchors
|
||||
.lock()
|
||||
.expect("recovery anchors")
|
||||
.insert(owner_id.clone(), "survivor".to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let late = manager
|
||||
.submit_heal_request(automatic_replacement_request(&["replacement-b"]))
|
||||
.await
|
||||
.expect("classify late target");
|
||||
assert_eq!(late, HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning), "{state}");
|
||||
let covered = manager
|
||||
.submit_heal_request_with_receipt(automatic_replacement_request(&["replacement-a"]))
|
||||
.await
|
||||
.expect("covered target remains owned");
|
||||
assert_eq!(covered.result, HealAdmissionResult::Merged, "{state}");
|
||||
assert_eq!(covered.task_id, owner_id, "{state}");
|
||||
let targets = match state {
|
||||
"active" => manager.active_heals.lock().await[&owner_id].heal_endpoints.clone(),
|
||||
"retrying" => manager.retrying_heals.lock().await[&owner_id].request.heal_endpoints.clone(),
|
||||
_ => {
|
||||
manager
|
||||
.heal_queue
|
||||
.lock()
|
||||
.await
|
||||
.pop_next()
|
||||
.expect("retained generation")
|
||||
.heal_endpoints
|
||||
}
|
||||
};
|
||||
assert_eq!(targets, ["replacement-a"], "{state}: frozen scope must not grow");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_replacement_merge_preserves_options_policy_and_recovery_identity() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
manager
|
||||
.submit_heal_request(automatic_replacement_request(&["replacement-a"]))
|
||||
.await
|
||||
.expect("queue fresh generation");
|
||||
for variant in [
|
||||
"dry_run",
|
||||
"bucket_scope",
|
||||
"pool_scope",
|
||||
"retried_incoming",
|
||||
"recovery_anchor",
|
||||
"merge_disabled",
|
||||
] {
|
||||
let mut incoming = automatic_replacement_request(&["replacement-b"]);
|
||||
match variant {
|
||||
"dry_run" => incoming.options.dry_run = true,
|
||||
"bucket_scope" => {
|
||||
if let HealType::ErasureSet { buckets, .. } = &mut incoming.heal_type {
|
||||
buckets.push("other-bucket".to_string());
|
||||
}
|
||||
}
|
||||
"pool_scope" => incoming.options.pool_index = Some(2),
|
||||
"retried_incoming" => {
|
||||
incoming.heal_endpoints = vec!["replacement-a".to_string()];
|
||||
incoming.retry_attempts = 1;
|
||||
}
|
||||
"recovery_anchor" => {
|
||||
incoming.heal_endpoints = vec!["replacement-a".to_string()];
|
||||
manager
|
||||
.replacement_recovery_anchors
|
||||
.lock()
|
||||
.expect("recovery anchors")
|
||||
.insert(incoming.id.clone(), "survivor".to_string());
|
||||
}
|
||||
"merge_disabled" => {
|
||||
manager.config.write().await.low_priority_merge_enable = false;
|
||||
incoming.priority = HealPriority::Low;
|
||||
}
|
||||
_ => unreachable!("test variant"),
|
||||
}
|
||||
let result = manager
|
||||
.submit_heal_request(incoming)
|
||||
.await
|
||||
.expect("classify incompatible generation");
|
||||
let reason = if variant == "merge_disabled" {
|
||||
HealAdmissionDropReason::PolicyDropped
|
||||
} else {
|
||||
HealAdmissionDropReason::AlreadyRunning
|
||||
};
|
||||
assert_eq!(result, HealAdmissionResult::Dropped(reason), "{variant}");
|
||||
let queue = manager.heal_queue.lock().await;
|
||||
assert_eq!(queue.len(), 1, "{variant}");
|
||||
assert_eq!(
|
||||
queue.requests().next().expect("original generation").heal_endpoints,
|
||||
["replacement-a"],
|
||||
"{variant}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn automatic_replacement_direct_admission_checks_frozen_targets_before_capacity() {
|
||||
let config = HealConfig {
|
||||
queue_size: 1,
|
||||
..HealConfig::default()
|
||||
};
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
let owner = automatic_replacement_request(&["replacement-a"]);
|
||||
let owner_id = owner.id.clone();
|
||||
assert_eq!(queue.push(owner), QueuePushOutcome::Accepted);
|
||||
let admit = |queue: &mut PriorityHealQueue, request, frozen| {
|
||||
HealManager::admit_request_to_queue(queue, request, &config, "auto_scan", frozen).result
|
||||
};
|
||||
assert_eq!(
|
||||
admit(&mut queue, automatic_replacement_request(&["replacement-b"]), false),
|
||||
HealAdmissionResult::Merged
|
||||
);
|
||||
let mut replay = automatic_replacement_request(&["replacement-a"]);
|
||||
replay.id = owner_id.clone();
|
||||
assert_eq!(admit(&mut queue, replay, true), HealAdmissionResult::Merged);
|
||||
assert!(
|
||||
queue
|
||||
.heap
|
||||
.iter()
|
||||
.any(|item| item.request.id == owner_id && item.replacement_targets_frozen)
|
||||
);
|
||||
assert_eq!(
|
||||
admit(&mut queue, automatic_replacement_request(&["replacement-c"]), false),
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
);
|
||||
assert_eq!(
|
||||
admit(&mut queue, automatic_replacement_request(&["replacement-a"]), false),
|
||||
HealAdmissionResult::Merged
|
||||
);
|
||||
assert_eq!(
|
||||
admit(&mut queue, automatic_replacement_request(&["replacement-a"]), true),
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
);
|
||||
assert_eq!(queue.len(), 1);
|
||||
let owner = queue.pop_next().expect("frozen owner retained");
|
||||
assert_eq!(owner.id, owner_id);
|
||||
assert_eq!(owner.heal_endpoints, ["replacement-a", "replacement-b"]);
|
||||
|
||||
let owner = automatic_replacement_request(&["replacement-a"]);
|
||||
let owner_id = owner.id.clone();
|
||||
assert_eq!(queue.push(owner), QueuePushOutcome::Accepted);
|
||||
let mut conflicting_replay = automatic_replacement_request(&["replacement-b"]);
|
||||
conflicting_replay.id = owner_id.clone();
|
||||
assert_eq!(
|
||||
admit(&mut queue, conflicting_replay, true),
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
);
|
||||
assert!(
|
||||
queue
|
||||
.heap
|
||||
.iter()
|
||||
.any(|item| item.request.id == owner_id && item.replacement_targets_frozen)
|
||||
);
|
||||
assert_eq!(
|
||||
admit(&mut queue, automatic_replacement_request(&["replacement-c"]), false),
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
);
|
||||
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
let owner = automatic_replacement_request(&["replacement-a"]);
|
||||
let owner_id = owner.id.clone();
|
||||
assert_eq!(queue.push(owner), QueuePushOutcome::Accepted);
|
||||
let mut incompatible_replay = automatic_replacement_request(&["replacement-a"]);
|
||||
incompatible_replay.id = owner_id.clone();
|
||||
incompatible_replay.options.dry_run = true;
|
||||
assert_eq!(
|
||||
admit(&mut queue, incompatible_replay, true),
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
);
|
||||
assert!(
|
||||
queue
|
||||
.heap
|
||||
.iter()
|
||||
.any(|item| item.request.id == owner_id && item.replacement_targets_frozen)
|
||||
);
|
||||
assert_eq!(
|
||||
admit(&mut queue, automatic_replacement_request(&["replacement-b"]), false),
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_returns_merged_for_duplicate() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
|
||||
@@ -258,6 +258,16 @@ impl ScannerFixture {
|
||||
assert_eq!(request.id, self.task_id, "scanner must replay the durable UUID");
|
||||
assert_eq!(request.heal_endpoints, vec![self.local_targets[0].clone()]);
|
||||
assert_eq!(request.source, HealRequestSource::AutoHeal);
|
||||
assert!(
|
||||
self.manager
|
||||
.heal_queue
|
||||
.lock()
|
||||
.await
|
||||
.heap
|
||||
.iter()
|
||||
.any(|item| item.request.id == request.id && item.replacement_targets_frozen),
|
||||
"replayed durable targets cannot expand"
|
||||
);
|
||||
assert!(matches!(
|
||||
request.heal_type,
|
||||
HealType::ErasureSet { set_disk_id, buckets }
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use rustfs_heal::{HealOptions, HealPriority, HealRequest, HealType};
|
||||
use rustfs_heal_contracts::heal_channel::HealRequestSource;
|
||||
use std::time::SystemTime;
|
||||
|
||||
#[test]
|
||||
fn existing_heal_request_literal_remains_constructible() {
|
||||
let now = SystemTime::now();
|
||||
let request = HealRequest {
|
||||
id: "request".to_string(),
|
||||
heal_type: HealType::ErasureSet {
|
||||
set_disk_id: "pool_0_set_0".to_string(),
|
||||
buckets: Vec::new(),
|
||||
},
|
||||
bucket_incarnation_id: None,
|
||||
expected_mrf_bucket_incarnation_id: None,
|
||||
options: HealOptions::default(),
|
||||
priority: HealPriority::Normal,
|
||||
source: HealRequestSource::AutoHeal,
|
||||
force_start: false,
|
||||
retry_attempts: 0,
|
||||
heal_endpoints: vec!["drive-a".to_string()],
|
||||
created_at: now,
|
||||
enqueued_at: now,
|
||||
};
|
||||
assert_eq!(request.heal_endpoints, ["drive-a"]);
|
||||
}
|
||||
@@ -26,6 +26,9 @@ use crate::{
|
||||
|
||||
/// Default shard count for guard storage (must be power of 2)
|
||||
const DEFAULT_GUARD_SHARD_COUNT: usize = 64;
|
||||
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
|
||||
const LOG_COMPONENT_LOCK: &str = "lock";
|
||||
const LOG_SUBSYSTEM_NAMESPACE: &str = "namespace";
|
||||
|
||||
type GuardShard = Arc<RwLock<HashMap<LockId, LocalGuardEntry>>>;
|
||||
type GuardStorage = Arc<Vec<GuardShard>>;
|
||||
@@ -333,7 +336,22 @@ impl LockClient for LocalClient {
|
||||
};
|
||||
if let Some(guard) = guard {
|
||||
// Guard automatically releases the lock when dropped
|
||||
let physical_guard_id = (!guard.guard.is_disabled()).then(|| guard.guard.guard_id());
|
||||
drop(guard.guard);
|
||||
if let Some(physical_guard_id) = physical_guard_id {
|
||||
tracing::trace!(
|
||||
target: "rustfs_lock::namespace_owner",
|
||||
event = EVENT_NAMESPACE_LOCK_RELEASE,
|
||||
component = LOG_COMPONENT_LOCK,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE,
|
||||
state = "mapped",
|
||||
peer_kind = "local",
|
||||
lock_id = %lock_id,
|
||||
process_id = std::process::id(),
|
||||
physical_guard_id,
|
||||
"Namespace peer lock mapped to physical guard"
|
||||
);
|
||||
}
|
||||
Ok(true)
|
||||
} else {
|
||||
// Lock not found or already released
|
||||
|
||||
@@ -412,6 +412,11 @@ impl DistributedLockGuard {
|
||||
&self.lock_id
|
||||
}
|
||||
|
||||
/// Actual granted peer identities, which may differ from the logical guard ID.
|
||||
pub fn granted_lock_ids(&self) -> impl Iterator<Item = &LockId> {
|
||||
self.entries.iter().map(|entry| &entry.lock_id)
|
||||
}
|
||||
|
||||
/// Whether the guard's heartbeat has observed a refresh-quorum loss.
|
||||
pub fn is_lock_lost(&self) -> bool {
|
||||
self.lock_lost.is_lost()
|
||||
|
||||
@@ -24,6 +24,9 @@ use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
/// Global counter for guard IDs to prevent double-release
|
||||
static GUARD_ID_COUNTER: AtomicU64 = AtomicU64::new(1);
|
||||
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
|
||||
const LOG_COMPONENT_LOCK: &str = "lock";
|
||||
const LOG_SUBSYSTEM_NAMESPACE: &str = "namespace";
|
||||
|
||||
/// RAII guard for fast object locks
|
||||
///
|
||||
@@ -109,6 +112,7 @@ impl FastLockGuard {
|
||||
record_lock_held_release(self.mode);
|
||||
// Unregister the guard after successful release
|
||||
shard.unregister_guard(self.guard_id);
|
||||
self.record_physical_release();
|
||||
}
|
||||
success
|
||||
} else {
|
||||
@@ -118,6 +122,22 @@ impl FastLockGuard {
|
||||
}
|
||||
}
|
||||
|
||||
fn record_physical_release(&self) {
|
||||
tracing::trace!(
|
||||
target: "rustfs_lock::namespace_owner",
|
||||
event = EVENT_NAMESPACE_LOCK_RELEASE,
|
||||
component = LOG_COMPONENT_LOCK,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE,
|
||||
state = "released",
|
||||
peer_kind = "local",
|
||||
process_id = std::process::id(),
|
||||
physical_guard_id = self.guard_id,
|
||||
resource = %self.key,
|
||||
owner = %self.owner,
|
||||
"Namespace physical guard release completed"
|
||||
);
|
||||
}
|
||||
|
||||
/// Check if the lock has been released
|
||||
pub fn is_released(&self) -> bool {
|
||||
self.released
|
||||
@@ -168,6 +188,9 @@ impl Drop for FastLockGuard {
|
||||
}
|
||||
// Always unregister the guard to prevent leaks, regardless of release success
|
||||
shard.unregister_guard(self.guard_id);
|
||||
if success {
|
||||
self.record_physical_release();
|
||||
}
|
||||
} else {
|
||||
// If guard was already released or disabled, just unregister it
|
||||
shard.unregister_guard(self.guard_id);
|
||||
|
||||
@@ -139,15 +139,9 @@ impl KeyName {
|
||||
Self::COMMON_KEYS.iter().filter(|k| k.is_server_derived()).map(|k| k.name())
|
||||
}
|
||||
|
||||
/// Returns the policy variable form of the key, e.g. `${jwt:sub}`.
|
||||
pub fn var_name(&self) -> String {
|
||||
match self {
|
||||
KeyName::Aws(s) => format!("${{aws:{}}}", Into::<&str>::into(s)),
|
||||
KeyName::Jwt(s) => format!("${{jwt:{}}}", Into::<&str>::into(s)),
|
||||
KeyName::Ldap(s) => format!("${{ldap:{}}}", Into::<&str>::into(s)),
|
||||
KeyName::Sts(s) => format!("${{sts:{}}}", Into::<&str>::into(s)),
|
||||
KeyName::Svc(s) => format!("${{svc:{}}}", Into::<&str>::into(s)),
|
||||
KeyName::S3(s) => format!("${{s3:{}}}", Into::<&str>::into(s)),
|
||||
}
|
||||
format!("${{{}}}", Into::<&str>::into(self))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -414,6 +408,14 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test_case(KeyName::Aws(AwsKeyName::AWSSourceIP), "${aws:SourceIp}")]
|
||||
#[test_case(KeyName::Jwt(JwtKeyName::JWTPrefUsername), "${jwt:preferred_username}")]
|
||||
#[test_case(KeyName::Ldap(LdapKeyName::Username), "${ldap:username}")]
|
||||
#[test_case(KeyName::S3(S3KeyName::S3LocationConstraint), "${s3:LocationConstraint}")]
|
||||
fn var_name_matches_policy_variable_syntax(key: KeyName, expected: &str) {
|
||||
assert_eq!(key.var_name(), expected);
|
||||
}
|
||||
|
||||
#[test_case("s3:x-amz-copy-source", KeyName::S3(S3KeyName::S3XAmzCopySource))]
|
||||
#[test_case("s3:VersionId", KeyName::S3(S3KeyName::S3VersionId) ; "aws_version_id")]
|
||||
#[test_case("s3:versionid", KeyName::S3(S3KeyName::S3VersionId) ; "minio_version_id")]
|
||||
|
||||
@@ -73,6 +73,20 @@ impl StringFunc {
|
||||
}
|
||||
}
|
||||
|
||||
/// Replaces every `${...}` reference to a common condition key that has a
|
||||
/// non-empty value in the request, not only the first such key.
|
||||
fn substitute_common_keys(mut c: String, values: &HashMap<String, Vec<String>>) -> String {
|
||||
for key in KeyName::COMMON_KEYS {
|
||||
if let Some(v) = values.get(key.name()).and_then(|x| x.first())
|
||||
&& !v.is_empty()
|
||||
{
|
||||
c = c.replace(&key.var_name(), v);
|
||||
}
|
||||
}
|
||||
|
||||
c
|
||||
}
|
||||
|
||||
impl FuncKeyValue<StringFuncValue> {
|
||||
async fn eval(
|
||||
&self,
|
||||
@@ -110,17 +124,7 @@ impl FuncKeyValue<StringFuncValue> {
|
||||
let fvalues = resolved_values
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|resolved_c| {
|
||||
let mut c = Cow::from(resolved_c);
|
||||
for key in KeyName::COMMON_KEYS {
|
||||
match values.get(key.name()).and_then(|x| x.first()) {
|
||||
Some(v) if !v.is_empty() => return Cow::Owned(c.to_mut().replace(&key.var_name(), v)),
|
||||
_ => continue,
|
||||
};
|
||||
}
|
||||
|
||||
c
|
||||
})
|
||||
.map(|resolved_c| Cow::from(substitute_common_keys(resolved_c, values)))
|
||||
.map(|x| if ignore_case { Cow::Owned(x.to_lowercase()) } else { x })
|
||||
.collect::<Set<_>>();
|
||||
|
||||
@@ -159,17 +163,7 @@ impl FuncKeyValue<StringFuncValue> {
|
||||
let matched = resolved_values
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|resolved_c| {
|
||||
let mut c = Cow::from(resolved_c);
|
||||
for key in KeyName::COMMON_KEYS {
|
||||
match values.get(key.name()).and_then(|x| x.first()) {
|
||||
Some(v) if !v.is_empty() => return Cow::Owned(c.to_mut().replace(&key.var_name(), v)),
|
||||
_ => continue,
|
||||
};
|
||||
}
|
||||
|
||||
c
|
||||
})
|
||||
.map(|resolved_c| substitute_common_keys(resolved_c, values))
|
||||
.any(|x| wildcard::is_match(x, v));
|
||||
|
||||
// Unqualified evaluation reports the plain aggregate match and lets the
|
||||
@@ -389,6 +383,44 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
// Real requests always carry earlier common keys such as `signatureversion`
|
||||
// and `SourceIp`; substitution must still reach later keys like `jwt:*`.
|
||||
#[test]
|
||||
fn test_condition_values_substitute_variables_after_other_common_keys() {
|
||||
let request = || {
|
||||
vec![
|
||||
("signatureversion", vec!["AWS4-HMAC-SHA256"]),
|
||||
("SourceIp", vec!["192.0.2.10"]),
|
||||
("preferred_username", vec!["alice"]),
|
||||
("prefix", vec!["alice/photos/"]),
|
||||
]
|
||||
};
|
||||
|
||||
assert!(test_eval_like(
|
||||
new_fkv("s3:prefix", vec!["${jwt:preferred_username}/*"]),
|
||||
false,
|
||||
false,
|
||||
request()
|
||||
));
|
||||
assert!(test_eval(
|
||||
new_fkv("s3:prefix", vec!["${jwt:preferred_username}/photos/"]),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
request()
|
||||
));
|
||||
assert!(!test_eval_like(
|
||||
new_fkv("s3:prefix", vec!["${jwt:preferred_username}/*"]),
|
||||
false,
|
||||
false,
|
||||
vec![
|
||||
("signatureversion", vec!["AWS4-HMAC-SHA256"]),
|
||||
("preferred_username", vec!["bob"]),
|
||||
("prefix", vec!["alice/photos/"]),
|
||||
]
|
||||
));
|
||||
}
|
||||
|
||||
#[test_case(new_fkv("s3:x-amz-copy-source", vec!["mybucket/myobject"]), false, vec![("x-amz-copy-source", vec!["mybucket/myobject"])] => true ; "1")]
|
||||
#[test_case(new_fkv("s3:x-amz-copy-source", vec!["mybucket/myobject"]), false, vec![("x-amz-copy-source", vec!["yourbucket/myobject"])] => false ; "2")]
|
||||
#[test_case(new_fkv("s3:x-amz-copy-source", vec!["mybucket/myobject"]), false, vec![] => false ; "3")]
|
||||
|
||||
@@ -352,6 +352,19 @@ mod tests {
|
||||
pollster::block_on(resource.is_match(object, &HashMap::new()))
|
||||
}
|
||||
|
||||
#[test_case("arn:aws:s3:::mybucket/${jwt:preferred_username}/*", "mybucket/alice/photo.jpg" => true; "jwt variable")]
|
||||
#[test_case("arn:aws:s3:::mybucket/${jwt:preferred_username}/*", "mybucket/bob/photo.jpg" => false; "jwt variable other user")]
|
||||
#[test_case("arn:aws:s3:::mybucket/${ldap:username}/*", "mybucket/alice-ldap/photo.jpg" => true; "ldap variable")]
|
||||
fn test_resource_is_match_substitutes_condition_variables(resource: &str, object: &str) -> bool {
|
||||
let resource: Resource = resource.try_into().unwrap();
|
||||
let conditions = HashMap::from([
|
||||
("preferred_username".to_string(), vec!["alice".to_string()]),
|
||||
("username".to_string(), vec!["alice-ldap".to_string()]),
|
||||
]);
|
||||
|
||||
pollster::block_on(resource.is_match(object, &conditions))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wildcard_resource_serialization_round_trip() {
|
||||
use crate::policy::resource::ResourceSet;
|
||||
|
||||
@@ -639,7 +639,7 @@ mod hot_reload_tests {
|
||||
format!("{d}BEGIN {l}{d}\n{body}\n{d}END {l}{d}\n", d = PEM_BOUNDARY_DASHES, l = PEM_OPENSSH_LABEL,)
|
||||
}
|
||||
|
||||
fn test_ed25519_pem() -> String {
|
||||
pub(super) fn test_ed25519_pem() -> String {
|
||||
build_pem_block(
|
||||
"b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\n\
|
||||
QyNTUxOQAAACCkeMEUpnJEbOMBXiQfjZcHZMEbHW3DlNRL+Jbi1cIqMgAAAKDviRiQ74kY\n\
|
||||
@@ -1010,7 +1010,7 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
|
||||
fn auth_publickey_offered(
|
||||
&mut self,
|
||||
user: &str,
|
||||
_key: &keys::PublicKey,
|
||||
_key: russh::server::PublicKeyOffer<'_>,
|
||||
) -> impl std::future::Future<Output = Result<Auth, Self::Error>> + Send {
|
||||
async { Ok(Auth::reject()) }
|
||||
}
|
||||
@@ -1019,7 +1019,7 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
|
||||
fn auth_publickey(
|
||||
&mut self,
|
||||
user: &str,
|
||||
_key: &keys::PublicKey,
|
||||
_key: russh::server::VerifiedPublicKey<'_>,
|
||||
) -> impl std::future::Future<Output = Result<Auth, Self::Error>> + Send {
|
||||
async { Ok(Auth::reject()) }
|
||||
}
|
||||
@@ -1594,6 +1594,80 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn ssh_handshake_rejects_public_key_authentication() {
|
||||
use crate::common::dummy_storage::DummyBackend;
|
||||
use russh::client::{self, AuthResult};
|
||||
use russh::keys::{PrivateKeyWithHashAlg, PublicKeyOrCertificate};
|
||||
|
||||
struct PinnedHostKey(keys::PublicKey);
|
||||
|
||||
impl client::Handler for PinnedHostKey {
|
||||
type Error = russh::Error;
|
||||
|
||||
async fn check_server_key(&mut self, offered: &PublicKeyOrCertificate) -> Result<bool, Self::Error> {
|
||||
Ok(matches!(offered, PublicKeyOrCertificate::PublicKey { key, .. } if key.key_data() == self.0.key_data()))
|
||||
}
|
||||
}
|
||||
|
||||
timeout(Duration::from_secs(30), async {
|
||||
let key = PrivateKey::from_openssh(super::hot_reload_tests::test_ed25519_pem()).expect("decode test SSH key");
|
||||
let trusted_host_key = key.public_key().clone();
|
||||
let config = build_ssh_config(vec![key.clone()], 600, "SSH-2.0-RustFS");
|
||||
let peer_addr = "127.0.0.1:12345".parse().expect("test peer address");
|
||||
let handler = SshSessionHandler {
|
||||
storage: Arc::new(DummyBackend::new()),
|
||||
peer_addr,
|
||||
session_context: None,
|
||||
channels: HashMap::new(),
|
||||
read_only: false,
|
||||
part_size: super::super::test_support::TEST_PART_SIZE,
|
||||
handles_per_session: DEFAULT_HANDLES_PER_SESSION,
|
||||
backend_op_timeout_secs: DEFAULT_BACKEND_OP_TIMEOUT_SECS,
|
||||
read_cache_window: READ_CACHE_WINDOW_DEFAULT,
|
||||
read_cache_total_mem_limit: READ_CACHE_TOTAL_MEM_DEFAULT,
|
||||
read_cache_in_use: Arc::new(AtomicU64::new(0)),
|
||||
session_diag: Arc::new(SessionDiag::new(peer_addr, peer_addr)),
|
||||
};
|
||||
let (client_stream, server_stream) = tokio::io::duplex(64 * 1024);
|
||||
let server = tokio::spawn(async move {
|
||||
let session = russh::server::run_stream(config, server_stream, handler)
|
||||
.await
|
||||
.expect("start test SSH session");
|
||||
// The client deliberately disconnects after the rejected authentication.
|
||||
let _ = session.await;
|
||||
});
|
||||
let mut client =
|
||||
client::connect_stream(Arc::new(client::Config::default()), client_stream, PinnedHostKey(trusted_host_key))
|
||||
.await
|
||||
.expect("complete SSH handshake with the pinned host key");
|
||||
let auth = client
|
||||
.authenticate_publickey("public-key-auth-disabled", PrivateKeyWithHashAlg::new(Arc::new(key), None))
|
||||
.await
|
||||
.expect("receive public-key authentication response");
|
||||
match auth {
|
||||
AuthResult::Failure {
|
||||
remaining_methods,
|
||||
partial_success,
|
||||
} => {
|
||||
assert!(!partial_success, "public keys must not grant partial authentication");
|
||||
assert!(remaining_methods.contains(&MethodKind::Password));
|
||||
assert!(!remaining_methods.contains(&MethodKind::PublicKey));
|
||||
}
|
||||
AuthResult::Success => panic!("public-key authentication must remain disabled"),
|
||||
}
|
||||
client
|
||||
.disconnect(russh::Disconnect::ByApplication, "test complete", "")
|
||||
.await
|
||||
.expect("disconnect test client");
|
||||
drop(client);
|
||||
server.await.expect("join test SSH server");
|
||||
})
|
||||
.await
|
||||
.expect("SSH authentication rejection must complete within the test deadline");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssh_config_advertises_only_password() {
|
||||
let config = build_ssh_config(Vec::new(), 600, "SSH-2.0-RustFS");
|
||||
|
||||
@@ -1410,27 +1410,46 @@ impl DataUsageCache {
|
||||
Some((flattened, visited.len()))
|
||||
}
|
||||
|
||||
fn flatten_with_guard(&self, root: &DataUsageEntry, visited: &mut HashSet<String>, depth: usize) -> DataUsageEntry {
|
||||
let mut root = root.clone();
|
||||
fn flatten_with_guard<'a>(
|
||||
&'a self,
|
||||
root: &'a DataUsageEntry,
|
||||
visited: &mut HashSet<&'a str>,
|
||||
depth: usize,
|
||||
) -> DataUsageEntry {
|
||||
// A summary never owns child links. Cloning and clearing the set
|
||||
// retains its allocation in every completed subtree and update.
|
||||
let mut flat = DataUsageEntry {
|
||||
children: HashSet::new(),
|
||||
size: root.size,
|
||||
objects: root.objects,
|
||||
versions: root.versions,
|
||||
delete_markers: root.delete_markers,
|
||||
obj_sizes: root.obj_sizes.clone(),
|
||||
obj_versions: root.obj_versions.clone(),
|
||||
replication_stats: root.replication_stats.clone(),
|
||||
compacted: root.compacted,
|
||||
failed_objects: root.failed_objects,
|
||||
all_tier_stats: root.all_tier_stats.clone(),
|
||||
unknown_tier_stats: root.unknown_tier_stats.clone(),
|
||||
tier_accounting_proof: root.tier_accounting_proof,
|
||||
};
|
||||
if depth >= MAX_DATA_USAGE_CACHE_DEPTH {
|
||||
root.children.clear();
|
||||
return root;
|
||||
return flat;
|
||||
}
|
||||
|
||||
for id in root.children.clone().iter() {
|
||||
if !visited.insert(id.clone()) {
|
||||
for id in &root.children {
|
||||
if !visited.insert(id.as_str()) {
|
||||
continue;
|
||||
}
|
||||
if let Some(e) = self.cache.get(id) {
|
||||
let mut e = e.clone();
|
||||
if !e.children.is_empty() {
|
||||
e = self.flatten_with_guard(&e, visited, depth + 1);
|
||||
if e.children.is_empty() {
|
||||
flat.merge(e);
|
||||
} else {
|
||||
flat.merge(&self.flatten_with_guard(e, visited, depth + 1));
|
||||
}
|
||||
root.merge(&e);
|
||||
}
|
||||
}
|
||||
root.children.clear();
|
||||
root
|
||||
flat
|
||||
}
|
||||
|
||||
pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
|
||||
@@ -1467,38 +1486,31 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn delete_recursive(&mut self, hash: &DataUsageHash) {
|
||||
let mut visited = HashSet::new();
|
||||
self.delete_recursive_guard(hash, &mut visited, 0);
|
||||
self.delete_recursive_guard(&hash.0, 0);
|
||||
}
|
||||
|
||||
fn delete_recursive_guard(&mut self, hash: &DataUsageHash, visited: &mut HashSet<String>, depth: usize) {
|
||||
if !visited.insert(hash.key()) {
|
||||
fn delete_recursive_guard(&mut self, path: &str, depth: usize) {
|
||||
// Removal transfers child ownership and makes cyclic/shared references
|
||||
// no-ops, without cloning paths or maintaining a second visited set.
|
||||
let Some(children) = self.cache.remove(path).map(|entry| entry.children) else {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut need_remove = Vec::new();
|
||||
if let Some(v) = self.cache.get(&hash.string()) {
|
||||
for child in v.children.iter() {
|
||||
need_remove.push(child.clone());
|
||||
}
|
||||
}
|
||||
self.cache.remove(&hash.string());
|
||||
};
|
||||
if depth >= MAX_DATA_USAGE_CACHE_DEPTH {
|
||||
return;
|
||||
}
|
||||
for child in need_remove {
|
||||
self.delete_recursive_guard(&DataUsageHash(child), visited, depth + 1);
|
||||
for child in children {
|
||||
self.delete_recursive_guard(&child, depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn size_recursive(&self, path: &str) -> Option<DataUsageEntry> {
|
||||
match self.find(path) {
|
||||
Some(root) => {
|
||||
match self.cache.get_key_value(&hash_path(path).key()) {
|
||||
Some((key, root)) => {
|
||||
if root.children.is_empty() {
|
||||
return Some(root.clone());
|
||||
}
|
||||
let mut visited = HashSet::new();
|
||||
visited.insert(hash_path(path).key());
|
||||
visited.insert(key.as_str());
|
||||
let mut flat = self.flatten_with_guard(root, &mut visited, 0);
|
||||
if flat.replication_stats.as_ref().is_some_and(|stats| stats.is_empty()) {
|
||||
flat.replication_stats = None;
|
||||
@@ -1553,7 +1565,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
|
||||
let Some(e) = self.cache.get(&path.key()).cloned() else {
|
||||
let Some(e) = self.cache.get(&path.0) else {
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -1605,12 +1617,13 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn total_children_rec(&self, path: &str) -> usize {
|
||||
let root = hash_path(path);
|
||||
let mut visited = HashSet::new();
|
||||
visited.insert(hash_path(path).key());
|
||||
visited.insert(root.0.as_str());
|
||||
self.total_children_rec_guard(path, &mut visited, 0)
|
||||
}
|
||||
|
||||
fn total_children_rec_guard(&self, path: &str, visited: &mut HashSet<String>, depth: usize) -> usize {
|
||||
fn total_children_rec_guard<'a>(&'a self, path: &str, visited: &mut HashSet<&'a str>, depth: usize) -> usize {
|
||||
let Some(root) = self.find(path) else {
|
||||
return 0;
|
||||
};
|
||||
@@ -1620,7 +1633,7 @@ impl DataUsageCache {
|
||||
|
||||
let mut n = 0;
|
||||
for ch in root.children.iter() {
|
||||
if visited.insert(ch.clone()) {
|
||||
if visited.insert(ch.as_str()) {
|
||||
n += 1 + self.total_children_rec_guard(ch, visited, depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2428,6 +2428,69 @@ fn test_data_usage_cache_recursive_helpers_tolerate_cycles() {
|
||||
assert!(!copied.cache.contains_key(&child_hash.key()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_recursive_preserves_depth_boundary_and_unrelated_entries() {
|
||||
let mut cache = DataUsageCache::default();
|
||||
for depth in 0..=MAX_DATA_USAGE_CACHE_DEPTH + 1 {
|
||||
let mut entry = DataUsageEntry::default();
|
||||
if depth <= MAX_DATA_USAGE_CACHE_DEPTH {
|
||||
entry.children.insert(format!("node-{}", depth + 1));
|
||||
}
|
||||
cache.cache.insert(format!("node-{depth}"), entry);
|
||||
}
|
||||
cache.cache.insert("unrelated".to_string(), DataUsageEntry::default());
|
||||
|
||||
cache.delete_recursive(&DataUsageHash("node-0".to_string()));
|
||||
|
||||
assert_eq!(cache.cache.len(), 2, "delete only reachable entries within the depth limit");
|
||||
assert!(cache.cache.contains_key(&format!("node-{}", MAX_DATA_USAGE_CACHE_DEPTH + 1)));
|
||||
assert!(cache.cache.contains_key("unrelated"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_summary_releases_wide_child_sets() {
|
||||
let root = hash_path("bucket");
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.replace_hashed(
|
||||
&root,
|
||||
&None,
|
||||
&DataUsageEntry {
|
||||
size: 17,
|
||||
objects: 1,
|
||||
versions: 2,
|
||||
delete_markers: 1,
|
||||
failed_objects: 1,
|
||||
all_tier_stats: Some(AllTierStats::new()),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
for index in 0..10_000 {
|
||||
let child = hash_path(&format!("bucket/{index:05}"));
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
size: 3,
|
||||
objects: 1,
|
||||
versions: 2,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let summary = cache.size_recursive("bucket").expect("wide bucket summary");
|
||||
assert_eq!((summary.size, summary.objects, summary.versions), (30_017, 10_001, 20_002));
|
||||
assert_eq!((summary.delete_markers, summary.failed_objects), (1, 1));
|
||||
assert_eq!(summary.children.capacity(), 0, "summaries must not retain the wide namespace allocation");
|
||||
assert!(summary.all_tier_stats.is_some(), "preserve the root's optional statistics representation");
|
||||
assert_eq!(cache.find("bucket").expect("unchanged root").children.len(), 10_000);
|
||||
|
||||
cache.cache.insert("unrelated".to_string(), DataUsageEntry::default());
|
||||
cache.delete_recursive(&root);
|
||||
assert_eq!(cache.cache.len(), 1, "wide subtree deletion must consume every child exactly once");
|
||||
assert!(cache.cache.contains_key("unrelated"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_usage_cache_flatten_does_not_count_root_twice_in_cycle() {
|
||||
let root_hash = hash_path("bucket");
|
||||
|
||||
@@ -236,6 +236,7 @@ async fn profile_case(objects: usize, scenario: &str, samples: usize) {
|
||||
|
||||
let mut clone_ns = Vec::new();
|
||||
let mut copy_ns = Vec::new();
|
||||
let mut delete_ns = Vec::new();
|
||||
let mut flatten_ns = Vec::new();
|
||||
let mut encode_ns = Vec::new();
|
||||
let mut save_ns = Vec::new();
|
||||
@@ -257,6 +258,14 @@ async fn profile_case(objects: usize, scenario: &str, samples: usize) {
|
||||
same_cache(&copied, &cache);
|
||||
drop(copied);
|
||||
|
||||
let mut deleted = cache.clone();
|
||||
let root = hash_path("bucket");
|
||||
let started = WallInstant::now();
|
||||
deleted.delete_recursive(black_box(&root));
|
||||
delete_ns.push(elapsed_ns(started));
|
||||
assert!(deleted.cache.is_empty(), "delete the root and every reachable entry");
|
||||
drop(deleted);
|
||||
|
||||
let started = WallInstant::now();
|
||||
let aggregate = black_box(cache.checked_flatten("bucket").expect("valid fixture tree"));
|
||||
flatten_ns.push(elapsed_ns(started));
|
||||
@@ -326,6 +335,7 @@ async fn profile_case(objects: usize, scenario: &str, samples: usize) {
|
||||
"changed_entry_wire_bytes": changed_entry_wire_bytes, "save_body_bytes_per_sample": expected_wire.len() * 2,
|
||||
"snapshot_complete": cache.info.snapshot_complete,
|
||||
"clone": quantiles(clone_ns), "copy_with_children": quantiles(copy_ns), "checked_flatten": quantiles(flatten_ns),
|
||||
"delete_recursive": quantiles(delete_ns),
|
||||
"encode": quantiles(encode_ns), "save_inclusive": quantiles(save_ns), "memory_backend_ingest": quantiles(ingest_ns),
|
||||
})
|
||||
);
|
||||
|
||||
@@ -28,7 +28,7 @@ use crate::storage_api::owner::{
|
||||
EcstoreSetDisks, EcstoreStore, ecstore_is_reserved_or_invalid_bucket, ecstore_resolve_object_store_handle,
|
||||
};
|
||||
use futures::future::join_all;
|
||||
use rustfs_data_usage::{PrefixUsageEntry, PrefixUsageSummary};
|
||||
use rustfs_data_usage::{PrefixUsageEntry, PrefixUsageQuery, PrefixUsageSummary};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, SystemTime};
|
||||
@@ -133,6 +133,7 @@ async fn compute_prefix_usage(
|
||||
) -> BucketPrefixUsageResponse {
|
||||
let sets: Vec<Arc<EcstoreSetDisks>> = store.all_set_disks();
|
||||
let sets_total = sets.len();
|
||||
let per_set_max_entries = if sets_total <= 1 { max_entries } else { usize::MAX };
|
||||
let cache_name = format!("{bucket}/{DATA_USAGE_CACHE_NAME}");
|
||||
|
||||
let per_set = join_all(sets.into_iter().map(|set| {
|
||||
@@ -175,12 +176,25 @@ async fn compute_prefix_usage(
|
||||
return None;
|
||||
}
|
||||
let last_update = loaded.info.last_update;
|
||||
let query = loaded.prefix_usage(bucket, prefix, max_entries);
|
||||
// Fetch all sub-prefixes per set when sets_total > 1 so cross-set
|
||||
// aggregation merges every set's contribution before truncating
|
||||
// to the caller's max_entries bound.
|
||||
let query = loaded.prefix_usage(bucket, prefix, per_set_max_entries);
|
||||
Some((query, last_update))
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
|
||||
aggregate_set_prefix_usage(bucket, prefix, max_entries, sets_total, per_set.into_iter().flatten())
|
||||
}
|
||||
|
||||
fn aggregate_set_prefix_usage(
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
max_entries: usize,
|
||||
sets_total: usize,
|
||||
per_set: impl IntoIterator<Item = (Option<PrefixUsageQuery>, Option<SystemTime>)>,
|
||||
) -> BucketPrefixUsageResponse {
|
||||
let mut usage = PrefixUsageSummary::default();
|
||||
let mut sub_prefix_map: HashMap<String, PrefixUsageSummary> = HashMap::new();
|
||||
let mut sets_reporting = 0usize;
|
||||
@@ -190,7 +204,7 @@ async fn compute_prefix_usage(
|
||||
let mut truncated = false;
|
||||
let mut last_update: Option<SystemTime> = None;
|
||||
|
||||
for (query, set_last_update) in per_set.into_iter().flatten() {
|
||||
for (query, set_last_update) in per_set {
|
||||
// last_update counts every set that has scanned the bucket, even
|
||||
// when the prefix itself is absent on that set.
|
||||
if let Some(set_last_update) = set_last_update
|
||||
@@ -229,8 +243,7 @@ async fn compute_prefix_usage(
|
||||
.cmp(&left.usage.size)
|
||||
.then_with(|| left.prefix.cmp(&right.prefix))
|
||||
});
|
||||
// Merged rows can exceed max_entries only when per-set truncation
|
||||
// already flagged; enforce the caller bound on the merged view too.
|
||||
// Enforce the caller bound on the merged cross-set breakdown.
|
||||
if sub_prefixes.len() > max_entries {
|
||||
truncated = true;
|
||||
sub_prefixes.truncate(max_entries);
|
||||
@@ -346,4 +359,84 @@ mod tests {
|
||||
let map = guard.as_ref().expect("seeded");
|
||||
assert!(map.len() <= CACHE_MAX_ENTRIES, "cache must stay bounded, got {}", map.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_set_aggregation_merges_untruncated_set_breakdowns_before_applying_max_entries() {
|
||||
use crate::data_usage_define::{DataUsageCache, DataUsageEntry};
|
||||
|
||||
let mut set0 = DataUsageCache::default();
|
||||
set0.info.name = "bucket".to_string();
|
||||
set0.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
size: 190,
|
||||
objects: 2,
|
||||
versions: 2,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
set0.replace(
|
||||
"bucket/a",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: 100,
|
||||
objects: 1,
|
||||
versions: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
set0.replace(
|
||||
"bucket/b",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: 90,
|
||||
objects: 1,
|
||||
versions: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let mut set1 = DataUsageCache::default();
|
||||
set1.info.name = "bucket".to_string();
|
||||
set1.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
size: 190,
|
||||
objects: 2,
|
||||
versions: 2,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
set1.replace(
|
||||
"bucket/c",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: 100,
|
||||
objects: 1,
|
||||
versions: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
set1.replace(
|
||||
"bucket/b",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: 90,
|
||||
objects: 1,
|
||||
versions: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let q0 = set0.prefix_usage("bucket", "", usize::MAX);
|
||||
let q1 = set1.prefix_usage("bucket", "", usize::MAX);
|
||||
let merged = super::aggregate_set_prefix_usage("bucket", "", 1, 2, vec![(q0, None), (q1, None)]);
|
||||
assert!(merged.truncated);
|
||||
assert_eq!(merged.sub_prefixes.len(), 1);
|
||||
assert_eq!(merged.sub_prefixes[0].prefix, "b");
|
||||
assert_eq!(merged.sub_prefixes[0].usage.size, 180);
|
||||
assert_eq!(merged.sub_prefixes[0].usage.objects, 2);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs::FileType;
|
||||
use std::io::ErrorKind;
|
||||
@@ -751,11 +752,36 @@ struct ScannerCheckpointAncestor {
|
||||
entry: DataUsageEntry,
|
||||
}
|
||||
|
||||
struct ScannerUsageUpdates {
|
||||
sender: mpsc::Sender<DataUsageEntry>,
|
||||
cache: DataUsageCache,
|
||||
last_update: SystemTime,
|
||||
}
|
||||
|
||||
impl ScannerUsageUpdates {
|
||||
fn new(sender: mpsc::Sender<DataUsageEntry>, bucket: String) -> Self {
|
||||
Self {
|
||||
sender,
|
||||
cache: DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: bucket,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
last_update: SystemTime::UNIX_EPOCH,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_due(&self) -> bool {
|
||||
self.last_update.elapsed().unwrap_or(Duration::ZERO) >= Duration::from_secs(60)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FolderScanner {
|
||||
root: String,
|
||||
old_cache: DataUsageCache,
|
||||
new_cache: DataUsageCache,
|
||||
update_cache: DataUsageCache,
|
||||
|
||||
data_usage_scanner_debug: bool,
|
||||
heal_object_select: u32,
|
||||
@@ -773,11 +799,10 @@ pub struct FolderScanner {
|
||||
disks: Vec<Arc<Disk>>,
|
||||
disks_quorum: usize,
|
||||
|
||||
updates: Option<mpsc::Sender<DataUsageEntry>>,
|
||||
usage_updates: Option<ScannerUsageUpdates>,
|
||||
checkpoint_tx: Option<mpsc::Sender<DataUsageCache>>,
|
||||
checkpoint_ancestors: Vec<ScannerCheckpointAncestor>,
|
||||
checkpoint_depth: usize,
|
||||
last_update: SystemTime,
|
||||
checkpoint_objects: u64,
|
||||
object_metadata_visits: metrics::Counter,
|
||||
last_checkpoint_objects: u64,
|
||||
@@ -798,10 +823,6 @@ pub struct FolderScanner {
|
||||
resume_failed_paths: Vec<String>,
|
||||
coverage_gap: bool,
|
||||
raw_enumeration_progress: Vec<RawEnumerationProgress>,
|
||||
pending_heal_sync_deferred: bool,
|
||||
pending_heal_batch_dirty: bool,
|
||||
#[cfg(test)]
|
||||
pending_heal_sync_count: usize,
|
||||
pending_size_reconciliation_keys: HashSet<String>,
|
||||
pending_size_reconciliation_scopes: HashSet<String>,
|
||||
pending_size_reconciliation_truncated: bool,
|
||||
@@ -1036,8 +1057,9 @@ impl FolderScanner {
|
||||
|
||||
fn reuse_clean_root_child(&mut self, child: &CachedFolder, child_hash: &DataUsageHash, into: &mut DataUsageEntry) {
|
||||
self.new_cache.copy_with_children(&self.old_cache, child_hash, &child.parent);
|
||||
self.update_cache
|
||||
.copy_with_children(&self.old_cache, child_hash, &child.parent);
|
||||
if let Some(updates) = &mut self.usage_updates {
|
||||
updates.cache.copy_with_children(&self.old_cache, child_hash, &child.parent);
|
||||
}
|
||||
into.add_child(child_hash);
|
||||
}
|
||||
|
||||
@@ -1145,8 +1167,8 @@ impl FolderScanner {
|
||||
}
|
||||
|
||||
/// Apply the per-object size-resolution ledger updates in one place. The
|
||||
/// scanner cache is the durable boundary; both working copies are updated
|
||||
/// so an incremental publication cannot lose a debt or its resolution.
|
||||
/// scanner cache is the durable boundary; progress updates contain only
|
||||
/// usage totals and never own a copy of the reconciliation ledger.
|
||||
fn apply_size_reconciliation(&mut self, summary: &SizeSummary) {
|
||||
let now = Self::now_secs();
|
||||
self.pending_size_reconciliation_keys
|
||||
@@ -1159,26 +1181,25 @@ impl FolderScanner {
|
||||
);
|
||||
self.pending_size_reconciliation_truncated |= summary.size_reconciliation_truncated;
|
||||
|
||||
for info in [&mut self.new_cache.info, &mut self.update_cache.info] {
|
||||
for incoming in &summary.size_reconciliation {
|
||||
if let Some(existing) = info.size_reconciliation.get_mut(&incoming.key) {
|
||||
existing.reason = incoming.reason.clone();
|
||||
existing.physical_size = incoming.physical_size;
|
||||
existing.generation = incoming.generation.clone();
|
||||
existing.version_id = incoming.version_id.clone();
|
||||
existing.attempts = existing.attempts.saturating_add(1);
|
||||
continue;
|
||||
}
|
||||
|
||||
if size_reconciliation_entry_bytes(incoming) > MAX_SIZE_RECONCILIATION_BYTES_PER_BUCKET {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut entry = incoming.clone();
|
||||
entry.first_seen = now;
|
||||
entry.attempts = 1;
|
||||
info.size_reconciliation.insert(entry.key.clone(), entry);
|
||||
let info = &mut self.new_cache.info;
|
||||
for incoming in &summary.size_reconciliation {
|
||||
if let Some(existing) = info.size_reconciliation.get_mut(&incoming.key) {
|
||||
existing.reason = incoming.reason.clone();
|
||||
existing.physical_size = incoming.physical_size;
|
||||
existing.generation = incoming.generation.clone();
|
||||
existing.version_id = incoming.version_id.clone();
|
||||
existing.attempts = existing.attempts.saturating_add(1);
|
||||
continue;
|
||||
}
|
||||
|
||||
if size_reconciliation_entry_bytes(incoming) > MAX_SIZE_RECONCILIATION_BYTES_PER_BUCKET {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut entry = incoming.clone();
|
||||
entry.first_seen = now;
|
||||
entry.attempts = 1;
|
||||
info.size_reconciliation.insert(entry.key.clone(), entry);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1188,27 +1209,24 @@ impl FolderScanner {
|
||||
let scopes = std::mem::take(&mut self.pending_size_reconciliation_scopes);
|
||||
let truncated = std::mem::replace(&mut self.pending_size_reconciliation_truncated, false);
|
||||
|
||||
for info in [&mut self.new_cache.info, &mut self.update_cache.info] {
|
||||
if !truncated {
|
||||
info.size_reconciliation.retain(|key, entry| {
|
||||
!scopes.contains(&size_reconciliation_scope_key(&entry.bucket, &entry.object)) || current_keys.contains(key)
|
||||
});
|
||||
}
|
||||
prune_size_reconciliation(info, now);
|
||||
let info = &mut self.new_cache.info;
|
||||
if !truncated {
|
||||
info.size_reconciliation.retain(|key, entry| {
|
||||
!scopes.contains(&size_reconciliation_scope_key(&entry.bucket, &entry.object)) || current_keys.contains(key)
|
||||
});
|
||||
}
|
||||
prune_size_reconciliation(info, now);
|
||||
}
|
||||
|
||||
fn record_scan_resume_hint(&mut self, folder: &str) {
|
||||
self.new_cache.info.scan_resume_after = Some(folder.to_string());
|
||||
self.update_cache.info.scan_resume_after = Some(folder.to_string());
|
||||
let checkpoint = DataUsageScanCheckpoint::new(folder.to_string(), DataUsageScanCheckpointReason::Unknown);
|
||||
global_metrics().record_scanner_checkpoint_set(
|
||||
checkpoint.version,
|
||||
checkpoint.resume_after.clone(),
|
||||
checkpoint.reason.as_str(),
|
||||
);
|
||||
self.new_cache.info.scan_checkpoint = Some(checkpoint.clone());
|
||||
self.update_cache.info.scan_checkpoint = Some(checkpoint);
|
||||
self.new_cache.info.scan_checkpoint = Some(checkpoint);
|
||||
}
|
||||
|
||||
fn record_completed_child(&mut self, folder: &str) {
|
||||
@@ -1511,8 +1529,7 @@ impl FolderScanner {
|
||||
self.record_scan_resume_hint_if_not_ancestor(&child_hash.key());
|
||||
parent_entry.add_child(child_hash);
|
||||
self.new_cache.replace_hashed(child_hash, parent, &child_entry);
|
||||
self.update_cache.delete_recursive(child_hash);
|
||||
self.update_cache.copy_with_children(&self.new_cache, child_hash, parent);
|
||||
self.refresh_usage_subtree(child_hash, parent);
|
||||
self.send_update().await;
|
||||
}
|
||||
}
|
||||
@@ -1655,29 +1672,75 @@ impl FolderScanner {
|
||||
self.data_usage_scanner_debug = debug;
|
||||
}
|
||||
|
||||
/// Send update if enough time has passed
|
||||
/// Should be called on a regular basis when the new_cache contains more recent total than previously.
|
||||
/// May or may not send an update upstream.
|
||||
fn should_send_update(&self) -> bool {
|
||||
if self.updates.is_none() {
|
||||
return false;
|
||||
fn compact_completed_folder(&mut self, folder: &CachedFolder, hash: &DataUsageHash, entry: &DataUsageEntry) {
|
||||
// Completed healthy subtrees can keep aggregate usage. Coverage gaps
|
||||
// and failed-object debt still need child entries for precise resume
|
||||
// and to avoid rereading healthy siblings on a retry.
|
||||
let forward_sweep = self.old_cache.info.scan_progress.is_some();
|
||||
if entry.compacted || self.new_cache.info.name == folder.name || (forward_sweep && self.coverage_gap) {
|
||||
return;
|
||||
}
|
||||
let summary = if entry.children.is_empty() {
|
||||
Cow::Borrowed(entry)
|
||||
} else if let Some(flat) = self.new_cache.size_recursive(&hash.0) {
|
||||
Cow::Owned(flat)
|
||||
} else {
|
||||
return;
|
||||
};
|
||||
if forward_sweep && summary.failed_objects != 0 {
|
||||
return;
|
||||
}
|
||||
let should_compact = summary.objects < DATA_SCANNER_COMPACT_LEAST_OBJECT
|
||||
|| entry.children.iter().all(|key| {
|
||||
self.new_cache
|
||||
.cache
|
||||
.get(key)
|
||||
.is_none_or(|child| child.children.is_empty() && child.objects <= 1)
|
||||
});
|
||||
if !should_compact {
|
||||
return;
|
||||
}
|
||||
// Heal revisits also reach this path without a parent advancing
|
||||
// coverage. Keep the endpoint addressable when its completed
|
||||
// subtree is replaced by an aggregate.
|
||||
if self.coverage_frontier.as_deref().is_some_and(|frontier| {
|
||||
frontier
|
||||
.strip_prefix(&folder.name)
|
||||
.is_some_and(|suffix| suffix.starts_with('/'))
|
||||
}) {
|
||||
self.record_completed_child(&folder.name);
|
||||
}
|
||||
match summary {
|
||||
Cow::Borrowed(_) => {
|
||||
// The leaf is already cached; only its sampling mode changes.
|
||||
if let Some(cached) = self.new_cache.cache.get_mut(&hash.0) {
|
||||
cached.compacted = true;
|
||||
}
|
||||
}
|
||||
Cow::Owned(mut flat) => {
|
||||
flat.compacted = true;
|
||||
self.new_cache.delete_recursive(hash);
|
||||
self.new_cache.replace_hashed(hash, &folder.parent, &flat);
|
||||
}
|
||||
}
|
||||
|
||||
let elapsed = self.last_update.elapsed().unwrap_or(Duration::from_secs(0));
|
||||
elapsed >= Duration::from_secs(60)
|
||||
}
|
||||
|
||||
fn refresh_usage_subtree(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
|
||||
if let Some(updates) = &mut self.usage_updates {
|
||||
updates.cache.delete_recursive(hash);
|
||||
updates.cache.copy_with_children(&self.new_cache, hash, parent);
|
||||
}
|
||||
}
|
||||
|
||||
/// Send at most one aggregate per minute when a caller requested progress.
|
||||
pub async fn send_update(&mut self) {
|
||||
// Send at most an update every minute.
|
||||
if !self.should_send_update() {
|
||||
let Some(updates) = &mut self.usage_updates else { return };
|
||||
if !updates.is_due() {
|
||||
return;
|
||||
}
|
||||
|
||||
if let Some(flat) = self.update_cache.size_recursive(&self.new_cache.info.name)
|
||||
&& let Some(ref updates) = self.updates
|
||||
{
|
||||
// Try to send without blocking
|
||||
if let Err(e) = updates.send(flat).await {
|
||||
if let Some(flat) = updates.cache.size_recursive(&self.new_cache.info.name) {
|
||||
if let Err(e) = updates.sender.send(flat).await {
|
||||
error!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_FOLDER_STATE,
|
||||
@@ -1689,17 +1752,17 @@ impl FolderScanner {
|
||||
"Scanner folder update send failed"
|
||||
);
|
||||
}
|
||||
self.last_update = SystemTime::now();
|
||||
updates.last_update = SystemTime::now();
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_update_for_entry(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>, entry: &DataUsageEntry) {
|
||||
if !self.should_send_update() {
|
||||
return;
|
||||
if let Some(updates) = &mut self.usage_updates
|
||||
&& updates.is_due()
|
||||
{
|
||||
updates.cache.replace_hashed(hash, parent, entry);
|
||||
self.send_update().await;
|
||||
}
|
||||
|
||||
self.update_cache.replace_hashed(hash, parent, entry);
|
||||
self.send_update().await;
|
||||
}
|
||||
|
||||
async fn record_missing_erasure_metadata(
|
||||
@@ -1838,8 +1901,7 @@ impl FolderScanner {
|
||||
wait_for_scanner_io(&ctx, self.sleeper.sleep_folder()).await?;
|
||||
}
|
||||
|
||||
let mut existing_folders: Vec<CachedFolder> = Vec::new();
|
||||
let mut new_folders: Vec<CachedFolder> = Vec::new();
|
||||
let mut queued_folders: Vec<QueuedFolder> = Vec::new();
|
||||
let mut found_object_metadata = false;
|
||||
let mut object_count: u64 = 0;
|
||||
let yield_every_objects = scanner_yield_every_n_objects();
|
||||
@@ -2147,12 +2209,21 @@ impl FolderScanner {
|
||||
self.reuse_clean_root_child(&this, &h, into);
|
||||
} else if exists {
|
||||
abandoned_children.remove(&h.key());
|
||||
existing_folders.push(this);
|
||||
self.update_cache
|
||||
.copy_with_children(&self.old_cache, &h, &Some(this_hash.clone()));
|
||||
queued_folders.push(QueuedFolder {
|
||||
folder: this,
|
||||
source: FolderScanSource::Existing,
|
||||
});
|
||||
if let Some(updates) = &mut self.usage_updates {
|
||||
updates
|
||||
.cache
|
||||
.copy_with_children(&self.old_cache, &h, &Some(this_hash.clone()));
|
||||
}
|
||||
} else {
|
||||
abandoned_children.remove(&h.key());
|
||||
new_folders.push(this);
|
||||
queued_folders.push(QueuedFolder {
|
||||
folder: this,
|
||||
source: FolderScanSource::New,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
@@ -2374,17 +2445,17 @@ impl FolderScanner {
|
||||
// frontier. Keep the emergency compaction limit for large trees.
|
||||
let should_compact = (self.new_cache.info.name != folder.name
|
||||
&& self.old_cache.info.scan_progress.is_none()
|
||||
&& existing_folders.len() + new_folders.len() >= DATA_SCANNER_COMPACT_AT_FOLDERS)
|
||||
|| existing_folders.len() + new_folders.len() >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS;
|
||||
&& queued_folders.len() >= DATA_SCANNER_COMPACT_AT_FOLDERS)
|
||||
|| queued_folders.len() >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS;
|
||||
|
||||
let total_folders = existing_folders.len() + new_folders.len();
|
||||
let total_folders = queued_folders.len();
|
||||
self.alert_excessive_folders(&folder.name, total_folders);
|
||||
|
||||
if !into.compacted && should_compact {
|
||||
into.compacted = true;
|
||||
new_folders.append(&mut existing_folders);
|
||||
|
||||
existing_folders.clear();
|
||||
for queued in &mut queued_folders {
|
||||
queued.source = FolderScanSource::New;
|
||||
}
|
||||
|
||||
if self.data_usage_scanner_debug {
|
||||
debug!(
|
||||
@@ -2393,20 +2464,13 @@ impl FolderScanner {
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_FOLDER,
|
||||
folder = %folder.name,
|
||||
entry_count = new_folders.len(),
|
||||
entry_count = queued_folders.len(),
|
||||
state = "preemptive_compaction",
|
||||
"Scanner folder switched to compacted mode"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if !into.compacted {
|
||||
for folder_item in &existing_folders {
|
||||
let h = hash_path(&folder_item.name);
|
||||
self.update_cache.copy_with_children(&self.old_cache, &h, &folder_item.parent);
|
||||
}
|
||||
}
|
||||
|
||||
let is_scan_root = folder.name == self.old_cache.info.name;
|
||||
let scan_checkpoint = self.old_cache.info.scan_checkpoint.as_ref();
|
||||
let checkpoint_resume_after = scan_checkpoint.and_then(|checkpoint| {
|
||||
@@ -2430,15 +2494,6 @@ impl FolderScanner {
|
||||
.and_then(|resume_after| folder_resume_match(&folder.name, resume_after))
|
||||
.is_some_and(|resume_match| matches!(resume_match, FolderResumeMatch::Descendant));
|
||||
let scan_resume_after = checkpoint_resume_after.or(self.old_cache.info.scan_resume_after.as_deref());
|
||||
let mut queued_folders = Vec::with_capacity(new_folders.len() + existing_folders.len());
|
||||
queued_folders.extend(new_folders.into_iter().map(|folder| QueuedFolder {
|
||||
folder,
|
||||
source: FolderScanSource::New,
|
||||
}));
|
||||
queued_folders.extend(existing_folders.into_iter().map(|folder| QueuedFolder {
|
||||
folder,
|
||||
source: FolderScanSource::Existing,
|
||||
}));
|
||||
let has_queued_folders = !queued_folders.is_empty();
|
||||
let forward_sweep = self.old_cache.info.scan_progress.is_some();
|
||||
let resume_order = if forward_sweep {
|
||||
@@ -2479,19 +2534,21 @@ impl FolderScanner {
|
||||
match queued_folder.source {
|
||||
FolderScanSource::New => {
|
||||
// Add new folders to the update tree so totals update for these.
|
||||
if !into.compacted {
|
||||
if !into.compacted
|
||||
&& let Some(updates) = &mut self.usage_updates
|
||||
{
|
||||
let mut found_any = false;
|
||||
let mut parent = this_hash.clone();
|
||||
let update_cache_name_hash = hash_path(&self.update_cache.info.name);
|
||||
let update_root_hash = hash_path(&updates.cache.info.name);
|
||||
|
||||
while parent != update_cache_name_hash {
|
||||
while parent != update_root_hash {
|
||||
let parent_key = parent.key();
|
||||
let e = self.update_cache.find(&parent_key);
|
||||
let e = updates.cache.find(&parent_key);
|
||||
if e.is_none_or(|v| v.compacted) {
|
||||
found_any = true;
|
||||
break;
|
||||
}
|
||||
if let Some(next) = self.update_cache.search_parent(&parent) {
|
||||
if let Some(next) = updates.cache.search_parent(&parent) {
|
||||
parent = next;
|
||||
} else {
|
||||
found_any = true;
|
||||
@@ -2500,7 +2557,8 @@ impl FolderScanner {
|
||||
}
|
||||
if !found_any {
|
||||
// Add non-compacted empty entry.
|
||||
self.update_cache
|
||||
updates
|
||||
.cache
|
||||
.replace_hashed(&h, &Some(this_hash.clone()), &DataUsageEntry::default());
|
||||
}
|
||||
}
|
||||
@@ -2574,13 +2632,17 @@ impl FolderScanner {
|
||||
};
|
||||
if outcome == FolderScanOutcome::Vanished {
|
||||
self.new_cache.delete_recursive(&h);
|
||||
self.update_cache.delete_recursive(&h);
|
||||
if let Some(updates) = &mut self.usage_updates {
|
||||
updates.cache.delete_recursive(&h);
|
||||
}
|
||||
into.children.remove(&h.key());
|
||||
continue;
|
||||
}
|
||||
if outcome == FolderScanOutcome::ErasureData {
|
||||
self.new_cache.delete_recursive(&h);
|
||||
self.update_cache.delete_recursive(&h);
|
||||
if let Some(updates) = &mut self.usage_updates {
|
||||
updates.cache.delete_recursive(&h);
|
||||
}
|
||||
into.children.remove(&h.key());
|
||||
// Earlier namespace siblings may have completed, but none
|
||||
// can certify coverage of this unresolved parent object.
|
||||
@@ -2600,23 +2662,13 @@ impl FolderScanner {
|
||||
self.send_update_for_entry(&this_hash, &folder.parent, into).await;
|
||||
tokio::task::yield_now().await;
|
||||
} else {
|
||||
tokio::task::yield_now().await;
|
||||
into.add_child(&h);
|
||||
self.record_completed_child(&folder_item.name);
|
||||
tokio::task::yield_now().await;
|
||||
self.maybe_send_checkpoint(&folder, &this_hash, into);
|
||||
self.update_cache.delete_recursive(&h);
|
||||
self.update_cache.copy_with_children(&self.new_cache, &h, &folder_item.parent);
|
||||
self.refresh_usage_subtree(&h, &folder_item.parent);
|
||||
self.send_update().await;
|
||||
}
|
||||
|
||||
if queued_folder.source == FolderScanSource::New
|
||||
&& !into.compacted
|
||||
&& self.update_cache.find(&this_hash.key()).is_some_and(|v| !v.compacted)
|
||||
{
|
||||
self.update_cache.delete_recursive(&h);
|
||||
self.update_cache
|
||||
.copy_with_children(&self.new_cache, &h, &Some(this_hash.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
// Scan for healing
|
||||
@@ -3013,8 +3065,7 @@ impl FolderScanner {
|
||||
|
||||
into.add_child(&h);
|
||||
// We scanned a folder, optionally send update.
|
||||
self.update_cache.delete_recursive(&h);
|
||||
self.update_cache.copy_with_children(&self.new_cache, &h, &folder_item.parent);
|
||||
self.refresh_usage_subtree(&h, &folder_item.parent);
|
||||
self.send_update().await;
|
||||
self.maybe_send_checkpoint(&folder, &this_hash, into);
|
||||
}
|
||||
@@ -3028,36 +3079,7 @@ impl FolderScanner {
|
||||
self.new_cache.replace_hashed(&this_hash, &folder.parent, into);
|
||||
}
|
||||
|
||||
// Keep independently accounted children while the sweep cursor may
|
||||
// reference them; the hard cardinality compaction below still applies.
|
||||
if !into.compacted
|
||||
&& self.old_cache.info.scan_progress.is_none()
|
||||
&& self.new_cache.info.name != folder.name
|
||||
&& let Some(mut flat) = self.new_cache.size_recursive(&this_hash.key())
|
||||
{
|
||||
flat.compacted = true;
|
||||
let mut should_compact = false;
|
||||
|
||||
if flat.objects < DATA_SCANNER_COMPACT_LEAST_OBJECT {
|
||||
should_compact = true;
|
||||
} else {
|
||||
// Compact if we only have objects as children...
|
||||
should_compact = true;
|
||||
for k in &into.children {
|
||||
if let Some(v) = self.new_cache.cache.get(k)
|
||||
&& (!v.children.is_empty() || v.objects > 1)
|
||||
{
|
||||
should_compact = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if should_compact {
|
||||
self.new_cache.delete_recursive(&this_hash);
|
||||
self.new_cache.replace_hashed(&this_hash, &folder.parent, &flat);
|
||||
}
|
||||
}
|
||||
self.compact_completed_folder(&folder, &this_hash, into);
|
||||
|
||||
// Compact if too many children...
|
||||
if !into.compacted {
|
||||
@@ -3070,12 +3092,13 @@ impl FolderScanner {
|
||||
done_compact();
|
||||
}
|
||||
|
||||
if self.update_cache.cache.contains_key(&this_hash.key()) && !was_compacted {
|
||||
// Replace if existed before.
|
||||
if let Some(flat) = self.new_cache.size_recursive(&this_hash.key()) {
|
||||
self.update_cache.delete_recursive(&this_hash);
|
||||
self.update_cache.replace_hashed(&this_hash, &folder.parent, &flat);
|
||||
}
|
||||
if !was_compacted
|
||||
&& let Some(updates) = &mut self.usage_updates
|
||||
&& updates.cache.cache.contains_key(&this_hash.key())
|
||||
&& let Some(flat) = self.new_cache.size_recursive(&this_hash.key())
|
||||
{
|
||||
updates.cache.delete_recursive(&this_hash);
|
||||
updates.cache.replace_hashed(&this_hash, &folder.parent, &flat);
|
||||
}
|
||||
|
||||
self.finish_size_reconciliation_batch();
|
||||
@@ -3173,10 +3196,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
info: cache.info.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let update_cache = DataUsageCache {
|
||||
info: cache.info.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let usage_updates = updates.map(|sender| ScannerUsageUpdates::new(sender, bucket_name.clone()));
|
||||
let resume_failed_paths = if resume_frontier.is_some() {
|
||||
failed_scan_paths(&cache)
|
||||
} else {
|
||||
@@ -3187,7 +3207,6 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
root: base_path,
|
||||
old_cache: cache,
|
||||
new_cache,
|
||||
update_cache,
|
||||
data_usage_scanner_debug: false,
|
||||
heal_object_select,
|
||||
scan_mode,
|
||||
@@ -3200,11 +3219,10 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
sleeper,
|
||||
disks,
|
||||
disks_quorum,
|
||||
updates,
|
||||
usage_updates,
|
||||
checkpoint_tx,
|
||||
checkpoint_ancestors: Vec::new(),
|
||||
checkpoint_depth: 0,
|
||||
last_update: SystemTime::UNIX_EPOCH,
|
||||
checkpoint_objects: 0,
|
||||
object_metadata_visits,
|
||||
last_checkpoint_objects: 0,
|
||||
@@ -3220,11 +3238,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
resume_frontier,
|
||||
resume_failed_paths,
|
||||
coverage_gap: false,
|
||||
pending_heal_sync_deferred: false,
|
||||
pending_heal_batch_dirty: false,
|
||||
raw_enumeration_progress: Vec::new(),
|
||||
#[cfg(test)]
|
||||
pending_heal_sync_count: 0,
|
||||
pending_size_reconciliation_keys: HashSet::new(),
|
||||
pending_size_reconciliation_scopes: HashSet::new(),
|
||||
pending_size_reconciliation_truncated: false,
|
||||
@@ -3234,7 +3248,6 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
|
||||
let now = FolderScanner::now_secs();
|
||||
prune_size_reconciliation(&mut scanner.new_cache.info, now);
|
||||
prune_size_reconciliation(&mut scanner.update_cache.info, now);
|
||||
|
||||
// Check if context is cancelled
|
||||
if ctx.is_cancelled() {
|
||||
|
||||
@@ -18,28 +18,6 @@ use super::*;
|
||||
const PENDING_HEAL_RETRY_BASE_SECS: u64 = 15 * 60;
|
||||
const PENDING_HEAL_RETRY_CAP_SECS: u64 = 6 * 60 * 60;
|
||||
|
||||
pub(super) struct PendingHealSyncBatch<'a> {
|
||||
pub(super) scanner: &'a mut FolderScanner,
|
||||
}
|
||||
|
||||
impl<'a> PendingHealSyncBatch<'a> {
|
||||
pub(super) fn new(scanner: &'a mut FolderScanner) -> Self {
|
||||
scanner.pending_heal_sync_deferred = true;
|
||||
scanner.pending_heal_batch_dirty = false;
|
||||
Self { scanner }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PendingHealSyncBatch<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.scanner.pending_heal_sync_deferred = false;
|
||||
if self.scanner.pending_heal_batch_dirty {
|
||||
self.scanner.pending_heal_batch_dirty = false;
|
||||
self.scanner.sync_pending_heals();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn record_pending_heal_attempt(entry: &mut PendingScannerHeal, now: u64) {
|
||||
entry.last_attempt = now;
|
||||
entry.attempts = entry.attempts.saturating_add(1);
|
||||
@@ -52,19 +30,6 @@ pub(super) fn observe_pending_heal_admission(entry: &mut PendingScannerHeal, res
|
||||
}
|
||||
|
||||
impl FolderScanner {
|
||||
pub(super) fn sync_pending_heals(&mut self) {
|
||||
self.pending_heals_changed = true;
|
||||
if self.pending_heal_sync_deferred {
|
||||
self.pending_heal_batch_dirty = true;
|
||||
return;
|
||||
}
|
||||
self.update_cache.info.pending_heals = self.new_cache.info.pending_heals.clone();
|
||||
#[cfg(test)]
|
||||
{
|
||||
self.pending_heal_sync_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn clear_pending_scanner_heal(
|
||||
&mut self,
|
||||
kind: PendingScannerHealKind,
|
||||
@@ -78,7 +43,7 @@ impl FolderScanner {
|
||||
.pending_heals
|
||||
.retain(|entry| !pending_scanner_heal_matches(entry, kind, bucket, object, version_id));
|
||||
if self.new_cache.info.pending_heals.len() != before {
|
||||
self.sync_pending_heals();
|
||||
self.pending_heals_changed = true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,7 +65,7 @@ impl FolderScanner {
|
||||
.find(|entry| pending_scanner_heal_matches(entry, kind, bucket, object, version_id))
|
||||
{
|
||||
observe_pending_heal_admission(entry, result);
|
||||
self.sync_pending_heals();
|
||||
self.pending_heals_changed = true;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -116,9 +81,8 @@ impl FolderScanner {
|
||||
last_admission_result: result.result_label().to_string(),
|
||||
last_admission_reason: result.reason_label().to_string(),
|
||||
});
|
||||
if self.prune_pending_scanner_heal_capacity() == 0 {
|
||||
self.sync_pending_heals();
|
||||
}
|
||||
self.pending_heals_changed = true;
|
||||
self.prune_pending_scanner_heal_capacity();
|
||||
}
|
||||
|
||||
/// Preserve the discovery reason when a candidate could not be admitted
|
||||
@@ -140,7 +104,7 @@ impl FolderScanner {
|
||||
.find(|entry| pending_scanner_heal_matches(entry, kind, bucket, object, version_id))
|
||||
{
|
||||
entry.last_admission_reason = reason.to_string();
|
||||
self.sync_pending_heals();
|
||||
self.pending_heals_changed = true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,22 +133,22 @@ impl FolderScanner {
|
||||
state = "pending_heal_expired",
|
||||
"Scanner pending heal ledger expired old entries"
|
||||
);
|
||||
self.sync_pending_heals();
|
||||
self.pending_heals_changed = true;
|
||||
}
|
||||
|
||||
self.prune_pending_scanner_heal_capacity();
|
||||
}
|
||||
|
||||
fn prune_pending_scanner_heal_capacity(&mut self) -> usize {
|
||||
fn prune_pending_scanner_heal_capacity(&mut self) {
|
||||
let len = self.new_cache.info.pending_heals.len();
|
||||
if len <= MAX_PENDING_SCANNER_HEALS_PER_BUCKET {
|
||||
return 0;
|
||||
return;
|
||||
}
|
||||
|
||||
sort_pending_scanner_heals_for_retry(&mut self.new_cache.info.pending_heals);
|
||||
let remove_count = len.saturating_sub(MAX_PENDING_SCANNER_HEALS_PER_BUCKET);
|
||||
self.new_cache.info.pending_heals.drain(..remove_count);
|
||||
self.sync_pending_heals();
|
||||
self.pending_heals_changed = true;
|
||||
counter!(
|
||||
METRIC_SCANNER_PENDING_HEAL_PRUNE_TOTAL,
|
||||
"bucket" => self.new_cache.info.name.clone()
|
||||
@@ -201,7 +165,6 @@ impl FolderScanner {
|
||||
state = "pending_heal_pruned",
|
||||
"Scanner pending heal ledger pruned oldest entries"
|
||||
);
|
||||
remove_count
|
||||
}
|
||||
|
||||
pub(super) fn update_pending_scanner_heal_after_admission(
|
||||
@@ -233,20 +196,18 @@ impl FolderScanner {
|
||||
.find(|entry| pending_scanner_heal_matches(entry, kind, bucket, object, version_id))
|
||||
{
|
||||
observe_pending_heal_admission(entry, result);
|
||||
self.sync_pending_heals();
|
||||
self.pending_heals_changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn retry_pending_scanner_heals(&mut self) -> Result<(), ScannerError> {
|
||||
let batch = PendingHealSyncBatch::new(self);
|
||||
let scanner = &mut *batch.scanner;
|
||||
if !scanner.should_heal().await {
|
||||
if !self.should_heal().await {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let bucket = scanner.new_cache.info.name.clone();
|
||||
let bucket = self.new_cache.info.name.clone();
|
||||
// Legacy notices cannot bind a verified disposition to the current
|
||||
// incarnation, kind, set scope and responsibility generation.
|
||||
let repaired = rustfs_common::mrf_channel::take_mrf_repaired_events_for(&bucket);
|
||||
@@ -254,14 +215,14 @@ impl FolderScanner {
|
||||
counter!("rustfs_scanner_unverified_repair_notices_total")
|
||||
.increment(u64::try_from(repaired.len()).unwrap_or(u64::MAX));
|
||||
}
|
||||
scanner.prune_pending_scanner_heals();
|
||||
for pending in pending_scanner_heal_retry_candidates(&scanner.new_cache.info.pending_heals, &bucket) {
|
||||
if !scanner.should_heal().await {
|
||||
self.prune_pending_scanner_heals();
|
||||
for pending in pending_scanner_heal_retry_candidates(&self.new_cache.info.pending_heals, &bucket) {
|
||||
if !self.should_heal().await {
|
||||
break;
|
||||
}
|
||||
|
||||
let Some(request) = build_pending_scanner_heal_request(&pending) else {
|
||||
scanner.clear_pending_scanner_heal(pending.kind, &pending.bucket, None, pending.version_id.as_deref());
|
||||
self.clear_pending_scanner_heal(pending.kind, &pending.bucket, None, pending.version_id.as_deref());
|
||||
counter!(
|
||||
METRIC_SCANNER_PENDING_HEAL_MALFORMED_TOTAL,
|
||||
"bucket" => pending.bucket.clone(),
|
||||
@@ -280,7 +241,7 @@ impl FolderScanner {
|
||||
continue;
|
||||
};
|
||||
|
||||
if let Some(entry) = scanner.new_cache.info.pending_heals.iter_mut().find(|entry| {
|
||||
if let Some(entry) = self.new_cache.info.pending_heals.iter_mut().find(|entry| {
|
||||
pending_scanner_heal_matches(
|
||||
entry,
|
||||
pending.kind,
|
||||
@@ -290,17 +251,16 @@ impl FolderScanner {
|
||||
)
|
||||
}) {
|
||||
record_pending_heal_attempt(entry, Self::now_secs());
|
||||
scanner.sync_pending_heals();
|
||||
self.pending_heals_changed = true;
|
||||
}
|
||||
scanner
|
||||
.send_required_scanner_heal_request(
|
||||
pending.kind,
|
||||
pending.bucket.clone(),
|
||||
pending.object.clone(),
|
||||
pending.version_id.clone(),
|
||||
request,
|
||||
)
|
||||
.await?;
|
||||
self.send_required_scanner_heal_request(
|
||||
pending.kind,
|
||||
pending.bucket.clone(),
|
||||
pending.object.clone(),
|
||||
pending.version_id.clone(),
|
||||
request,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -31,6 +31,7 @@ pub(super) mod checkpoint_fixture;
|
||||
pub(super) mod enumeration_restart;
|
||||
mod incremental_enumeration;
|
||||
mod metadata_visits;
|
||||
mod usage_updates;
|
||||
|
||||
/// Reset the process-global alert cooldown map; test-only.
|
||||
fn reset_alert_cooldowns() {
|
||||
@@ -353,7 +354,6 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
|
||||
root: temp_dir.to_string_lossy().to_string(),
|
||||
old_cache: DataUsageCache::default(),
|
||||
new_cache: DataUsageCache::default(),
|
||||
update_cache: DataUsageCache::default(),
|
||||
data_usage_scanner_debug: false,
|
||||
heal_object_select: 0,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
@@ -366,11 +366,10 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
|
||||
sleeper: SCANNER_SLEEPER.clone(),
|
||||
disks: Vec::new(),
|
||||
disks_quorum: 0,
|
||||
updates: None,
|
||||
usage_updates: None,
|
||||
checkpoint_tx: None,
|
||||
checkpoint_ancestors: Vec::new(),
|
||||
checkpoint_depth: 0,
|
||||
last_update: SystemTime::UNIX_EPOCH,
|
||||
checkpoint_objects: 0,
|
||||
object_metadata_visits: metrics::Counter::noop(),
|
||||
last_checkpoint_objects: 0,
|
||||
@@ -391,9 +390,6 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
|
||||
resume_failed_paths: Vec::new(),
|
||||
coverage_gap: false,
|
||||
raw_enumeration_progress: Vec::new(),
|
||||
pending_heal_sync_deferred: false,
|
||||
pending_heal_batch_dirty: false,
|
||||
pending_heal_sync_count: 0,
|
||||
pending_size_reconciliation_keys: HashSet::new(),
|
||||
pending_size_reconciliation_scopes: HashSet::new(),
|
||||
pending_size_reconciliation_truncated: false,
|
||||
@@ -710,7 +706,6 @@ async fn malformed_size_reconciliation_replays_after_restart() {
|
||||
scanner.finish_size_reconciliation_batch();
|
||||
|
||||
assert_eq!(scanner.new_cache.info.size_reconciliation.len(), 1);
|
||||
assert_eq!(scanner.update_cache.info.size_reconciliation.len(), 1);
|
||||
assert_eq!(scanner.new_cache.info.size_reconciliation[&entry.key].attempts, 2);
|
||||
|
||||
let encoded = rmp_serde::to_vec_named(&scanner.new_cache.info).expect("size ledger should encode");
|
||||
@@ -724,7 +719,6 @@ async fn malformed_size_reconciliation_replays_after_restart() {
|
||||
scanner.apply_size_reconciliation(&resolved);
|
||||
scanner.finish_size_reconciliation_batch();
|
||||
assert!(scanner.new_cache.info.size_reconciliation.is_empty());
|
||||
assert!(scanner.update_cache.info.size_reconciliation.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1532,7 +1526,6 @@ async fn mrf_ownership_legacy_notices_preserve_pending_entries() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
scanner.heal_object_select = 1;
|
||||
|
||||
let version = Uuid::new_v4().to_string();
|
||||
@@ -1679,7 +1672,6 @@ async fn test_pending_heal_prune_expires_stale_entries() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
|
||||
let mut stale = pending_heal(PendingScannerHealKind::Object, "bucket", Some("stale"), None, 1, 1);
|
||||
stale.first_seen = FolderScanner::now_secs().saturating_sub(MAX_PENDING_SCANNER_HEAL_AGE_SECS + 1);
|
||||
@@ -1690,7 +1682,6 @@ async fn test_pending_heal_prune_expires_stale_entries() {
|
||||
|
||||
assert_eq!(scanner.new_cache.info.pending_heals.len(), 1);
|
||||
assert_eq!(scanner.new_cache.info.pending_heals[0].object.as_deref(), Some("fresh"));
|
||||
assert_eq!(scanner.update_cache.info.pending_heals, scanner.new_cache.info.pending_heals);
|
||||
assert!(scanner.pending_heals_changed);
|
||||
}
|
||||
|
||||
@@ -1699,7 +1690,6 @@ async fn test_pending_heal_update_keeps_stale_entry_until_retry_prune() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
|
||||
let mut stale = pending_heal(PendingScannerHealKind::Object, "bucket", Some("object"), None, 1, 1);
|
||||
stale.first_seen = FolderScanner::now_secs().saturating_sub(MAX_PENDING_SCANNER_HEAL_AGE_SECS + 1);
|
||||
@@ -1717,7 +1707,6 @@ async fn test_pending_heal_update_keeps_stale_entry_until_retry_prune() {
|
||||
assert_eq!(scanner.new_cache.info.pending_heals.len(), 1);
|
||||
assert_eq!(scanner.new_cache.info.pending_heals[0].attempts, 1);
|
||||
assert_eq!(scanner.new_cache.info.pending_heals[0].object.as_deref(), Some("object"));
|
||||
assert_eq!(scanner.update_cache.info.pending_heals, scanner.new_cache.info.pending_heals);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1725,7 +1714,6 @@ async fn test_pending_heal_queue_full_deduplicates_object_entry() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
|
||||
scanner.update_pending_scanner_heal_after_admission(
|
||||
PendingScannerHealKind::Object,
|
||||
@@ -1751,7 +1739,6 @@ async fn test_pending_heal_queue_full_deduplicates_object_entry() {
|
||||
assert_eq!(pending.attempts, 1);
|
||||
assert_eq!(pending.last_admission_result, "dropped");
|
||||
assert_eq!(pending.last_admission_reason, "queue_full");
|
||||
assert_eq!(scanner.update_cache.info.pending_heals, scanner.new_cache.info.pending_heals);
|
||||
assert!(scanner.pending_heals_changed);
|
||||
}
|
||||
|
||||
@@ -1877,7 +1864,6 @@ async fn test_scan_folder_keeps_old_cache_bounded_by_loaded_entries() {
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
scanner.is_erasure_mode = true;
|
||||
for directory in 0..16 {
|
||||
for object in 0..32 {
|
||||
@@ -2297,7 +2283,6 @@ async fn test_scan_folder_skips_unreadable_child_directory() {
|
||||
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
|
||||
let folder = CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
@@ -2325,7 +2310,6 @@ async fn test_scan_folder_treats_child_removed_before_open_as_empty() {
|
||||
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
|
||||
let folder = CachedFolder {
|
||||
name: "bucket/gone".to_string(),
|
||||
@@ -2358,7 +2342,6 @@ async fn scan_folder_charges_each_enumerated_entry_to_the_disk_budget() {
|
||||
scanner.sleeper = SCANNER_SLEEPER.for_disk(&temp_dir);
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
|
||||
let folder = CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
@@ -2485,7 +2468,6 @@ async fn test_scan_folder_missing_bucket_root_still_reports_not_found() {
|
||||
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
|
||||
let folder = CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
@@ -2630,7 +2612,6 @@ async fn test_scan_folder_xl_meta_named_directory_uses_namespace_descent() {
|
||||
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
scanner.is_erasure_mode = true;
|
||||
|
||||
let parent = CancellationToken::new();
|
||||
@@ -2690,7 +2671,6 @@ async fn scan_corrupt_metadata_with_captured_logs(logs: &CapturedLogs) {
|
||||
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
scanner.is_erasure_mode = true;
|
||||
|
||||
let parent = CancellationToken::new();
|
||||
@@ -2841,7 +2821,6 @@ async fn test_scan_folder_missing_xl_meta_stops_erasure_data_dir_descent() {
|
||||
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
scanner.is_erasure_mode = true;
|
||||
scanner.sleeper = DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest).for_disk_with_budget_for_tests(
|
||||
&temp_dir,
|
||||
@@ -2934,8 +2913,9 @@ async fn test_scan_folder_uuid_namespace_part_name_directory_is_not_data_dir() {
|
||||
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
scanner.is_erasure_mode = true;
|
||||
let (sender, _receiver) = mpsc::channel(1);
|
||||
scanner.usage_updates = Some(ScannerUsageUpdates::new(sender, "bucket".to_string()));
|
||||
let namespace_hash = hash_path(&format!("bucket/{namespace_name}"));
|
||||
scanner.old_cache.replace_hashed(
|
||||
&namespace_hash,
|
||||
@@ -2970,7 +2950,13 @@ async fn test_scan_folder_uuid_namespace_part_name_directory_is_not_data_dir() {
|
||||
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
|
||||
assert!(scanner.new_cache.info.failed_objects.is_empty());
|
||||
assert!(
|
||||
scanner.update_cache.find(&namespace_hash.key()).is_some(),
|
||||
scanner
|
||||
.usage_updates
|
||||
.as_ref()
|
||||
.expect("requested progress")
|
||||
.cache
|
||||
.find(&namespace_hash.key())
|
||||
.is_some(),
|
||||
"an existing UUID namespace must retain its cached subtree"
|
||||
);
|
||||
}
|
||||
@@ -2991,7 +2977,6 @@ async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() {
|
||||
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
scanner.is_erasure_mode = false;
|
||||
|
||||
let parent = CancellationToken::new();
|
||||
@@ -3032,7 +3017,6 @@ async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entri
|
||||
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
|
||||
scanner.old_cache.replace(
|
||||
"bucket/clean",
|
||||
@@ -3091,7 +3075,6 @@ async fn scan_hot_cold_segment_fixture(scoped: bool) -> (DataUsageEntry, Vec<Str
|
||||
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
if scoped {
|
||||
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
|
||||
scanner.old_cache.replace(
|
||||
@@ -3177,7 +3160,6 @@ async fn scoped_root_scan_preserves_erasure_health_walks() {
|
||||
scanner.is_erasure_mode = true;
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
|
||||
scanner.old_cache.replace(
|
||||
"bucket/clean",
|
||||
@@ -3221,11 +3203,9 @@ async fn test_scan_folder_compacted_parent_sends_partial_update() {
|
||||
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
scanner.last_update = SystemTime::UNIX_EPOCH;
|
||||
|
||||
let (tx, mut rx) = mpsc::channel(1);
|
||||
scanner.updates = Some(tx);
|
||||
scanner.usage_updates = Some(ScannerUsageUpdates::new(tx, "bucket".to_string()));
|
||||
|
||||
let folder = CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
@@ -4226,7 +4206,6 @@ async fn test_scan_folder_ignores_symlinked_child_directory() {
|
||||
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
|
||||
let folder = CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
|
||||
@@ -20,7 +20,9 @@ use crate::{DataUsageCacheSource, DataUsageScanPlanDigest};
|
||||
use std::io::Cursor;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
mod completed_compaction;
|
||||
mod deep_compacted;
|
||||
mod memory_budget;
|
||||
mod periodic_snapshot;
|
||||
mod segment_observation;
|
||||
|
||||
@@ -783,7 +785,13 @@ async fn check_complete_sampling_resumption(resume_mode: HealScanMode) {
|
||||
matches!(state, crate::scanner_io::DataUsageCacheScanState::Prepared { .. }),
|
||||
"Normal complete is not Deep Current"
|
||||
);
|
||||
cache.prepare_bucket_checkpoint("bucket", 12, 7, SOURCE, PLAN, identity);
|
||||
// The first sweep already compacts this prefix. Select its sampling
|
||||
// cycle so the bounded scan exercises interruption rather than reuse.
|
||||
let sampling_cycles = data_usage_update_dir_cycles();
|
||||
let sampling_cycle = (12..12 + sampling_cycles)
|
||||
.find(|cycle| hash_path("bucket/prefix").mod_(*cycle, sampling_cycles))
|
||||
.expect("one prefix sampling cycle per rotation");
|
||||
cache.prepare_bucket_checkpoint("bucket", u64::from(sampling_cycle), 7, SOURCE, PLAN, identity);
|
||||
assert!(cache.info.scan_progress.is_none(), "complete unchanged baseline uses existing sampling");
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
@@ -835,7 +843,7 @@ async fn check_complete_sampling_resumption(resume_mode: HealScanMode) {
|
||||
.load_with_revisions(store.clone(), CACHE_NAME)
|
||||
.await
|
||||
.expect("reload partial each recovery round");
|
||||
loaded.prepare_bucket_checkpoint("bucket", 12, 7, SOURCE, PLAN, resumed_identity);
|
||||
loaded.prepare_bucket_checkpoint("bucket", u64::from(sampling_cycle), 7, SOURCE, PLAN, resumed_identity);
|
||||
assert!(loaded.info.scan_progress.is_some(), "sampling partial must enter forward validation");
|
||||
loaded.info.skip_healing = true;
|
||||
let parent = CancellationToken::new();
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn completed_leaf_compaction_reuses_cached_storage() {
|
||||
let (mut scanner, root) = build_test_scanner().await;
|
||||
let _cleanup = TestGuard { temp_dir: Some(root) };
|
||||
scanner.old_cache = bound_checkpoint().0;
|
||||
let folder = CachedFolder {
|
||||
name: "bucket/object".to_string(),
|
||||
parent: Some(hash_path("bucket")),
|
||||
object_heal_prob_div: 1,
|
||||
};
|
||||
let hash = hash_path(&folder.name);
|
||||
|
||||
for (coverage_gap, failed_objects, should_compact) in [(false, 0, true), (true, 0, false), (false, 1, false)] {
|
||||
scanner.coverage_gap = coverage_gap;
|
||||
scanner.new_cache = DataUsageCache {
|
||||
info: scanner.old_cache.info.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut target = String::with_capacity(4096);
|
||||
target.push_str("arn:test:replica");
|
||||
let allocation = target.as_ptr();
|
||||
let capacity = target.capacity();
|
||||
let mut leaf = DataUsageEntry {
|
||||
size: 2048,
|
||||
objects: 1,
|
||||
versions: 3,
|
||||
failed_objects,
|
||||
replication_stats: Some(rustfs_data_usage::ReplicationAllStats {
|
||||
targets: HashMap::from([(target, rustfs_data_usage::ReplicationTargetUsage::default())]),
|
||||
..Default::default()
|
||||
}),
|
||||
all_tier_stats: Some(rustfs_data_usage::AllTierStats::new()),
|
||||
..Default::default()
|
||||
};
|
||||
leaf.obj_sizes.add(2048);
|
||||
leaf.obj_versions.add(3);
|
||||
let accumulator = leaf.clone();
|
||||
let mut expected = serde_json::to_value(&leaf).expect("leaf metadata");
|
||||
expected["compacted"] = serde_json::json!(should_compact);
|
||||
scanner.new_cache.replace(&folder.name, "bucket", leaf);
|
||||
|
||||
scanner.compact_completed_folder(&folder, &hash, &accumulator);
|
||||
|
||||
let cached = scanner.new_cache.find(&folder.name).expect("retained leaf");
|
||||
assert_eq!(serde_json::to_value(cached).expect("compacted metadata"), expected);
|
||||
let target = cached
|
||||
.replication_stats
|
||||
.as_ref()
|
||||
.expect("retained replication statistics")
|
||||
.targets
|
||||
.keys()
|
||||
.next()
|
||||
.expect("retained replication target");
|
||||
assert_eq!(target.capacity(), capacity, "leaf compaction must preserve owned allocations");
|
||||
assert_eq!(target.as_ptr(), allocation);
|
||||
assert!(!accumulator.compacted, "cache compaction must not change the caller's accounting mode");
|
||||
}
|
||||
}
|
||||
|
||||
async fn scan(
|
||||
disk: &Arc<Disk>,
|
||||
cache: DataUsageCache,
|
||||
mode: HealScanMode,
|
||||
max_objects: u64,
|
||||
) -> (ScannerDiskScanOutcome, Arc<ScannerCycleBudget>) {
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&CancellationToken::new(),
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(max_objects),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let outcome = disk
|
||||
.clone()
|
||||
.nsscanner_disk(budget.token(), budget.clone(), vec![disk.clone()], cache, None, scan_options(mode))
|
||||
.await
|
||||
.expect("scan checkpoint fixture through the real disk path");
|
||||
(outcome, budget)
|
||||
}
|
||||
|
||||
async fn assert_completed_compaction_resumes(mode: HealScanMode) {
|
||||
let (scanner, root) = build_test_scanner().await;
|
||||
let _cleanup = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
for (prefix, size) in [("a", 2), ("b", 3), ("c", 5)] {
|
||||
for object in 0..4 {
|
||||
write_checkpoint_object(&root, &format!("nested/{prefix}/{object:04}"), &[(None, size)]).await;
|
||||
}
|
||||
}
|
||||
let identity = crate::DataUsageScanIdentity {
|
||||
scan_mode: mode,
|
||||
tier_registry_generation: crate::runtime_tier_registry_for_cycle(11, 7).await.generation,
|
||||
..bound_checkpoint().1
|
||||
};
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, identity);
|
||||
cache.info.skip_healing = true;
|
||||
let (outcome, budget) = scan(&scanner.local_disk, cache, mode, 6).await;
|
||||
assert_eq!(budget.progress().0, 6);
|
||||
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects));
|
||||
let ScannerDiskScanOutcome::Partial(cache) = outcome else {
|
||||
panic!("the object budget must interrupt the second prefix")
|
||||
};
|
||||
let completed = cache.find("bucket/nested/a").expect("completed prefix remains addressable");
|
||||
assert!(completed.compacted, "a completed prefix must release leaves during the first sweep");
|
||||
assert!(completed.children.is_empty());
|
||||
assert_eq!((completed.objects, completed.size), (4, 8));
|
||||
assert!(
|
||||
cache.find("bucket/nested/a/0000").is_none(),
|
||||
"completed leaves must not remain in the cache"
|
||||
);
|
||||
for prefix in ["bucket", "bucket/nested", "bucket/nested/b"] {
|
||||
assert!(!cache.find(prefix).expect("unfinished ancestor retained").compacted);
|
||||
}
|
||||
assert!(
|
||||
cache.find("bucket/nested/b/0000").is_some(),
|
||||
"unfinished prefix retains its resume boundary"
|
||||
);
|
||||
let partial = cache
|
||||
.checked_flatten_complete_scope("bucket")
|
||||
.expect("partial graph has no orphan entries");
|
||||
assert_eq!((partial.objects, partial.versions, partial.size), (6, 0, 14));
|
||||
assert_eq!(cache.validated_scan_frontier(), Some("bucket/nested/b/0000"));
|
||||
|
||||
let store = FixtureStore::new();
|
||||
let revisions = DataUsageCache::default()
|
||||
.load_with_revisions(store.clone(), CACHE_NAME)
|
||||
.await
|
||||
.expect("initial fixture revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist compacted partial cache through the production codec and CAS path");
|
||||
let mut reloaded = store.strict_load().await;
|
||||
assert_eq!(reloaded.info.scan_coverage_receipt, cache.info.scan_coverage_receipt);
|
||||
assert_eq!(reloaded.validated_scan_frontier(), cache.validated_scan_frontier());
|
||||
assert_eq!(
|
||||
reloaded.prepare_bucket_checkpoint("bucket", 12, 8, SOURCE, PLAN, identity),
|
||||
crate::DataUsageCachePrepareOutcome::Reused,
|
||||
"a later cycle and leader must adopt compacted completed coverage"
|
||||
);
|
||||
let (outcome, budget) = scan(&scanner.local_disk, reloaded, mode, 13).await;
|
||||
assert_eq!(
|
||||
budget.progress().0,
|
||||
7,
|
||||
"only the interrupted boundary object and six unseen objects may be read on resume"
|
||||
);
|
||||
let ScannerDiskScanOutcome::Complete(completed) = outcome else {
|
||||
panic!("resumed sweep must complete")
|
||||
};
|
||||
let totals = completed
|
||||
.checked_flatten_complete_scope("bucket")
|
||||
.expect("completed cache has no orphan entries");
|
||||
assert_eq!((totals.objects, totals.versions, totals.size), (12, 0, 40));
|
||||
assert!(completed.info.snapshot_complete);
|
||||
assert!(completed.info.scan_progress.is_none());
|
||||
assert!(!completed.has_scan_position());
|
||||
assert!(completed.find("bucket/nested").expect("completed nested prefix").compacted);
|
||||
assert_eq!(completed.cache.len(), 2, "only the bucket root and completed prefix aggregate remain");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn completed_compaction_normal_resumes_without_revisiting_completed_prefix() {
|
||||
assert_completed_compaction_resumes(HealScanMode::Normal).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn completed_compaction_deep_resumes_without_revisiting_completed_prefix() {
|
||||
assert_completed_compaction_resumes(HealScanMode::Deep).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn completed_compaction_preserves_frontier_inside_coverage_gap() {
|
||||
assert_compaction_frontier(true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn completed_compaction_promotes_frontier_without_parent_bookkeeping() {
|
||||
assert_compaction_frontier(false).await;
|
||||
}
|
||||
|
||||
async fn assert_compaction_frontier(coverage_gap: bool) {
|
||||
let (mut scanner, root) = build_test_scanner().await;
|
||||
let _cleanup = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
for object in 0..3 {
|
||||
write_checkpoint_object(&root, &format!("prefix/{object:04}"), &[(None, 2)]).await;
|
||||
}
|
||||
let identity = crate::DataUsageScanIdentity {
|
||||
tier_registry_generation: crate::runtime_tier_registry_for_cycle(11, 7).await.generation,
|
||||
..bound_checkpoint().1
|
||||
};
|
||||
scanner
|
||||
.old_cache
|
||||
.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, identity);
|
||||
scanner.new_cache.info = scanner.old_cache.info.clone();
|
||||
scanner.is_erasure_mode = true;
|
||||
scanner.skip_heal.store(true, Ordering::SeqCst);
|
||||
scanner.sleeper = DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest);
|
||||
// A failed child freezes the frontier. Without a gap, completing this
|
||||
// subtree must preserve coverage even when the caller is a heal rewalk
|
||||
// that does not advance the parent frontier after scan_folder returns.
|
||||
scanner.coverage_gap = coverage_gap;
|
||||
scanner.coverage_frontier = Some("bucket/prefix/0000".to_string());
|
||||
let mut entry = DataUsageEntry::default();
|
||||
scanner
|
||||
.scan_folder(
|
||||
CancellationToken::new(),
|
||||
CachedFolder {
|
||||
name: "bucket/prefix".to_string(),
|
||||
parent: Some(hash_path("bucket")),
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut entry,
|
||||
)
|
||||
.await
|
||||
.expect("walk the ancestor of a frozen coverage frontier");
|
||||
let expected_frontier = if coverage_gap { "bucket/prefix/0000" } else { "bucket/prefix" };
|
||||
assert_eq!(scanner.coverage_frontier.as_deref(), Some(expected_frontier));
|
||||
assert_eq!(
|
||||
scanner.new_cache.find("bucket/prefix").expect("completed ancestor").compacted,
|
||||
!coverage_gap
|
||||
);
|
||||
assert_eq!(scanner.new_cache.find("bucket/prefix/0000").is_some(), coverage_gap);
|
||||
scanner
|
||||
.new_cache
|
||||
.seal_scan_frontier(scanner.coverage_frontier.as_deref())
|
||||
.expect("seal the retained frontier");
|
||||
assert_eq!(scanner.new_cache.validated_scan_frontier(), Some(expected_frontier));
|
||||
let totals = scanner
|
||||
.new_cache
|
||||
.checked_flatten_complete_scope("bucket")
|
||||
.expect("coverage-gap cache remains connected");
|
||||
assert_eq!((totals.objects, totals.size), (3, 6));
|
||||
}
|
||||
@@ -72,18 +72,16 @@ impl CompactedFixture {
|
||||
..bound_checkpoint().1
|
||||
};
|
||||
let mut cache = DataUsageCache::default();
|
||||
// The first real scan builds coverage; the second same-plan scan takes
|
||||
// the normal compaction path. No synthetic compacted cache is injected.
|
||||
for cycle in [11, 12] {
|
||||
cache.prepare_bucket_checkpoint("bucket", cycle, 7, SOURCE, PLAN, identity);
|
||||
cache.info.skip_healing = true;
|
||||
let (outcome, budget) = scan(&scanner.local_disk, cache, mode, OBJECTS + 1).await;
|
||||
let ScannerDiskScanOutcome::Complete(completed) = outcome else {
|
||||
panic!("fixture baseline must complete")
|
||||
};
|
||||
assert_eq!(budget.progress().0, OBJECTS, "baseline must read every metadata object");
|
||||
cache = completed;
|
||||
}
|
||||
// Build and compact coverage in the first real sweep. No synthetic
|
||||
// compacted cache is injected.
|
||||
cache.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, identity);
|
||||
cache.info.skip_healing = true;
|
||||
let (outcome, budget) = scan(&scanner.local_disk, cache, mode, OBJECTS + 1).await;
|
||||
let ScannerDiskScanOutcome::Complete(completed) = outcome else {
|
||||
panic!("fixture baseline must complete")
|
||||
};
|
||||
assert_eq!(budget.progress().0, OBJECTS, "baseline must read every metadata object");
|
||||
cache = completed;
|
||||
let store = FixtureStore::new();
|
||||
cache = save_reload(&store, &cache).await;
|
||||
assert!(cache.find(PREFIX).expect("baseline prefix").compacted);
|
||||
@@ -263,9 +261,11 @@ async fn check_large_prefix_resumption(mode: HealScanMode) {
|
||||
assert!(cache.info.scan_checkpoint.is_none());
|
||||
assert!(cache.info.scan_coverage_receipt.is_none());
|
||||
|
||||
// Completing the sweep must restore compaction on the next same-plan scan.
|
||||
// A completed prefix releases its children before the first sweep
|
||||
// returns; a later same-plan scan keeps that compact representation.
|
||||
let retained_entries = cache.cache.len();
|
||||
assert!(!cache.find(PREFIX).expect("completed prefix").compacted);
|
||||
assert!(cache.find(PREFIX).expect("completed prefix").compacted);
|
||||
assert_eq!(retained_entries, 3, "only the bucket, compacted prefix and tail remain");
|
||||
cache.prepare_bucket_checkpoint("bucket", 12 + round, 7 + round, SOURCE, PLAN, identity);
|
||||
assert!(cache.info.scan_progress.is_none(), "a complete same-plan cache needs no forward sweep");
|
||||
let (outcome, _) = scan(&scanner.local_disk, cache, mode, expected_reads + 1).await;
|
||||
@@ -275,8 +275,8 @@ async fn check_large_prefix_resumption(mode: HealScanMode) {
|
||||
let reloaded = save_reload(&store, &compacted).await;
|
||||
assert!(reloaded.find(PREFIX).expect("compacted prefix").compacted);
|
||||
assert!(
|
||||
reloaded.cache.len() < retained_entries,
|
||||
"completed sweeps must release child cache entries"
|
||||
reloaded.cache.len() <= retained_entries,
|
||||
"later sweeps must not retain the compacted prefix's children"
|
||||
);
|
||||
let total = reloaded.checked_flatten("bucket").expect("compacted bucket coverage");
|
||||
assert_eq!((total.objects, total.size), (expected, expected));
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
const OBJECTS_PER_PREFIX: usize = 16;
|
||||
const FROZEN_FRONTIER: &str = "bucket/prefix-0000/0000";
|
||||
|
||||
fn cache_shape(cache: &DataUsageCache, wire_bytes: usize) -> serde_json::Value {
|
||||
let mut key_capacity_bytes = 0;
|
||||
let mut child_links = 0;
|
||||
let mut child_set_capacity = 0;
|
||||
let mut child_key_capacity_bytes = 0;
|
||||
for (key, entry) in &cache.cache {
|
||||
key_capacity_bytes += key.capacity();
|
||||
child_links += entry.children.len();
|
||||
child_set_capacity += entry.children.capacity();
|
||||
child_key_capacity_bytes += entry.children.iter().map(String::capacity).sum::<usize>();
|
||||
}
|
||||
// The canonical histograms own eleven and seven u64 counters. Map control
|
||||
// bytes, allocator overhead, optional statistics and transient work are excluded.
|
||||
let histogram_payload_lower_bound_bytes = cache.cache.len() * (11 + 7) * std::mem::size_of::<u64>();
|
||||
let map_slot_size = std::mem::size_of::<(String, DataUsageEntry)>();
|
||||
let retained_lower_bound_bytes = cache.cache.capacity() * map_slot_size
|
||||
+ key_capacity_bytes
|
||||
+ child_set_capacity * std::mem::size_of::<String>()
|
||||
+ child_key_capacity_bytes
|
||||
+ histogram_payload_lower_bound_bytes;
|
||||
serde_json::json!({
|
||||
"entries": cache.cache.len(),
|
||||
"entry_capacity": cache.cache.capacity(),
|
||||
"entry_size_bytes": std::mem::size_of::<DataUsageEntry>(),
|
||||
"map_slot_size_bytes": map_slot_size,
|
||||
"map_key_capacity_bytes": key_capacity_bytes,
|
||||
"child_links": child_links,
|
||||
"child_set_capacity": child_set_capacity,
|
||||
"child_key_capacity_bytes": child_key_capacity_bytes,
|
||||
"histogram_payload_lower_bound_bytes": histogram_payload_lower_bound_bytes,
|
||||
"retained_lower_bound_bytes": retained_lower_bound_bytes,
|
||||
"wire_bytes": wire_bytes,
|
||||
})
|
||||
}
|
||||
|
||||
async fn characterize_memory_budget(prefixes: usize, coverage_gap: bool) {
|
||||
let (mut scanner, root) = build_test_scanner().await;
|
||||
let _cleanup = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
for prefix in 0..prefixes {
|
||||
for object in 0..OBJECTS_PER_PREFIX {
|
||||
write_checkpoint_object(&root, &format!("prefix-{prefix:04}/{object:04}"), &[(None, 1)]).await;
|
||||
}
|
||||
}
|
||||
let identity = crate::DataUsageScanIdentity {
|
||||
tier_registry_generation: crate::runtime_tier_registry_for_cycle(11, 7).await.generation,
|
||||
..bound_checkpoint().1
|
||||
};
|
||||
scanner
|
||||
.old_cache
|
||||
.prepare_bucket_checkpoint("bucket", 11, 7, SOURCE, PLAN, identity);
|
||||
scanner.new_cache.info = scanner.old_cache.info.clone();
|
||||
scanner.is_erasure_mode = true;
|
||||
scanner.skip_heal.store(true, Ordering::SeqCst);
|
||||
scanner.sleeper = DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest);
|
||||
// Isolate the already-established gap state from filesystem fault injection.
|
||||
// Every object below is healthy; this control does not reproduce the fault
|
||||
// or durable debt that caused a real scan's coverage gap.
|
||||
scanner.coverage_gap = coverage_gap;
|
||||
if coverage_gap {
|
||||
scanner.coverage_frontier = Some(FROZEN_FRONTIER.to_string());
|
||||
}
|
||||
scanner
|
||||
.scan_folder(
|
||||
CancellationToken::new(),
|
||||
CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut DataUsageEntry::default(),
|
||||
)
|
||||
.await
|
||||
.expect("walk the real wide-root metadata fixture");
|
||||
|
||||
let expected_objects = prefixes * OBJECTS_PER_PREFIX;
|
||||
let totals = scanner
|
||||
.new_cache
|
||||
.checked_flatten_complete_scope("bucket")
|
||||
.expect("the measured cache must be connected and contain no orphan entries");
|
||||
assert_eq!((totals.objects, totals.size, totals.versions), (expected_objects, expected_objects, 0));
|
||||
assert_eq!(totals.failed_objects, 0, "the gap control must not fabricate object failure debt");
|
||||
assert!(
|
||||
scanner.usage_updates.is_none(),
|
||||
"production-style scans must not retain a progress mirror"
|
||||
);
|
||||
let bucket = scanner.new_cache.find("bucket").expect("retained root inventory");
|
||||
assert!(!bucket.compacted, "these root fanouts preserve prefix statistics");
|
||||
assert_eq!(bucket.children.len(), prefixes);
|
||||
if coverage_gap {
|
||||
assert_eq!(scanner.coverage_frontier.as_deref(), Some(FROZEN_FRONTIER));
|
||||
assert!(
|
||||
scanner.new_cache.find(FROZEN_FRONTIER).is_some(),
|
||||
"the frozen endpoint must remain addressable"
|
||||
);
|
||||
} else {
|
||||
for prefix in 0..prefixes {
|
||||
let path = format!("bucket/prefix-{prefix:04}");
|
||||
let entry = scanner.new_cache.find(&path).expect("completed root prefix");
|
||||
assert!(entry.compacted);
|
||||
assert!(entry.children.is_empty());
|
||||
assert_eq!((entry.objects, entry.size), (OBJECTS_PER_PREFIX, OBJECTS_PER_PREFIX));
|
||||
}
|
||||
assert_eq!(scanner.new_cache.cache.len(), prefixes + 1, "healthy completed leaves must be released");
|
||||
}
|
||||
|
||||
scanner
|
||||
.new_cache
|
||||
.seal_scan_frontier(scanner.coverage_frontier.as_deref())
|
||||
.expect("seal the measured coverage endpoint");
|
||||
assert!(scanner.new_cache.validated_scan_frontier().is_some());
|
||||
let encoded = scanner
|
||||
.new_cache
|
||||
.marshal_msg()
|
||||
.expect("encode the measured cache with the production codec");
|
||||
let restored = DataUsageCache::unmarshal(&encoded).expect("reload the measured coverage graph");
|
||||
assert_eq!(restored.validated_scan_frontier(), scanner.new_cache.validated_scan_frontier());
|
||||
assert_eq!(restored.info.scan_coverage_receipt, scanner.new_cache.info.scan_coverage_receipt);
|
||||
let restored_totals = restored
|
||||
.checked_flatten_complete_scope("bucket")
|
||||
.expect("reloaded accounting graph");
|
||||
assert_eq!((restored_totals.objects, restored_totals.size), (expected_objects, expected_objects));
|
||||
if coverage_gap {
|
||||
assert_eq!(restored.validated_scan_frontier(), Some(FROZEN_FRONTIER));
|
||||
}
|
||||
|
||||
println!(
|
||||
"SCANNER_MEMORY_BUDGET {}",
|
||||
serde_json::json!({
|
||||
"schema": 1,
|
||||
"measurement_point": "folder_walk_return_before_bucket_finalization",
|
||||
"scenario": if coverage_gap { "seeded_coverage_gap" } else { "healthy_forward_sweep" },
|
||||
"synthetic_gap_control": coverage_gap,
|
||||
"prefixes": prefixes,
|
||||
"objects_per_prefix": OBJECTS_PER_PREFIX,
|
||||
"objects": expected_objects,
|
||||
"object_bytes": expected_objects,
|
||||
"validated_frontier": restored.validated_scan_frontier(),
|
||||
"cache": cache_shape(&scanner.new_cache, encoded.len()),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn memory_budget_characterizes_wide_root_and_seeded_coverage_gap() {
|
||||
let prefixes: &[usize] = match std::env::var("RUSTFS_SCANNER_MEMORY_BUDGET_PROFILE") {
|
||||
Err(std::env::VarError::NotPresent) => &[64],
|
||||
Ok(value) if value == "1" => &[64, 128, 256],
|
||||
_ => panic!("RUSTFS_SCANNER_MEMORY_BUDGET_PROFILE must be absent or 1"),
|
||||
};
|
||||
for &prefixes in prefixes {
|
||||
assert!(prefixes * (OBJECTS_PER_PREFIX + 1) < DATA_SCANNER_COMPACT_AT_CHILDREN);
|
||||
for coverage_gap in [false, true] {
|
||||
characterize_memory_budget(prefixes, coverage_gap).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -30,10 +30,6 @@ fn prepare_scanner(scanner: &mut FolderScanner, cache: DataUsageCache) {
|
||||
info: cache.info.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
scanner.update_cache = DataUsageCache {
|
||||
info: cache.info.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
scanner.old_cache = cache;
|
||||
scanner.is_erasure_mode = true;
|
||||
scanner.skip_heal.store(true, Ordering::SeqCst);
|
||||
|
||||
@@ -250,7 +250,6 @@ async fn walk_and_save(observe: bool) -> (Vec<String>, serde_json::Value) {
|
||||
assert_ne!(before, after, "a changed key requires an observable successful fixture write");
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
let paths = Arc::new(Mutex::new(Vec::<String>::new()));
|
||||
let proposed_walked = Arc::new(Mutex::new(BTreeSet::<String>::new()));
|
||||
scanner.update_current_path = Arc::new({
|
||||
|
||||
@@ -148,11 +148,7 @@ async fn erasure_candidate_finds_late_part_without_scanning_data_children() {
|
||||
|
||||
async fn candidate_scanner() -> (FolderScanner, std::path::PathBuf, CachedFolder) {
|
||||
let (mut scanner, root) = build_test_scanner().await;
|
||||
for info in [
|
||||
&mut scanner.old_cache.info,
|
||||
&mut scanner.new_cache.info,
|
||||
&mut scanner.update_cache.info,
|
||||
] {
|
||||
for info in [&mut scanner.old_cache.info, &mut scanner.new_cache.info] {
|
||||
info.name = "bucket".into();
|
||||
}
|
||||
scanner.is_erasure_mode = true;
|
||||
@@ -354,8 +350,7 @@ fn enable_checkpoint(scanner: &mut FolderScanner) {
|
||||
cache.prepare_bucket_checkpoint("bucket", 7, 3, crate::DataUsageCacheSource::new(1, 0), plan, identity);
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
scanner.old_cache = cache.clone();
|
||||
scanner.new_cache = cache.clone();
|
||||
scanner.update_cache = cache;
|
||||
scanner.new_cache = cache;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -462,8 +457,7 @@ async fn erasure_candidate_cancelled_batch_keeps_partial_checkpoint_and_reclassi
|
||||
let bytes = serde_json::to_vec(&snapshot).expect("serialize partial checkpoint");
|
||||
let reloaded: DataUsageCache = serde_json::from_slice(&bytes).expect("reload partial checkpoint");
|
||||
scanner.old_cache = reloaded.clone();
|
||||
scanner.new_cache = reloaded.clone();
|
||||
scanner.update_cache = reloaded;
|
||||
scanner.new_cache = reloaded;
|
||||
scanner.raw_enumeration_progress.clear();
|
||||
let next_budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, Default::default());
|
||||
scanner.budget = next_budget.clone();
|
||||
|
||||
@@ -110,7 +110,7 @@ fn mrf_ownership_retry_due_boundaries_and_priority_are_bounded() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_ownership_full_hint_table_has_bounded_multicycle_work_and_sync() {
|
||||
async fn mrf_ownership_full_hint_table_has_bounded_multicycle_work() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
|
||||
let base = 1_700_000_000;
|
||||
@@ -131,39 +131,23 @@ async fn mrf_ownership_full_hint_table_has_bounded_multicycle_work_and_sync() {
|
||||
.enumerate()
|
||||
.map(|(index, entry)| (entry.object.clone().expect("object identity"), index))
|
||||
.collect();
|
||||
scanner.sync_pending_heals();
|
||||
let initial_syncs = scanner.pending_heal_sync_count;
|
||||
let mut requests = 0usize;
|
||||
let mut nonempty_batches = 0;
|
||||
for minute in 0..24 * 60 {
|
||||
let now = base + minute * 60;
|
||||
let candidates = pending_scanner_heal_retry_candidates_at(&scanner.new_cache.info.pending_heals, "bucket", now);
|
||||
assert!(candidates.len() <= MAX_PENDING_SCANNER_HEAL_RETRIES_PER_BUCKET);
|
||||
if candidates.is_empty() {
|
||||
continue;
|
||||
for candidate in candidates {
|
||||
assert_eq!(
|
||||
build_pending_scanner_heal_request(&candidate)
|
||||
.expect("retry request")
|
||||
.priority,
|
||||
HealChannelPriority::Low
|
||||
);
|
||||
let index = indices[candidate.object.as_ref().expect("object identity")];
|
||||
record_pending_heal_attempt(&mut scanner.new_cache.info.pending_heals[index], now);
|
||||
observe_pending_heal_admission(&mut scanner.new_cache.info.pending_heals[index], HealAdmissionResult::Accepted);
|
||||
requests += 1;
|
||||
}
|
||||
nonempty_batches += 1;
|
||||
let before = scanner.pending_heal_sync_count;
|
||||
{
|
||||
let batch = PendingHealSyncBatch::new(&mut scanner);
|
||||
for candidate in candidates {
|
||||
assert_eq!(
|
||||
build_pending_scanner_heal_request(&candidate)
|
||||
.expect("retry request")
|
||||
.priority,
|
||||
HealChannelPriority::Low
|
||||
);
|
||||
let index = indices[candidate.object.as_ref().expect("object identity")];
|
||||
record_pending_heal_attempt(&mut batch.scanner.new_cache.info.pending_heals[index], now);
|
||||
observe_pending_heal_admission(
|
||||
&mut batch.scanner.new_cache.info.pending_heals[index],
|
||||
HealAdmissionResult::Accepted,
|
||||
);
|
||||
batch.scanner.sync_pending_heals();
|
||||
requests += 1;
|
||||
}
|
||||
}
|
||||
assert_eq!(scanner.pending_heal_sync_count, before + 1, "one table clone per changed retry batch");
|
||||
assert_eq!(scanner.new_cache.info.pending_heals.len(), MAX_PENDING_SCANNER_HEALS_PER_BUCKET);
|
||||
}
|
||||
assert!(requests >= MAX_PENDING_SCANNER_HEALS_PER_BUCKET, "every retained hint receives a retry");
|
||||
@@ -172,12 +156,13 @@ async fn mrf_ownership_full_hint_table_has_bounded_multicycle_work_and_sync() {
|
||||
"15min..6h backoff bounds repeated work within 24h"
|
||||
);
|
||||
assert!(scanner.new_cache.info.pending_heals.iter().all(|entry| entry.attempts >= 2));
|
||||
assert_eq!(scanner.pending_heal_sync_count - initial_syncs, nonempty_batches);
|
||||
assert_eq!(scanner.update_cache.info.pending_heals, scanner.new_cache.info.pending_heals);
|
||||
let restored =
|
||||
DataUsageCache::unmarshal(&scanner.new_cache.marshal_msg().expect("encode retry ledger")).expect("reload retry ledger");
|
||||
assert_eq!(restored.info.pending_heals, scanner.new_cache.info.pending_heals);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_ownership_cancelled_batch_restores_sync_without_per_item_clones() {
|
||||
async fn mrf_ownership_cancelled_retry_retains_canonical_ledger() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
|
||||
scanner
|
||||
@@ -185,34 +170,45 @@ async fn mrf_ownership_cancelled_batch_restores_sync_without_per_item_clones() {
|
||||
.info
|
||||
.pending_heals
|
||||
.push(pending_heal(PendingScannerHealKind::Object, "bucket", Some("object"), None, 1, 1));
|
||||
let before = scanner.pending_heal_sync_count;
|
||||
let mut work = Box::pin(async {
|
||||
let batch = PendingHealSyncBatch::new(&mut scanner);
|
||||
record_pending_heal_attempt(&mut batch.scanner.new_cache.info.pending_heals[0], 100);
|
||||
batch.scanner.sync_pending_heals();
|
||||
batch.scanner.sync_pending_heals();
|
||||
record_pending_heal_attempt(&mut scanner.new_cache.info.pending_heals[0], 100);
|
||||
scanner.update_pending_scanner_heal_after_admission(
|
||||
PendingScannerHealKind::Object,
|
||||
"bucket",
|
||||
Some("object"),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
HealAdmissionResult::Accepted,
|
||||
);
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
assert!(futures::poll!(&mut work).is_pending());
|
||||
drop(work);
|
||||
assert!(!scanner.pending_heal_sync_deferred);
|
||||
assert!(!scanner.pending_heal_batch_dirty);
|
||||
assert_eq!(scanner.pending_heal_sync_count, before + 1);
|
||||
assert!(scanner.pending_heals_changed, "cancellation must retain the dirty marker");
|
||||
assert_eq!(scanner.new_cache.info.pending_heals[0].attempts, 2);
|
||||
assert!(pending_scanner_heal_retry_candidates_at(&scanner.new_cache.info.pending_heals, "bucket", 101).is_empty());
|
||||
assert_eq!(scanner.update_cache.info.pending_heals, scanner.new_cache.info.pending_heals);
|
||||
let restored = DataUsageCache::unmarshal(&scanner.new_cache.marshal_msg().expect("encode cancelled retry"))
|
||||
.expect("reload cancelled retry");
|
||||
assert_eq!(restored.info.pending_heals, scanner.new_cache.info.pending_heals);
|
||||
|
||||
scanner.pending_heals_changed = false;
|
||||
let result: std::result::Result<(), &'static str> = async {
|
||||
let batch = PendingHealSyncBatch::new(&mut scanner);
|
||||
record_pending_heal_attempt(&mut batch.scanner.new_cache.info.pending_heals[0], 200);
|
||||
observe_pending_heal_admission(&mut batch.scanner.new_cache.info.pending_heals[0], HealAdmissionResult::Merged);
|
||||
batch.scanner.sync_pending_heals();
|
||||
Err("injected retry batch failure")
|
||||
record_pending_heal_attempt(&mut scanner.new_cache.info.pending_heals[0], 200);
|
||||
scanner.update_pending_scanner_heal_after_admission(
|
||||
PendingScannerHealKind::Object,
|
||||
"bucket",
|
||||
Some("object"),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
HealAdmissionResult::Merged,
|
||||
);
|
||||
Err("injected retry failure")
|
||||
}
|
||||
.await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(scanner.pending_heal_sync_count, before + 2);
|
||||
assert!(!scanner.pending_heal_sync_deferred);
|
||||
assert_eq!(scanner.update_cache.info.pending_heals, scanner.new_cache.info.pending_heals);
|
||||
assert!(scanner.pending_heals_changed, "an error must retain the dirty marker");
|
||||
assert_eq!(scanner.new_cache.info.pending_heals[0].attempts, 3);
|
||||
assert_eq!(scanner.new_cache.info.pending_heals[0].last_admission_result, "merged");
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -436,7 +432,6 @@ async fn mrf_ownership_manager_completion_preserves_scanner_pending() {
|
||||
let _guard = TestGuard::new(u64::MAX, usize::MAX, &mut scanner, temp_dir);
|
||||
let bucket = format!("mrf-ownership-{}", Uuid::new_v4());
|
||||
scanner.new_cache.info.name = bucket.clone();
|
||||
scanner.update_cache.info.name = bucket.clone();
|
||||
scanner.heal_object_select = 1;
|
||||
let storage = Arc::new(NoticeStorage {
|
||||
bucket_incarnation_id: Uuid::new_v4(),
|
||||
@@ -521,12 +516,12 @@ async fn mrf_ownership_manager_completion_preserves_scanner_pending() {
|
||||
"the second object call remains blocked without a duplicate execution"
|
||||
);
|
||||
note_mrf_repaired(&bucket, object, Some(*version.as_bytes()));
|
||||
let syncs_before_retry = scanner.pending_heal_sync_count;
|
||||
scanner.pending_heals_changed = false;
|
||||
scanner
|
||||
.retry_pending_scanner_heals()
|
||||
.await
|
||||
.expect("real scanner ledger retry");
|
||||
assert_eq!(scanner.pending_heal_sync_count, syncs_before_retry + 1, "the real retry batch syncs once");
|
||||
assert!(scanner.pending_heals_changed, "the real retry marks its canonical ledger for persistence");
|
||||
assert_eq!(
|
||||
scanner.new_cache.info.pending_heals.len(),
|
||||
index + 1,
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn usage_updates_preserve_totals_without_allocating_an_unrequested_cache() {
|
||||
for observed in [false, true] {
|
||||
let (mut scanner, root) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.sleeper = DynamicSleeper::new(rustfs_config::ScannerSpeed::Fastest);
|
||||
scanner.skip_heal.store(true, Ordering::SeqCst);
|
||||
for prefix in ["existing", "new"] {
|
||||
for index in 0..8 {
|
||||
let object = format!("{prefix}/{index:04}");
|
||||
write_test_object_metadata_bytes(&root, "bucket", &object, &metadata_for_object_version("bucket", &object, None))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
|
||||
scanner
|
||||
.old_cache
|
||||
.replace("bucket/existing", "bucket", DataUsageEntry::default());
|
||||
scanner.old_cache.replace(
|
||||
"bucket/existing/0000",
|
||||
"bucket/existing",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 99,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let (sender, mut receiver) = mpsc::channel(2);
|
||||
if observed {
|
||||
scanner.usage_updates = Some(ScannerUsageUpdates::new(sender, "bucket".to_string()));
|
||||
}
|
||||
scanner
|
||||
.scan_folder(
|
||||
CancellationToken::new(),
|
||||
CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut DataUsageEntry::default(),
|
||||
)
|
||||
.await
|
||||
.expect("scan existing and new prefixes");
|
||||
let totals = scanner
|
||||
.new_cache
|
||||
.checked_flatten_complete_scope("bucket")
|
||||
.expect("complete accounting");
|
||||
assert_eq!((totals.objects, totals.size), (16, 16));
|
||||
assert_eq!(
|
||||
scanner
|
||||
.old_cache
|
||||
.find("bucket/existing/0000")
|
||||
.expect("retained baseline")
|
||||
.size,
|
||||
99
|
||||
);
|
||||
if observed {
|
||||
while receiver.try_recv().is_ok() {}
|
||||
let updates = scanner.usage_updates.as_mut().expect("requested progress owner");
|
||||
let progress = updates.cache.checked_flatten("bucket").expect("progress cache is connected");
|
||||
assert_eq!((progress.objects, progress.size), (16, 16));
|
||||
updates.last_update = SystemTime::UNIX_EPOCH;
|
||||
scanner.send_update().await;
|
||||
let update = receiver.try_recv().expect("requested aggregate update");
|
||||
assert_eq!((update.objects, update.size), (16, 16));
|
||||
scanner.send_update().await;
|
||||
assert!(receiver.try_recv().is_err(), "unchanged minute interval must not emit another aggregate");
|
||||
} else {
|
||||
assert!(scanner.usage_updates.is_none(), "unobserved scans must never allocate a progress cache");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_updates_do_not_mirror_durable_ledgers_or_checkpoint_metadata() {
|
||||
let (mut scanner, root) = build_test_scanner().await;
|
||||
let _guard = TestGuard { temp_dir: Some(root) };
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
let (sender, _receiver) = mpsc::channel(1);
|
||||
scanner.usage_updates = Some(ScannerUsageUpdates::new(sender, "bucket".to_string()));
|
||||
scanner.record_pending_scanner_heal(
|
||||
PendingScannerHealKind::Object,
|
||||
"bucket",
|
||||
Some("object"),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
HealAdmissionResult::Full,
|
||||
);
|
||||
scanner.record_scan_resume_hint("bucket/object");
|
||||
let mut summary = SizeSummary::default();
|
||||
summary.record_size_reconciliation(SizeReconciliationEntry {
|
||||
key: "unresolved-size".to_string(),
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
reason: "invalid_declared_size".to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
scanner.apply_size_reconciliation(&summary);
|
||||
scanner.finish_size_reconciliation_batch();
|
||||
let restored = DataUsageCache::unmarshal(&scanner.new_cache.marshal_msg().expect("encode canonical state"))
|
||||
.expect("reload canonical state");
|
||||
assert_eq!(restored.info.pending_heals.len(), 1);
|
||||
assert_eq!(restored.info.size_reconciliation.len(), 1);
|
||||
assert_eq!(restored.info.scan_checkpoint, scanner.new_cache.info.scan_checkpoint);
|
||||
let progress = &scanner.usage_updates.as_ref().expect("requested progress owner").cache;
|
||||
assert_eq!(progress.info.name, "bucket");
|
||||
assert!(progress.info.pending_heals.is_empty());
|
||||
assert!(progress.info.size_reconciliation.is_empty());
|
||||
assert!(progress.info.scan_checkpoint.is_none());
|
||||
assert!(progress.info.scan_resume_after.is_none());
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
[package]
|
||||
name = "zstd"
|
||||
version = "0.13.3"
|
||||
publish = false
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
description = "RustFS internal adapter for zstd 0.13 consumers using the fixed codec"
|
||||
|
||||
[dependencies]
|
||||
zstd_fixed = { package = "zstd", version = "=0.14.3", default-features = false }
|
||||
|
||||
[features]
|
||||
default = ["legacy", "arrays", "zdict_builder"]
|
||||
arrays = ["zstd_fixed/arrays"]
|
||||
bindgen = ["zstd_fixed/bindgen"]
|
||||
debug = ["zstd_fixed/debug"]
|
||||
doc-cfg = ["zstd_fixed/doc-cfg"]
|
||||
experimental = ["zstd_fixed/experimental"]
|
||||
fat-lto = ["zstd_fixed/fat-lto"]
|
||||
legacy = ["zstd_fixed/legacy"]
|
||||
no_asm = ["zstd_fixed/no_asm"]
|
||||
pkg-config = ["zstd_fixed/pkg-config"]
|
||||
thin = ["zstd_fixed/thin"]
|
||||
thin-lto = ["zstd_fixed/thin-lto"]
|
||||
wasm = ["zstd_fixed/wasm"]
|
||||
zdict_builder = ["zstd_fixed/zdict_builder"]
|
||||
zstdmt = ["zstd_fixed/zstdmt"]
|
||||
|
||||
[dev-dependencies]
|
||||
zstd-safe = { version = "8.2.0", features = ["seekable"] }
|
||||
toml = "0.9"
|
||||
semver = "1.0.27"
|
||||
apache-avro.workspace = true
|
||||
arrow-array = "59.3.2"
|
||||
arrow-ipc = { version = "59.3.2", features = ["zstd"] }
|
||||
arrow-schema = "59.3.2"
|
||||
parquet = { version = "59.3.2", default-features = false, features = ["arrow", "zstd"] }
|
||||
kafka-protocol = { version = "0.18.0", default-features = false, features = ["zstd"] }
|
||||
bytes.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user