mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-27 14:27:10 +00:00
Compare commits
73 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3e337bad97 | |||
| 12a23859a1 | |||
| 081d3b1469 | |||
| 56a80f7b62 | |||
| 0a366d967b | |||
| 31c9d85645 | |||
| b1f7fb2769 | |||
| c20448dfce | |||
| 887716ca61 | |||
| dae95d7afe | |||
| b0f60ed8d9 | |||
| 78c803e1ef | |||
| a2d6aa5a7f | |||
| 553be4b4c6 | |||
| 96858cd1f1 | |||
| 3d357cec44 | |||
| 9bce929ae5 | |||
| a74ddfab3c | |||
| 1676878676 | |||
| b3031bed59 | |||
| c86faf193a | |||
| e30012c4ef | |||
| 2babf7624d | |||
| 757b751482 | |||
| 09fb813a00 | |||
| 738e5fa14a | |||
| 8380466f3a | |||
| 170ec8d983 | |||
| bd0671ba78 | |||
| f954164048 | |||
| 6afdec9fe1 | |||
| 87bd36b313 | |||
| a4c02d5ac3 | |||
| 62ff96774f | |||
| 85c706cac0 | |||
| 7951bb6298 | |||
| a3230599a8 | |||
| 1c7fc818dd | |||
| 5475bd06bc | |||
| a3ba311ab0 | |||
| e517d35fa2 | |||
| 1b7a197267 | |||
| 3b416b76cd | |||
| bebee3d5d0 | |||
| 1aa9be87d1 | |||
| e224eb48e5 | |||
| 16cf19817e | |||
| 32fd1e704a | |||
| ac197ea655 | |||
| e18ed73942 | |||
| 3c131f2b2e | |||
| c5f4b5a4f7 | |||
| 03fb3d78a0 | |||
| b8126e50f7 | |||
| 59fb72aa82 | |||
| 9de01c9170 | |||
| 2ec26fa4b9 | |||
| f4507178a3 | |||
| 9b4cc40abd | |||
| b14ff4eeb1 | |||
| ae9b5cbabf | |||
| 6c78b0d996 | |||
| 0066e34764 | |||
| fbddcc2965 | |||
| b1793b55a0 | |||
| 835bd3f52d | |||
| cb32d55878 | |||
| 864e58592c | |||
| 3016209330 | |||
| 64e0eeba91 | |||
| 35fa7488a5 | |||
| 1b77846f8e | |||
| b0c861841e |
@@ -6,17 +6,7 @@
|
||||
- Route every new sync/flush through the durability-mode and bucket override
|
||||
gates; mode `none` must not pay the new fsync.
|
||||
- Keep blocking filesystem/CPU work off async runtime threads, but do not split
|
||||
one small operation into many `spawn_blocking` round trips; coalesce `open`,
|
||||
`read`/`write`, `fsync`, and `close` into one blocking closure per disk.
|
||||
- Check for per-disk syscall amplification (`disks × parts/files`): pin a
|
||||
directory once (`openat2` on Linux) and reuse the handle across sibling reads
|
||||
instead of re-walking ancestor components per file or re-opening identical
|
||||
`source_parent == destination_parent` directories.
|
||||
- After `fstat`/`file.metadata()`, require `Vec::with_capacity(len + 1)` before
|
||||
`read_to_end` so small metadata reads do not trigger 6–7 exponential-growth
|
||||
`read()` syscalls.
|
||||
- Avoid redundant `store.load()` or `root_missing()` round-trips under the same
|
||||
exclusive lock when the state or original bytes were just loaded or persisted.
|
||||
one small operation into many `spawn_blocking` round trips.
|
||||
- Measure lock hold time across I/O and compare acquisition order for ABBA.
|
||||
- Keep cleanup, extra stat/rename, and diagnostics out of the PUT commit critical
|
||||
section when they need not be there.
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-linux=6190506b9e4bdb60fd392544a7bc47be708077bb4fe7a2ef689d047455e1af86
|
||||
sha256-darwin=6190506b9e4bdb60fd392544a7bc47be708077bb4fe7a2ef689d047455e1af86
|
||||
sha256-linux=9bd4b5ed4d135c6f61e0d144ddeb6b897fd5a6037b51f31088693cf1b23e0718
|
||||
sha256-darwin=9bd4b5ed4d135c6f61e0d144ddeb6b897fd5a6037b51f31088693cf1b23e0718
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=a67a347f26be186c598b5beaf68e6bcbe1855350a35e1423dd5b8a81bcba06a5
|
||||
sha256-linux=ad4fabe811c6cca1aba89fad34ccd404b5832a6a955875c2826f6e0fa1ea4f1f
|
||||
sha256-darwin=da00472db3d8181ed413b8e9919f43e86fe644ccee444d92035ce189a33ab037
|
||||
sha256-linux=42e95c9480e5bfa0d717be6652c6500b43e3a337cb63907869104bb155b5bd91
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81
|
||||
sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db
|
||||
sha256-darwin=347a0d4e5e547d5388703062ce66a66167a845b6e57ec591ccb4c732a567c68d
|
||||
sha256-linux=4f9efa47194279185bb68249e82c788fa147abe944b6c764050e3d47f31e7926
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
{
|
||||
"invariant": "list-uncommitted-version",
|
||||
"suite": "rustfs-filemeta",
|
||||
"name": "metacache::tests::resolve_merge_keeps_the_newest_version_at_listing_quorum"
|
||||
"name": "metacache::tests::resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge"
|
||||
},
|
||||
{
|
||||
"invariant": "minio-object-fixture",
|
||||
|
||||
@@ -52,7 +52,6 @@ script-tests: ## Run shell script tests
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_glibc.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_upgrade_workflow.py
|
||||
@@ -64,6 +63,8 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
|
||||
|
||||
@@ -68,17 +68,6 @@ command = ['sh', '-c', 'echo RUST_MIN_STACK=33554432 >> "$NEXTEST_ENV"']
|
||||
# boundaries. Reserve the run's capacity so unrelated fixtures cannot starve
|
||||
# their existing deadlines and typed-failure assertions (rustfs/backlog#2706).
|
||||
# Keep each test's internal concurrency and zero-retry policy unchanged.
|
||||
[[profile.default.overrides]]
|
||||
filter = """
|
||||
package(rustfs-ecstore) & (
|
||||
test(/^set_disk::ops::multipart::tests::complete_multipart_(quorum_success_releases_logical_guards_before_physical_tail|waits_for_tail_before_releasing_guards_and_marking_capacity)$/)
|
||||
| test(=store::heal::tests::full_format_heal_lost_after_last_save_does_not_publish_or_renew_and_latches)
|
||||
| test(/^store::init::tests::(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page|dispatch_manifest_shutdown_(drains_admitted_delete_batch_without_tail_admission|stops_aborting_validation_tail_reads|stops_authorized_tail_reads_before_receipt_or_cas))$/)
|
||||
| (binary(metadata_read_lock_reuse_test) & test(=replica_multipart_completes_while_metadata_writer_waits_for_its_snapshot))
|
||||
)
|
||||
"""
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & test(/^connect::diagnostics::schedule::tests::/)'
|
||||
threads-required = "num-test-threads"
|
||||
@@ -362,17 +351,6 @@ path = "junit.xml"
|
||||
|
||||
# Match the local durable-progress reservations. All members, deadlines,
|
||||
# assertions, and internal concurrency remain enabled without retries.
|
||||
[[profile.ci.overrides]]
|
||||
filter = """
|
||||
package(rustfs-ecstore) & (
|
||||
test(/^set_disk::ops::multipart::tests::complete_multipart_(quorum_success_releases_logical_guards_before_physical_tail|waits_for_tail_before_releasing_guards_and_marking_capacity)$/)
|
||||
| test(=store::heal::tests::full_format_heal_lost_after_last_save_does_not_publish_or_renew_and_latches)
|
||||
| test(/^store::init::tests::(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page|dispatch_manifest_shutdown_(drains_admitted_delete_batch_without_tail_admission|stops_aborting_validation_tail_reads|stops_authorized_tail_reads_before_receipt_or_cas))$/)
|
||||
| (binary(metadata_read_lock_reuse_test) & test(=replica_multipart_completes_while_metadata_writer_waits_for_its_snapshot))
|
||||
)
|
||||
"""
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & test(/^connect::diagnostics::schedule::tests::/)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
@@ -74,6 +74,7 @@ Notes:
|
||||
- `.github/workflows/e2e-s3tests.yml` (weekly full s3-tests sweep)
|
||||
- `.github/workflows/mint.yml` (weekly multi-SDK mint run)
|
||||
- `.github/workflows/fuzz.yml` (nightly fuzz corpus jobs)
|
||||
- `.github/workflows/performance-ab.yml` (nightly warp A/B gate)
|
||||
- **Pending:** the ci-7 e2e nightly-full workflow does not exist yet
|
||||
(backlog#1149 ci-7). When it lands, wire it up with the snippet above.
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
"never_ran_grace_until": "2026-09-08T00:00:00Z"
|
||||
},
|
||||
{ "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
|
||||
{
|
||||
"workflow": ".github/workflows/runner-hygiene.yml",
|
||||
"max_age_hours": 792,
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
This repository is public. For a public repository, GitHub's standard hosted runners (`ubuntu-latest`, `windows-latest`, `macos-latest`) cost nothing and have no minute cap, whatever plan the organisation is on. GitHub bills only two things: larger hosted runners, which have custom labels and are not used here, and standard runners in private repositories.
|
||||
|
||||
Every other label in this repository is capacity RustFS pays for. ARC pods and lab machines accrue cost while the machine exists. A Ubicloud label is metered for the minutes a job holds its VM:
|
||||
Every other label in this repository is capacity RustFS pays for by the hour whether or not a job is running well:
|
||||
|
||||
| Label | Paid by | What it is | Use it for |
|
||||
|---|---|---|---|
|
||||
@@ -15,7 +15,7 @@ Every other label in this repository is capacity RustFS pays for. ARC pods and l
|
||||
| `windows-latest`, `macos-latest` | nobody | GitHub-hosted standard runners | Windows and macOS targets |
|
||||
| `sm-standard-2` | RustFS | ARC pod, Ubuntu 24.04, minimal image: no `gh`, no C toolchain; Docker only on the `dind-` label | only a job that a hosted runner cannot run (see the placement rule) |
|
||||
| `sm-standard-4` | RustFS | ARC pod, Ubuntu 24.04, 14 CPUs and 28 GB (the label is not the core count), large disk | workspace-scale Cargo builds |
|
||||
| `ubicloud-standard-16-arm` | RustFS | Ubicloud VM, alias of `ubicloud-standard-16-arm-ubuntu-2404`: Ubuntu 24.04 arm64, 16 vCPU, 48 GB, 300 GB disk. Reached through the `build.yml` matrix, so `actionlint.yaml` does not list it | native aarch64 release builds (`linux-aarch64-musl`, `linux-aarch64-gnu`) |
|
||||
| `sm-standard-4-arm` | RustFS | ARC pod, aarch64; reached through the `build.yml` matrix, so `actionlint.yaml` does not list it | native aarch64 release builds |
|
||||
| `dind-sm-standard-2` | RustFS | ARC pod with Docker-in-Docker | image builds that need `buildx` and a registry push |
|
||||
| `smoke-testing`, `pf-testing` | RustFS | long-lived lab machines | the packaged functional chain and performance suites |
|
||||
|
||||
@@ -40,7 +40,7 @@ A hosted runner is also the only choice for a few things the pods cannot do: `mo
|
||||
|
||||
- Every `shared-key` has exactly one writer. `cache-warm.yml` owns `ci-dev`, `ci-feat-rio`, `ci-feat-proto` and `ci-uring`; every other `./.github/actions/setup` call states `cache-save-if` explicitly, and `scripts/security/check_cache_save_if.sh` fails the build when one does not.
|
||||
- All readers of a key run on the same image as its writer. Keys that live on hosted runners say so in their name (`ci-uring` is warmed on `ubuntu-latest`; `ci-e2e-distributed-hosted`, `e2e-upgrade-server-hosted`, `e2e-upgrade-tests-hosted`).
|
||||
- Moving a job between pools therefore means renaming its key. The new key is empty until a run from `main` (a scheduled run, or a manual dispatch of that workflow from `main`) saves it; until then that lane does a cold build. The `build.yml` aarch64 release legs append `-ubicloud-arm` for this reason. `cache-warm.yml` does not write those keys.
|
||||
- Moving a job between pools therefore means renaming its key. The new key is empty until a run from `main` (a scheduled run, or a manual dispatch of that workflow from `main`) saves it; until then that lane does a cold build.
|
||||
- `CARGO_BUILD_JOBS` and every other `CARGO*`, `CC*`, `CFLAGS*`, `CXX*`, `CMAKE*` and `RUST*` variable is hashed into the key. `ci.yml` sets `CARGO_BUILD_JOBS` at step level for that reason; promoting it to job level rotates the key for every lane that shares it.
|
||||
|
||||
## Where self-hosted minutes leak
|
||||
@@ -62,7 +62,7 @@ gh api "repos/rustfs/rustfs/actions/runs/<run_id>/jobs?per_page=100" \
|
||||
--jq '.jobs[] | "\(.conclusion)\t\(.labels | join(","))\t\(.started_at)\t\(.completed_at)\t\(.name)"'
|
||||
```
|
||||
|
||||
Only rows whose label starts with `sm-`, `dind-`, `ubicloud-`, `smoke-` or `pf-` cost anything. At the October 2026 review, a full pull-request run of `ci.yml` spent about 150 self-hosted minutes, a third of it in jobs that never compiled; treat that as the scale of what a placement mistake costs, not as a current figure.
|
||||
Only rows whose label starts with `sm-`, `dind-`, `smoke-` or `pf-` cost anything. At the October 2026 review, a full pull-request run of `ci.yml` spent about 150 self-hosted minutes, a third of it in jobs that never compiled; treat that as the scale of what a placement mistake costs, not as a current figure.
|
||||
|
||||
## Checklist for a new or moved job
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ on:
|
||||
- 'scripts/test_package_service_scripts.sh'
|
||||
- 'scripts/test_package_versions.sh'
|
||||
- 'scripts/test_docker_workflow.py'
|
||||
- 'scripts/security/check_performance_ab_workflow.sh'
|
||||
- 'scripts/security/check_preview_release_workflow.sh'
|
||||
- 'scripts/security/check_tier_artifact_workflow.sh'
|
||||
- 'scripts/security/check_workflow_pins.sh'
|
||||
@@ -47,6 +48,7 @@ on:
|
||||
- 'scripts/test_package_service_scripts.sh'
|
||||
- 'scripts/test_package_versions.sh'
|
||||
- 'scripts/test_docker_workflow.py'
|
||||
- 'scripts/security/check_performance_ab_workflow.sh'
|
||||
- 'scripts/security/check_preview_release_workflow.sh'
|
||||
- 'scripts/security/check_tier_artifact_workflow.sh'
|
||||
- 'scripts/security/check_workflow_pins.sh'
|
||||
@@ -153,6 +155,9 @@ jobs:
|
||||
- name: Check preview release workflow policy
|
||||
run: ./scripts/security/check_preview_release_workflow.sh
|
||||
|
||||
- name: Check performance A/B workflow trust boundary
|
||||
run: ./scripts/security/check_performance_ab_workflow.sh
|
||||
|
||||
- name: Check tier evidence workflow isolation
|
||||
run: ./scripts/security/check_tier_artifact_workflow.sh
|
||||
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
name: BTF prototype reference
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "scripts/inspect_btf_prototypes.py"
|
||||
- "scripts/test_inspect_btf_prototypes.py"
|
||||
- "scripts/fixtures/btf_prototypes.c"
|
||||
- ".github/workflows/btf-prototype-reference.yml"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: btf-prototype-reference-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
reference:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Check prototype and CLI contracts
|
||||
run: python3 -B scripts/test_inspect_btf_prototypes.py
|
||||
- name: Install only missing native reference tools
|
||||
run: |
|
||||
select_bpftool() {
|
||||
local candidate
|
||||
local candidates=()
|
||||
if command -v bpftool >/dev/null; then
|
||||
candidates+=("$(command -v bpftool)")
|
||||
fi
|
||||
for candidate in /usr/lib/linux-tools-*/bpftool /usr/lib/linux-tools/*/bpftool; do
|
||||
if [[ -x "$candidate" ]]; then
|
||||
candidates+=("$candidate")
|
||||
fi
|
||||
done
|
||||
while IFS= read -r candidate; do
|
||||
if [[ -x "$candidate" ]] && file -Lb "$candidate" | grep -q 'ELF' \
|
||||
&& "$candidate" version >/dev/null 2>&1; then
|
||||
printf '%s\n' "$candidate"
|
||||
return 0
|
||||
fi
|
||||
done < <(printf '%s\n' "${candidates[@]}" | sort -Vr)
|
||||
return 1
|
||||
}
|
||||
packages=()
|
||||
command -v clang >/dev/null || packages+=(clang)
|
||||
command -v llvm-objcopy >/dev/null || packages+=(llvm)
|
||||
select_bpftool >/dev/null || packages+=(linux-tools-generic)
|
||||
if (( ${#packages[@]} )); then
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y "${packages[@]}"
|
||||
fi
|
||||
# File parsing needs a working binary, not a host-kernel wrapper.
|
||||
selected_bpftool="$(select_bpftool)"
|
||||
"$selected_bpftool" version
|
||||
printf 'BPFTOOL=%s\n' "$selected_bpftool" >> "$GITHUB_ENV"
|
||||
- name: Compare compiled declarations with bpftool
|
||||
run: |
|
||||
clang -target bpf -O2 -g -c scripts/fixtures/btf_prototypes.c -o "$RUNNER_TEMP/prototypes.o"
|
||||
llvm-objcopy --dump-section ".BTF=$RUNNER_TEMP/prototypes.btf" "$RUNNER_TEMP/prototypes.o"
|
||||
python3 -B scripts/test_inspect_btf_prototypes.py --native-reference "$RUNNER_TEMP/prototypes.btf"
|
||||
@@ -181,18 +181,12 @@ jobs:
|
||||
selected="all"
|
||||
fi
|
||||
|
||||
# linux-aarch64-* need a real aarch64 host (placement rule: hardware)
|
||||
# so the packaged binary and console smoke execute the artifact.
|
||||
# ubicloud-standard-16-arm is that host: Ubuntu 24.04 arm64, alias of
|
||||
# ubicloud-standard-16-arm-ubuntu-2404 (16 vCPU, 48 GB). linux-x86_64-* stay on the
|
||||
# ARC sm-standard-4 pool. GNU rows keep cross=true so cargo-zigbuild
|
||||
# pins the glibc 2.31 floor on the same architecture; musl stays a
|
||||
# native cargo build.
|
||||
|
||||
all='{"include":[
|
||||
{"target_id":"linux-x86_64-musl","os":"sm-standard-4","target":"x86_64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
|
||||
{"target_id":"linux-aarch64-musl","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
|
||||
{"target_id":"linux-aarch64-musl","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
|
||||
{"target_id":"linux-x86_64-gnu","os":"sm-standard-4","target":"x86_64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"linux-aarch64-gnu","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"linux-aarch64-gnu","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
|
||||
{"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":"","features":"","glibc":""}
|
||||
]}'
|
||||
@@ -250,11 +244,7 @@ jobs:
|
||||
with:
|
||||
rust-version: stable
|
||||
target: ${{ matrix.target }}
|
||||
# rust-cache keys include OS and architecture but not the runner image.
|
||||
# The Ubicloud aarch64 legs therefore use their own suffix so they
|
||||
# cannot restore a cache written on a different arm64 image.
|
||||
# x86_64, macOS, and Windows keys stay unsuffixed.
|
||||
cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}${{ matrix.os == 'ubicloud-standard-16-arm' && '-ubicloud-arm' || '' }}
|
||||
cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}
|
||||
# main only. A cache saved on refs/tags/X is scoped to that tag: no
|
||||
# other tag, no main run and no PR can restore it, so every release
|
||||
# cycle wrote up to 12 entries of 1-2GB (preview tag plus final tag,
|
||||
@@ -774,9 +764,9 @@ jobs:
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ARC pods (the x86_64 legs) do not ship the aws CLI. Ubicloud arm64
|
||||
# images follow the GitHub runner image and often do. Install the
|
||||
# official CLI bundle when it is missing so R2 uploads work on either.
|
||||
# The self-hosted Linux runners do not ship the aws CLI (GitHub-hosted
|
||||
# images did). Install the official CLI bundle on demand so R2 uploads
|
||||
# survive a fresh runner without relying on a distro package.
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
echo "aws CLI not found on runner; installing..."
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
|
||||
@@ -45,7 +45,7 @@ on:
|
||||
# 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
|
||||
# build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
|
||||
# (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
|
||||
# and e2e-replication-nightly (04:00) lanes.
|
||||
# e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
|
||||
- cron: "43 7 * * 0"
|
||||
|
||||
concurrency:
|
||||
|
||||
@@ -38,8 +38,8 @@ name: e2e-nightly
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00)
|
||||
# and stale (01:30).
|
||||
# 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00),
|
||||
# stale (01:30) and performance-ab (06:00).
|
||||
- cron: "29 4 * * *"
|
||||
|
||||
# Only alert-on-failure needs more than read access; it declares its own
|
||||
|
||||
@@ -0,0 +1,384 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Warp A/B budget gate for the hotpath series (rustfs/backlog#935 HP-14, item 4).
|
||||
#
|
||||
# Two entry points, honestly scoped:
|
||||
# * schedule (nightly, on main): post-merge detection — catches a regression
|
||||
# within 24h of landing, not before merge.
|
||||
# * workflow_dispatch: an explicitly selected trusted ref.
|
||||
# The dispatch input can run the gate with --allow-regression so a deliberate
|
||||
# correctness cost (e.g. the #4221 fsync durability fix) is recorded, not
|
||||
# blocked (rustfs/backlog#935 correction 1).
|
||||
|
||||
name: Performance A/B
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "31 6 * * *" # 06:31 UTC nightly, against main
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
duration:
|
||||
description: "warp duration per round"
|
||||
required: false
|
||||
default: "12s"
|
||||
type: string
|
||||
allow_regression:
|
||||
description: "Pass the gate despite a FAIL (deliberate tradeoff)"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
warp-ab:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Warp A/B budget gate
|
||||
runs-on: sm-standard-2
|
||||
# A normal nightly restores the last successful binary and builds only the
|
||||
# candidate; daily access keeps that cache warm. A cache miss may build both
|
||||
# and needs room for the A/B run plus artifact and cache publication.
|
||||
timeout-minutes: 180
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0 # baseline may be an earlier successful scheduled head
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
|
||||
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Install warp
|
||||
run: |
|
||||
set -euo pipefail
|
||||
WARP_VERSION="1.0.0"
|
||||
curl -fsSL "https://github.com/minio/warp/releases/download/v${WARP_VERSION}/warp_Linux_x86_64.tar.gz" \
|
||||
| sudo tar -xz -C /usr/local/bin warp
|
||||
warp --version
|
||||
|
||||
- name: Decide exemption
|
||||
id: exempt
|
||||
env:
|
||||
INPUT_ALLOW_REGRESSION: ${{ github.event.inputs.allow_regression }}
|
||||
run: |
|
||||
allow="false"
|
||||
if [[ "$INPUT_ALLOW_REGRESSION" == "true" ]]; then
|
||||
allow="true"
|
||||
fi
|
||||
echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# A failed regression run must keep comparing against the last known-good
|
||||
# scheduled head. Otherwise the next nightly would absorb the regression
|
||||
# into its baseline and turn green without a fix.
|
||||
- name: Find last successful scheduled baseline
|
||||
id: scheduled_baseline
|
||||
if: github.event_name == 'schedule'
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
result-encoding: string
|
||||
script: |
|
||||
const { data } = await github.rest.actions.listWorkflowRuns({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
workflow_id: "performance-ab.yml",
|
||||
event: "schedule",
|
||||
status: "success",
|
||||
per_page: 1,
|
||||
});
|
||||
return data.workflow_runs[0]?.head_sha ?? "";
|
||||
|
||||
# Manual runs compare a selected ref with current main. Scheduled runs
|
||||
# compare current main with the last successful scheduled head. With no
|
||||
# history, the first run measures the candidate against itself and seeds
|
||||
# that head only if the complete rig succeeds.
|
||||
- name: Resolve baseline / candidate commits
|
||||
id: commits
|
||||
env:
|
||||
SCHEDULED_BASELINE_SHA: ${{ steps.scheduled_baseline.outputs.result }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
candidate_sha="$(git rev-parse HEAD)"
|
||||
if [[ "${{ github.event_name }}" == "schedule" ]]; then
|
||||
baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}"
|
||||
else
|
||||
baseline_sha="$(git rev-parse origin/main)"
|
||||
fi
|
||||
git cat-file -e "${baseline_sha}^{commit}"
|
||||
if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
|
||||
echo "::error::baseline $baseline_sha is not an ancestor of candidate $candidate_sha; update the selected ref before comparing" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
|
||||
echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
|
||||
echo "baseline commit: $baseline_sha"
|
||||
echo "candidate commit: $candidate_sha"
|
||||
|
||||
# Exact-key restore of the candidate binary saved by its successful
|
||||
# scheduled run. A miss leaves cache-hit unset and falls back to a source
|
||||
# build of that known-good head.
|
||||
- name: Restore cached baseline binary
|
||||
id: baseline_cache
|
||||
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: baseline-bin/rustfs
|
||||
key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}
|
||||
|
||||
# Self-heal: on a nightly/dispatch run where the candidate commit IS the
|
||||
# baseline commit, a cache miss would make the rig build the same commit
|
||||
# twice (~65min per side with the post-#4806 LTO profile — no job budget
|
||||
# fits that). Build it once here, reuse it for both phases, and save it
|
||||
# back to the cache so the next run hits.
|
||||
- name: Build baseline on cache miss (same-commit self-heal)
|
||||
id: selfheal
|
||||
if: >-
|
||||
steps.baseline_cache.outputs.cache-hit != 'true' &&
|
||||
steps.commits.outputs.baseline_sha == steps.commits.outputs.candidate_sha
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cargo build --release --bin rustfs
|
||||
mkdir -p baseline-bin
|
||||
cp target/release/rustfs baseline-bin/rustfs
|
||||
echo "built=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build baseline on cache miss (different candidate)
|
||||
id: baseline_build
|
||||
if: >-
|
||||
steps.baseline_cache.outputs.cache-hit != 'true' &&
|
||||
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
run: |
|
||||
set -euo pipefail
|
||||
baseline_root="$RUNNER_TEMP/rustfs-baseline-${{ github.run_id }}"
|
||||
baseline_target="$RUNNER_TEMP/rustfs-baseline-target-${{ github.run_id }}"
|
||||
git worktree add --detach "$baseline_root" "${{ steps.commits.outputs.baseline_sha }}"
|
||||
cargo build --release --manifest-path "$baseline_root/Cargo.toml" --bin rustfs --target-dir "$baseline_target"
|
||||
mkdir -p baseline-bin
|
||||
cp "$baseline_target/release/rustfs" baseline-bin/rustfs
|
||||
git worktree remove --force "$baseline_root"
|
||||
echo "built=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build candidate binary
|
||||
id: candidate_build
|
||||
if: steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cargo build --release --bin rustfs
|
||||
mkdir -p candidate-bin
|
||||
cp target/release/rustfs candidate-bin/rustfs
|
||||
echo "built=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Save self-healed baseline to cache
|
||||
if: steps.selfheal.outputs.built == 'true' || steps.baseline_build.outputs.built == 'true'
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: baseline-bin/rustfs
|
||||
key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}
|
||||
|
||||
- name: Run warp A/B and gate
|
||||
id: ab
|
||||
env:
|
||||
INPUT_DURATION: ${{ github.event.inputs.duration }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# The formal runner executes A1 baseline -> B1 candidate -> B2 candidate
|
||||
# -> A2 baseline for each workload and drive-sync cell. It requires three
|
||||
# rounds per leg to emit tail latency and error-rate evidence.
|
||||
# --health-timeout 180 outlasts the server's own 120s startup-readiness
|
||||
# budget, which the rig's previous 60s health poll undershot (the first
|
||||
# two nightly failures). perf-6 recalibrates these once the noise study
|
||||
# lands.
|
||||
duration="${INPUT_DURATION:-12s}"
|
||||
baseline_sha="${{ steps.commits.outputs.baseline_sha }}"
|
||||
candidate_sha="${{ steps.commits.outputs.candidate_sha }}"
|
||||
baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}"
|
||||
selfheal_built="${{ steps.selfheal.outputs.built }}"
|
||||
baseline_built="${{ steps.baseline_build.outputs.built }}"
|
||||
candidate_built="${{ steps.candidate_build.outputs.built }}"
|
||||
|
||||
args=(--duration "$duration" --rounds 3 --cooldown 5 --health-timeout 180 --baseline-revision "$baseline_sha" --candidate-revision "$candidate_sha")
|
||||
|
||||
if [[ "$baseline_hit" == "true" || "$selfheal_built" == "true" || "$baseline_built" == "true" ]]; then
|
||||
chmod +x baseline-bin/rustfs
|
||||
base_bin="$PWD/baseline-bin/rustfs"
|
||||
args+=(--baseline-bin "$base_bin")
|
||||
if [[ "$baseline_hit" == "true" ]]; then
|
||||
base_src="actions-cache (rustfs-baseline-$baseline_sha)"
|
||||
elif [[ "$selfheal_built" == "true" ]]; then
|
||||
base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
|
||||
else
|
||||
base_src="isolated baseline source build (saved as rustfs-baseline-$baseline_sha)"
|
||||
fi
|
||||
if [[ "$candidate_sha" == "$baseline_sha" ]]; then
|
||||
# No commits landed since the last successful baseline, so reuse
|
||||
# the one binary for both phases and measure only rig drift.
|
||||
args+=(--candidate-bin "$base_bin")
|
||||
cand_src="same binary as baseline (same commit)"
|
||||
elif [[ "$candidate_built" == "true" ]]; then
|
||||
chmod +x candidate-bin/rustfs
|
||||
args+=(--candidate-bin "$PWD/candidate-bin/rustfs")
|
||||
cand_src="source build of the checked-out ref"
|
||||
else
|
||||
echo "::error::candidate binary was not built" >&2
|
||||
exit 2
|
||||
fi
|
||||
else
|
||||
echo "::error::baseline binary was not restored or built" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo "baseline binary: $base_src"
|
||||
echo "candidate binary: $cand_src"
|
||||
|
||||
if [[ "${{ steps.exempt.outputs.allow_regression }}" == "true" ]]; then
|
||||
args+=(--allow-regression --exemption-reason "workflow dispatch override")
|
||||
fi
|
||||
# Do not let a gate FAIL abort the job here; capture status and surface
|
||||
# it after the step summary is written.
|
||||
set +e
|
||||
bash scripts/run_hotpath_warp_abba.sh "${args[@]}"
|
||||
echo "status=$?" >> "$GITHUB_OUTPUT"
|
||||
set -e
|
||||
# Locate the newest run dir + gate.md for the summary/comment/artifact
|
||||
# steps. On a startup failure there is no gate.md, but the run dir still
|
||||
# holds server-logs/ for diagnosis.
|
||||
# Run dirs are UTC-timestamp names (no special chars); ls is safe here.
|
||||
# shellcheck disable=SC2012
|
||||
run_dir="$(ls -td target/hotpath-abba/*/ 2>/dev/null | head -n1 || true)"
|
||||
echo "run_dir=${run_dir%/}" >> "$GITHUB_OUTPUT"
|
||||
# shellcheck disable=SC2012
|
||||
gate_md="$(ls -t target/hotpath-abba/*/candidate_gate.md 2>/dev/null | head -n1 || true)"
|
||||
echo "gate_md=$gate_md" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload A/B results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: hotpath-warp-ab-${{ github.run_number }}
|
||||
# Includes per-cell median_summary.csv / baseline_compare.csv, both gates,
|
||||
# and server-logs/ (rustfs.log + startup env per phase) so a failed run
|
||||
# is diagnosable. Short retention: this is churny nightly debug data.
|
||||
path: target/hotpath-abba/
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
||||
- name: Write gate summary
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
status="${{ steps.ab.outputs.status }}"
|
||||
gate_md="${{ steps.ab.outputs.gate_md }}"
|
||||
run_dir="${{ steps.ab.outputs.run_dir }}"
|
||||
{
|
||||
echo "## Hotpath warp A/B — run ${{ github.run_number }}"
|
||||
echo
|
||||
if [[ "$status" == "0" ]]; then
|
||||
echo "Rig/gate exit: \`0\` (pass or warn)."
|
||||
else
|
||||
echo "Rig/gate exit: \`${status:-unknown}\` — **FAILED**."
|
||||
fi
|
||||
echo
|
||||
if [[ -n "$gate_md" && -f "$gate_md" ]]; then
|
||||
cat "$gate_md"
|
||||
else
|
||||
echo "No \`gate.md\` produced — the rig failed **before** the gate"
|
||||
echo "(most likely server startup / health). Failing phase(s) below;"
|
||||
echo "full logs in the \`hotpath-warp-ab-${{ github.run_number }}\` artifact."
|
||||
if [[ -n "$run_dir" && -d "$run_dir/server-logs" ]]; then
|
||||
for f in "$run_dir"/server-logs/*.log; do
|
||||
[[ -f "$f" ]] || continue
|
||||
echo
|
||||
echo "<details><summary>$(basename "$f")</summary>"
|
||||
echo
|
||||
echo '```'
|
||||
tail -n 30 "$f"
|
||||
echo '```'
|
||||
echo "</details>"
|
||||
done
|
||||
fi
|
||||
fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stage successful candidate baseline
|
||||
if: >-
|
||||
steps.ab.outputs.status == '0' &&
|
||||
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cp candidate-bin/rustfs baseline-bin/rustfs
|
||||
|
||||
- name: Cache successful candidate baseline
|
||||
if: >-
|
||||
steps.ab.outputs.status == '0' &&
|
||||
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: baseline-bin/rustfs
|
||||
key: rustfs-baseline-${{ steps.commits.outputs.candidate_sha }}
|
||||
|
||||
# Scheduled failure alerting is handled by the alert-on-failure job below
|
||||
# (perf-2 consuming ci-8's schedule-failure-issue composite action).
|
||||
|
||||
- name: Enforce gate
|
||||
if: always()
|
||||
run: |
|
||||
status="${{ steps.ab.outputs.status }}"
|
||||
if [[ -z "$status" ]]; then
|
||||
echo "::error::warp A/B setup failed before the rig ran. Check the first failed workflow step." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$status" != "0" ]]; then
|
||||
echo "::error::warp A/B budget gate failed (exit $status). See the step summary / gate.md artifact." >&2
|
||||
exit "$status"
|
||||
fi
|
||||
echo "warp A/B budget gate passed."
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [warp-ab]
|
||||
# `always()` is required: without it this job is skipped when a needed
|
||||
# job fails. Alerts only for scheduled (nightly) runs (backlog#1149
|
||||
# ci-8); manual dispatch failures are already watched by a human.
|
||||
# `cancelled` is included alongside `failure` on purpose: a job that hits
|
||||
# timeout-minutes ends as `cancelled`, and the 2026-07-11..07-14 nightly
|
||||
# timeouts went silent precisely because the guard was failure-only. The
|
||||
# composite action already reports cancelled/timed-out jobs in the issue
|
||||
# body.
|
||||
if: >-
|
||||
github.repository == 'rustfs/rustfs' && (
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
@@ -20,13 +20,91 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
build_run_id:
|
||||
description: Successful nightly build run on main
|
||||
description: Successful nightly or validation package run; empty for package-only build
|
||||
type: string
|
||||
required: true
|
||||
required: false
|
||||
default: ''
|
||||
build_run_attempt:
|
||||
description: Exact successful build attempt
|
||||
type: string
|
||||
required: true
|
||||
required: false
|
||||
default: ''
|
||||
validation_package_sha:
|
||||
description: Exact source SHA for an isolated package-only build
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
validation_case:
|
||||
description: Canonical case ID, environment.inspect for read-only resources, or empty for functional chain
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
i05_diagnostic_branch:
|
||||
description: I05 diagnostic-only branch; queued-successor requires the exact native evidence bundle
|
||||
type: choice
|
||||
options: [timeout, queued-successor]
|
||||
required: false
|
||||
default: timeout
|
||||
auto_testing_sha:
|
||||
description: Exact auto-testing commit for a validation case
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
validation_retain_evidence_run_id:
|
||||
description: Upload existing persistent evidence from one interrupted validation run without fleet access
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
validation_retain_evidence_run_attempt:
|
||||
description: Exact interrupted run attempt to retain
|
||||
type: string
|
||||
required: false
|
||||
default: '1'
|
||||
validation_runner_inspect_only:
|
||||
description: Inspect GitHub runner registration and queued jobs on a hosted runner without fleet access
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
validation_tools_only:
|
||||
description: Build candidate-bound validation tools without running a fleet case
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
validation_native_tools:
|
||||
description: Include exact-source native regressions in a tools-only build
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
validation_shared_tools:
|
||||
description: Include the controlled C10 shared-reference encoder in a tools-only build
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
validation_manager_tools:
|
||||
description: Include exact checkpoint/MRF/incarnation regressions in a tools-only build
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
validation_cleanup_run_id:
|
||||
description: Failed validation run whose supported retained environment needs evidence-based finalization
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
validation_cleanup_retained_evidence:
|
||||
description: Recover from verified persistent runner-local evidence when the failed run uploaded no artifact
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
validation_tools_run_id:
|
||||
description: Successful tool build run; required by Heal physical metadata checks
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
validation_tools_run_attempt:
|
||||
description: Exact tool build attempt
|
||||
type: string
|
||||
required: false
|
||||
default: '1'
|
||||
workflow_run:
|
||||
workflows: ["Nightly GNU Build"]
|
||||
types: [completed]
|
||||
@@ -36,12 +114,103 @@ permissions:
|
||||
actions: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-functional-chain-runs
|
||||
group: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_runner_inspect_only && 'rustfs-functional-validation-runner-inspection' || github.event_name == 'workflow_dispatch' && inputs.validation_package_sha != '' && format('rustfs-functional-validation-package-{0}', inputs.validation_package_sha) || github.event_name == 'workflow_dispatch' && inputs.validation_tools_only && 'rustfs-functional-validation-tools' || 'rustfs-functional-chain-runs' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
validation-runner-inspection:
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_runner_inspect_only }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Inspect shared runner registration and queue without fleet access
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
RUNNER_INSPECTION_INPUTS: ${{ toJSON(inputs) }}
|
||||
run: >-
|
||||
python3 -B scripts/inspect_validation_runner.py
|
||||
--output "${RUNNER_TEMP}/validation-runner-inspection/report.json"
|
||||
- name: Upload read-only runner inspection including incomplete reports
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: validation-runner-inspection-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/validation-runner-inspection/report.json
|
||||
if-no-files-found: error
|
||||
validation-evidence-retention:
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id != '' }}
|
||||
# The original interrupted evidence exists only on this long-lived runner.
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 15
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests-v2
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Verify existing interrupted evidence without changing it
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
EVIDENCE_RETAIN_INPUTS: ${{ toJSON(inputs) }}
|
||||
run: >-
|
||||
python3 -B scripts/retain_validation_evidence.py
|
||||
--report "${RUNNER_TEMP}/validation-evidence-retention/report.json"
|
||||
- name: Upload admitted original evidence under collection provenance
|
||||
if: ${{ env.VALIDATION_RETAIN_ROOT != '' }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-validation-retained-${{ env.VALIDATION_RETAIN_SOURCE_RUN }}-${{ env.VALIDATION_RETAIN_SOURCE_ATTEMPT }}-by-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ env.VALIDATION_RETAIN_ROOT }}
|
||||
if-no-files-found: error
|
||||
include-hidden-files: true
|
||||
- name: Upload retention receipt including incomplete reports
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: validation-evidence-retention-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/validation-evidence-retention/report.json
|
||||
if-no-files-found: error
|
||||
validation-package-request:
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha != '' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Validate exclusive package-only request
|
||||
env:
|
||||
PACKAGE_INPUTS: ${{ toJSON(inputs) }}
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
value = json.loads(os.environ["PACKAGE_INPUTS"])
|
||||
if re.fullmatch(r"[0-9a-f]{40}", value["validation_package_sha"]) is None:
|
||||
raise ValueError("package build requires an exact source SHA")
|
||||
exclusive = ("validation_retain_evidence_run_id", "validation_runner_inspect_only", "build_run_id", "build_run_attempt", "validation_case", "auto_testing_sha",
|
||||
"validation_tools_only", "validation_native_tools", "validation_shared_tools",
|
||||
"validation_manager_tools", "validation_cleanup_run_id", "validation_cleanup_retained_evidence", "validation_tools_run_id")
|
||||
if any(value.get(name) for name in exclusive):
|
||||
raise ValueError("package-only build cannot select existing candidate, tools, fleet or recovery execution")
|
||||
PY
|
||||
|
||||
validation-package:
|
||||
needs: validation-package-request
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha != '' }}
|
||||
uses: ./.github/workflows/rustfs-validation-package.yml
|
||||
with:
|
||||
source_sha: ${{ inputs.validation_package_sha }}
|
||||
secrets:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
|
||||
prepare:
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule' }}
|
||||
if: ${{ (github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && inputs.validation_case == '' && !inputs.validation_tools_only && !inputs.validation_native_tools && !inputs.validation_shared_tools && !inputs.validation_manager_tools && !inputs.validation_cleanup_retained_evidence && inputs.validation_cleanup_run_id == '') || github.event.workflow_run.event == 'schedule' }}
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
@@ -89,6 +258,395 @@ jobs:
|
||||
echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
validation:
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '' || inputs.validation_cleanup_retained_evidence) }}
|
||||
runs-on: ${{ inputs.validation_tools_only && 'ubuntu-latest' || 'smoke-testing' }}
|
||||
timeout-minutes: 90
|
||||
outputs:
|
||||
source_sha: ${{ steps.candidate.outputs.source_sha }}
|
||||
concurrency:
|
||||
group: ${{ inputs.validation_tools_only && 'rustfs-validation-tools-resolver' || 'rustfs-shared-functional-tests-v2' }}
|
||||
cancel-in-progress: false
|
||||
env:
|
||||
RUSTFS_ACCESS_KEY: ${{ !inputs.validation_tools_only && secrets.RUSTFS_ACCESS_KEY || '' }}
|
||||
RUSTFS_SECRET_KEY: ${{ !inputs.validation_tools_only && secrets.RUSTFS_SECRET_KEY || '' }}
|
||||
RUSTFS_API_ENDPOINT: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT) || '' }}
|
||||
RUSTFS_NODES: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_NODES || vars.RUSTFS_NODES) || '' }}
|
||||
RUSTFS_SSH_USER: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER) || '' }}
|
||||
steps:
|
||||
- name: Checkout candidate resolver
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Initialize validation evidence
|
||||
env:
|
||||
VALIDATION_TOOLS_ONLY: ${{ inputs.validation_tools_only }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
python3 -B scripts/initialize_validation_evidence.py
|
||||
|
||||
- name: Resolve immutable candidate
|
||||
id: candidate
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
BUILD_RUN_ID: ${{ inputs.build_run_id }}
|
||||
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
|
||||
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
|
||||
CLEANUP_RUN_ID: ${{ inputs.validation_cleanup_run_id }}
|
||||
CLEANUP_RETAINED_EVIDENCE: ${{ inputs.validation_cleanup_retained_evidence }}
|
||||
SELECTED_CASE: ${{ inputs.validation_case }}
|
||||
TOOLS_ONLY: ${{ inputs.validation_tools_only }}
|
||||
NATIVE_TOOLS: ${{ inputs.validation_native_tools }}
|
||||
SHARED_TOOLS: ${{ inputs.validation_shared_tools }}
|
||||
MANAGER_TOOLS: ${{ inputs.validation_manager_tools }}
|
||||
I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
from scripts.resolve_functional_candidate import resolve
|
||||
|
||||
revision = os.environ["AUTO_TESTING_SHA"]
|
||||
cleanup = os.environ["CLEANUP_RUN_ID"]
|
||||
if os.environ["CLEANUP_RETAINED_EVIDENCE"] == "true" and not cleanup:
|
||||
raise ValueError("persistent recovery input requires one exact cleanup run")
|
||||
if os.environ["TOOLS_ONLY"] == "true" and (cleanup or os.environ["SELECTED_CASE"]):
|
||||
raise ValueError("tools-only execution cannot select a fleet case or cleanup")
|
||||
if os.environ["NATIVE_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
|
||||
raise ValueError("native regressions require tools-only execution, without a fleet case or cleanup")
|
||||
if os.environ["SHARED_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
|
||||
raise ValueError("shared fixture builds require tools-only execution, without a fleet case or cleanup")
|
||||
if os.environ["MANAGER_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
|
||||
raise ValueError("manager regressions require tools-only execution, without a fleet case or cleanup")
|
||||
if cleanup and (not cleanup.isdigit() or int(cleanup) <= 0 or os.environ["SELECTED_CASE"] or os.environ["TOOLS_ONLY"] == "true"):
|
||||
raise ValueError("cleanup requires one exact failed run and is exclusive with case/tool execution")
|
||||
diagnostic_branch = os.environ["I05_DIAGNOSTIC_BRANCH"] or "timeout"
|
||||
if diagnostic_branch not in {"timeout", "queued-successor"}:
|
||||
raise ValueError("I05 diagnostic branch is not registered")
|
||||
if diagnostic_branch == "queued-successor" and os.environ["SELECTED_CASE"] != "heal.lock-timeout-and-caller-abort":
|
||||
raise ValueError("queued-successor is available only for the I05 diagnostic case")
|
||||
if re.fullmatch(r"[0-9a-f]{40}", revision) is None:
|
||||
raise ValueError("validation requires an exact auto-testing commit SHA")
|
||||
candidate = resolve(int(os.environ["BUILD_RUN_ID"]), int(os.environ["BUILD_RUN_ATTEMPT"]), allow_validation=True)
|
||||
manifest = candidate["manifest"]
|
||||
root = Path(os.environ["VALIDATION_ROOT"])
|
||||
(root / "candidate-source.json").write_text(json.dumps(candidate, sort_keys=True) + "\n")
|
||||
(root / "candidate.json").write_text(json.dumps({
|
||||
"package_url": manifest["package_url"],
|
||||
"package_sha256": manifest["package_sha256"],
|
||||
"source_sha": manifest["source_sha"],
|
||||
}, sort_keys=True) + "\n")
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write("source_sha=" + manifest["source_sha"] + "\n")
|
||||
PY
|
||||
|
||||
- name: Checkout exact auto-testing revision
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ inputs.auto_testing_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Prepare validation Python runtime
|
||||
if: ${{ !inputs.validation_tools_only && (startsWith(inputs.validation_case, 'ilm.') || startsWith(inputs.validation_case, 'pool.') || startsWith(inputs.validation_case, 'bucket-replication.') || startsWith(inputs.validation_case, 'site-replication.') || startsWith(inputs.validation_case, 'heal.')) }}
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install validation Python client
|
||||
if: ${{ !inputs.validation_tools_only && (startsWith(inputs.validation_case, 'ilm.') || startsWith(inputs.validation_case, 'pool.') || startsWith(inputs.validation_case, 'bucket-replication.') || startsWith(inputs.validation_case, 'site-replication.') || startsWith(inputs.validation_case, 'heal.')) }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
test ! -e "${validation_python}"
|
||||
python3 -m venv "${validation_python}"
|
||||
"${validation_python}/bin/python" -m pip install \
|
||||
-r auto-testing/scripts/validation_ilm/requirements.txt
|
||||
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Validate Heal tool build identity
|
||||
if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TOOL_RUN_ID: ${{ inputs.validation_tools_run_id }}
|
||||
TOOL_RUN_ATTEMPT: ${{ inputs.validation_tools_run_attempt }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "${TOOL_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
|
||||
[[ "${TOOL_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
|
||||
gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${TOOL_RUN_ID}/attempts/${TOOL_RUN_ATTEMPT}" \
|
||||
> "${VALIDATION_ROOT}/tool-run.json"
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
value = json.loads((Path(os.environ["VALIDATION_ROOT"]) / "tool-run.json").read_text())
|
||||
if (value.get("conclusion") != "success" or value.get("status") != "completed"
|
||||
or value.get("run_attempt") != int(os.environ["TOOL_RUN_ATTEMPT"])
|
||||
or value.get("path") not in {".github/workflows/rustfs-functional-chain.yml", ".github/workflows/rustfs-validation-tools.yml"}):
|
||||
raise ValueError("Heal tool run is not an exact successful validation build attempt")
|
||||
PY
|
||||
|
||||
- name: Download pinned Heal metadata inspector
|
||||
if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.validation_tools_run_id }}
|
||||
name: rustfs-validation-tools-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
|
||||
path: ${{ runner.temp }}/rustfs-validation-tools-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
|
||||
- name: Verify Heal metadata inspector
|
||||
if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TOOL_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-tools-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
python3 -B auto-testing/scripts/validation_heal/inspector.py \
|
||||
--directory "${TOOL_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
|
||||
> "${VALIDATION_ROOT}/tool-manifest.json"
|
||||
printf 'VALIDATION_HEAL_TOOL_DIRECTORY=%s\n' "${TOOL_DIRECTORY}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Select exact-source race native evidence
|
||||
id: race_native
|
||||
if: ${{ inputs.validation_case == 'heal.race-unversioned-overwrite' || inputs.validation_case == 'heal.race-versioned-put' || inputs.validation_case == 'heal.race-version-delete' || inputs.validation_case == 'heal.race-markers-and-batch-delete' || inputs.validation_case == 'heal.race-multipart-complete-abort' || inputs.validation_case == 'heal.admin-nolock-safety' || inputs.validation_case == 'heal.race-copy-and-metadata-update' }}
|
||||
env:
|
||||
CANDIDATE_SHA: ${{ steps.candidate.outputs.source_sha }}
|
||||
VALIDATION_CASE_ID: ${{ inputs.validation_case }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
required=false
|
||||
if [ -f auto-testing/scripts/validation_heal/race_native.py ]; then
|
||||
status=0
|
||||
python3 -B auto-testing/scripts/validation_heal/race_native.py supports \
|
||||
--commit "${CANDIDATE_SHA}" --case-id "${VALIDATION_CASE_ID}" || status=$?
|
||||
if [ "${status}" -eq 0 ]; then
|
||||
required=true
|
||||
elif [ "${status}" -ne 2 ]; then
|
||||
exit "${status}"
|
||||
fi
|
||||
fi
|
||||
reviewed=false
|
||||
case "${CANDIDATE_SHA}:${VALIDATION_CASE_ID}" in
|
||||
6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca:heal.race-unversioned-overwrite | \
|
||||
bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-unversioned-overwrite | \
|
||||
bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-versioned-put | \
|
||||
bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-version-delete | \
|
||||
8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-unversioned-overwrite | \
|
||||
8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-versioned-put | \
|
||||
8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-version-delete | \
|
||||
8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-markers-and-batch-delete | \
|
||||
757b751482bb970bc321e92059690a7698554358:heal.admin-nolock-safety | \
|
||||
757b751482bb970bc321e92059690a7698554358:heal.race-multipart-complete-abort | \
|
||||
757b751482bb970bc321e92059690a7698554358:heal.race-copy-and-metadata-update) reviewed=true ;;
|
||||
esac
|
||||
if [ "${reviewed}" = true ] && [ "${required}" != true ]; then
|
||||
printf 'Race candidate requires its reviewed native collector before fleet execution\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
printf 'required=%s\n' "${required}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Download pinned native storage evidence
|
||||
if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || steps.race_native.outputs.required == 'true' || (inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch == 'queued-successor') }}
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.validation_tools_run_id }}
|
||||
name: rustfs-validation-native-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
|
||||
path: ${{ runner.temp }}/rustfs-validation-native-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
|
||||
- name: Verify C07 native source and exact test evidence
|
||||
if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
|
||||
python3 -B auto-testing/scripts/validation_heal/authority.py \
|
||||
--directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
|
||||
> "${VALIDATION_ROOT}/native-authority-manifest.json"
|
||||
printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Verify C11 physical storage and exact final-rename barrier
|
||||
if: ${{ inputs.validation_case == 'heal.bucket-incarnation-recreation' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
|
||||
python3 -B auto-testing/scripts/validation_heal/incarnation_native.py \
|
||||
--directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
|
||||
--testing-commit "${{ inputs.auto_testing_sha }}" \
|
||||
> "${VALIDATION_ROOT}/native-incarnation-manifest.json"
|
||||
printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Verify exact-source race native and runtime evidence
|
||||
if: ${{ steps.race_native.outputs.required == 'true' }}
|
||||
env:
|
||||
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
|
||||
VALIDATION_CASE_ID: ${{ inputs.validation_case }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
|
||||
python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \
|
||||
--directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
|
||||
--testing-commit "${AUTO_TESTING_SHA}" --case-id "${VALIDATION_CASE_ID}" \
|
||||
> "${VALIDATION_ROOT}/native-race-manifest.json"
|
||||
runtime_driver=race.py
|
||||
if [ "${VALIDATION_CASE_ID}" = 'heal.admin-nolock-safety' ]; then
|
||||
runtime_driver=nolock.py
|
||||
fi
|
||||
VALIDATION_HEAL_NATIVE_DIRECTORY="${NATIVE_DIRECTORY}" \
|
||||
python3 -B "auto-testing/scripts/validation_heal/${runtime_driver}" preflight \
|
||||
--candidate "${VALIDATION_ROOT}/candidate.json" --testing-commit "${AUTO_TESTING_SHA}" \
|
||||
--case-id "${VALIDATION_CASE_ID}" \
|
||||
> "${VALIDATION_ROOT}/native-race-runtime-preflight.json"
|
||||
printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Verify I05 queued-successor native prerequisite
|
||||
if: ${{ inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch == 'queued-successor' }}
|
||||
env:
|
||||
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
|
||||
python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \
|
||||
--directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
|
||||
--testing-commit "${AUTO_TESTING_SHA}" --case-id heal.lock-timeout-and-caller-abort \
|
||||
> "${VALIDATION_ROOT}/native-i05-prerequisite.json"
|
||||
printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Download pinned manager persistence and incarnation evidence
|
||||
if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || inputs.validation_case == 'heal.outcome-counters-and-error-truth' }}
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.validation_tools_run_id }}
|
||||
name: rustfs-validation-manager-native-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
|
||||
path: ${{ runner.temp }}/rustfs-validation-manager-native-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
|
||||
- name: Verify exact manager native prerequisites before fleet mutation
|
||||
if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || inputs.validation_case == 'heal.outcome-counters-and-error-truth' }}
|
||||
env:
|
||||
VALIDATION_CASE: ${{ inputs.validation_case }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
MANAGER_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-manager-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
cp "${VALIDATION_ROOT}/tool-run.json" "${MANAGER_DIRECTORY}/tool-run.json"
|
||||
python3 -B auto-testing/scripts/validation_heal/manager_fixture.py \
|
||||
--directory "${MANAGER_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
|
||||
--case-id "${VALIDATION_CASE}" > "${VALIDATION_ROOT}/manager-prerequisite-manifest.json"
|
||||
printf 'VALIDATION_HEAL_MANAGER_DIRECTORY=%s\n' "${MANAGER_DIRECTORY}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Download pinned C10 shared-reference encoder
|
||||
if: ${{ inputs.validation_case == 'heal.dangling-and-orphan-safety' }}
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
run-id: ${{ inputs.validation_tools_run_id }}
|
||||
name: rustfs-validation-shared-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
|
||||
path: ${{ runner.temp }}/rustfs-validation-shared-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
|
||||
- name: Verify C10 exact-source codec and original tests
|
||||
if: ${{ inputs.validation_case == 'heal.dangling-and-orphan-safety' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
SHARED_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-shared-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
cp "${VALIDATION_ROOT}/tool-run.json" "${SHARED_DIRECTORY}/tool-run.json"
|
||||
python3 -B auto-testing/scripts/validation_heal/shared_fixture.py \
|
||||
--directory "${SHARED_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
|
||||
> "${VALIDATION_ROOT}/shared-fixture-manifest.json"
|
||||
printf 'VALIDATION_HEAL_SHARED_DIRECTORY=%s\n' "${SHARED_DIRECTORY}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Run selected canonical validation case
|
||||
if: ${{ !inputs.validation_tools_only && inputs.validation_cleanup_run_id == '' && inputs.validation_case != 'environment.inspect' }}
|
||||
env:
|
||||
VALIDATION_CASE: ${{ inputs.validation_case }}
|
||||
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
|
||||
I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
|
||||
./auto-testing/scripts/run_validation_case.sh \
|
||||
"${VALIDATION_CASE}" "${VALIDATION_ROOT}/candidate.json" \
|
||||
"${AUTO_TESTING_SHA}" "${VALIDATION_ROOT}/run"
|
||||
|
||||
- name: Inspect current Heal fleet resources without mutation
|
||||
if: ${{ !inputs.validation_tools_only && inputs.validation_case == 'environment.inspect' }}
|
||||
env:
|
||||
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
|
||||
python3 -B auto-testing/scripts/validation_heal/environment.py \
|
||||
--candidate "${VALIDATION_ROOT}/candidate.json" \
|
||||
--testing-commit "${AUTO_TESTING_SHA}" --evidence "${VALIDATION_ROOT}/environment"
|
||||
|
||||
- name: Audit and finalize retained validation environment
|
||||
if: ${{ inputs.validation_cleanup_run_id != '' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
FAILED_RUN_ID: ${{ inputs.validation_cleanup_run_id }}
|
||||
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
|
||||
CLEANUP_RETAINED_EVIDENCE: ${{ inputs.validation_cleanup_retained_evidence }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
|
||||
gh run view "${FAILED_RUN_ID}" --repo "${GITHUB_REPOSITORY}" \
|
||||
--json status,conclusion,headBranch,workflowName > "${VALIDATION_ROOT}/failed-run.json"
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
run = json.loads((Path(os.environ["VALIDATION_ROOT"]) / "failed-run.json").read_text())
|
||||
if (run["status"] != "completed" or run["conclusion"] not in {"failure", "cancelled"}
|
||||
or run["workflowName"] != "RustFS Functional Chain" or run["headBranch"] != os.environ["GITHUB_REF_NAME"]):
|
||||
raise ValueError("cleanup source must be a completed failed or cancelled Functional Chain run")
|
||||
PY
|
||||
if [ "${CLEANUP_RETAINED_EVIDENCE}" = true ]; then
|
||||
python3 -B scripts/prepare_retained_validation_recovery.py \
|
||||
--report "${VALIDATION_ROOT}/retained-recovery-input.json"
|
||||
else
|
||||
gh run download "${FAILED_RUN_ID}" --repo "${GITHUB_REPOSITORY}" \
|
||||
--pattern "rustfs-validation-${FAILED_RUN_ID}-*" --dir "${VALIDATION_ROOT}/failed-artifact"
|
||||
fi
|
||||
python3 -B auto-testing/scripts/validation_heal/recover_environment.py \
|
||||
--failed-run-id "${FAILED_RUN_ID}" --artifact "${VALIDATION_ROOT}/failed-artifact" \
|
||||
--candidate "${VALIDATION_ROOT}/candidate.json" --evidence "${VALIDATION_ROOT}/recovery"
|
||||
|
||||
- name: Show validation report
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
if [ -f "${VALIDATION_ROOT}/run/report.md" ]; then
|
||||
cat "${VALIDATION_ROOT}/run/report.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
fi
|
||||
|
||||
- name: Upload validation evidence
|
||||
if: ${{ always() && env.VALIDATION_ROOT != '' }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-validation-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ env.VALIDATION_ROOT }}
|
||||
if-no-files-found: error
|
||||
|
||||
validation-tools:
|
||||
needs: [validation]
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_package_sha == '' && inputs.validation_tools_only }}
|
||||
uses: ./.github/workflows/rustfs-validation-tools.yml
|
||||
with:
|
||||
candidate_sha: ${{ needs.validation.outputs.source_sha }}
|
||||
auto_testing_sha: ${{ inputs.auto_testing_sha }}
|
||||
build_native_heal: ${{ inputs.validation_native_tools }}
|
||||
build_shared_heal: ${{ inputs.validation_shared_tools }}
|
||||
build_manager_native: ${{ inputs.validation_manager_tools }}
|
||||
secrets: inherit
|
||||
|
||||
upgrade:
|
||||
needs: [prepare]
|
||||
if: ${{ always() && needs.prepare.result == 'success' }}
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
||||
name: RustFS Validation Package
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
source_sha:
|
||||
description: Exact RustFS commit SHA to build
|
||||
required: true
|
||||
type: string
|
||||
secrets:
|
||||
R2_ACCESS_KEY_ID:
|
||||
required: true
|
||||
R2_SECRET_ACCESS_KEY:
|
||||
required: true
|
||||
R2_ENDPOINT:
|
||||
required: true
|
||||
R2_BUCKET:
|
||||
required: true
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source_sha:
|
||||
description: Exact RustFS commit SHA to build
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-validation-package-${{ inputs.source_sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
resolve-source:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Resolve exact validation source
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
source_sha: ${{ steps.source.outputs.sha }}
|
||||
source_ref: ${{ steps.source.outputs.ref }}
|
||||
steps:
|
||||
- name: Validate requested source
|
||||
id: source
|
||||
env:
|
||||
REQUESTED_SHA: ${{ inputs.source_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "${REQUESTED_SHA}" =~ ^[0-9a-f]{40}$ ]]
|
||||
source_directory="$(mktemp -d "${RUNNER_TEMP}/validation-source.XXXXXX")"
|
||||
trap 'rm -rf "${source_directory}"' EXIT
|
||||
git init -q "${source_directory}"
|
||||
git -C "${source_directory}" fetch --no-tags --depth=1 https://github.com/rustfs/rustfs.git "${REQUESTED_SHA}"
|
||||
test "$(git -C "${source_directory}" rev-parse FETCH_HEAD)" = "${REQUESTED_SHA}"
|
||||
echo "sha=${REQUESTED_SHA}" >> "${GITHUB_OUTPUT}"
|
||||
echo "ref=${REQUESTED_SHA}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
build:
|
||||
needs: resolve-source
|
||||
name: Build immutable validation DEB
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 150
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }}
|
||||
steps:
|
||||
- name: Checkout exact source
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.resolve-source.outputs.source_sha }}
|
||||
|
||||
- name: Confirm checked out source
|
||||
env:
|
||||
EXPECTED_SHA: ${{ needs.resolve-source.outputs.source_sha }}
|
||||
run: test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}"
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
cache-shared-key: validation-package-hosted-x86_64-unknown-linux-gnu
|
||||
cache-save-if: "false"
|
||||
install-build-packaging-tools: "false"
|
||||
install-test-tools: "false"
|
||||
|
||||
- name: Build RustFS
|
||||
run: cargo build --release --locked --target x86_64-unknown-linux-gnu -p rustfs --bins
|
||||
|
||||
- name: Build validation DEB
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
command -v fakeroot >/dev/null 2>&1 || sudo apt-get install -y -qq fakeroot
|
||||
package_dir="rustfs-validation-${VALIDATION_SOURCE_SHA}"
|
||||
package_file="${RUNNER_TEMP}/validation-package/rustfs.deb"
|
||||
mkdir -p "${RUNNER_TEMP}/validation-package"
|
||||
binary="target/x86_64-unknown-linux-gnu/release/rustfs"
|
||||
test -x "${binary}"
|
||||
mkdir -p "${package_dir}/DEBIAN" "${package_dir}/usr/bin" "${package_dir}/etc/default" \
|
||||
"${package_dir}/lib/systemd/system" "${package_dir}/usr/share/doc/rustfs"
|
||||
cp "${binary}" "${package_dir}/usr/bin/rustfs"
|
||||
chmod 755 "${package_dir}/usr/bin/rustfs"
|
||||
cp deploy/build/rustfs.service "${package_dir}/lib/systemd/system/"
|
||||
cat > "${package_dir}/etc/default/rustfs" <<'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# RUSTFS_ACCESS_KEY=<changeme>
|
||||
# RUSTFS_SECRET_KEY=<changeme>
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ADDRESS=":9000"
|
||||
ENVEOF
|
||||
cat > "${package_dir}/DEBIAN/control" <<'CONTROLEOF'
|
||||
Package: rustfs
|
||||
Version: 0.0.0~validation
|
||||
Section: utils
|
||||
Priority: optional
|
||||
Architecture: amd64
|
||||
Depends: libc6 (>= 2.31)
|
||||
Maintainer: RustFS Team <support@rustfs.com>
|
||||
Description: RustFS immutable validation package
|
||||
Candidate package for RustFS validation only.
|
||||
CONTROLEOF
|
||||
printf '%s\n' /etc/default/rustfs > "${package_dir}/DEBIAN/conffiles"
|
||||
./scripts/release/package_service_scripts.sh after-install > "${package_dir}/DEBIAN/postinst"
|
||||
./scripts/release/package_service_scripts.sh before-remove > "${package_dir}/DEBIAN/prerm"
|
||||
./scripts/release/package_service_scripts.sh after-remove > "${package_dir}/DEBIAN/postrm"
|
||||
chmod 755 "${package_dir}/DEBIAN/postinst" "${package_dir}/DEBIAN/prerm" "${package_dir}/DEBIAN/postrm"
|
||||
cp LICENSE README.md "${package_dir}/usr/share/doc/rustfs/"
|
||||
fakeroot dpkg-deb --build "${package_dir}" "${package_file}"
|
||||
test -s "${package_file}"
|
||||
|
||||
- name: Upload opaque validation package
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: validation-package-bytes-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/validation-package/rustfs.deb
|
||||
if-no-files-found: error
|
||||
|
||||
publish:
|
||||
needs: [resolve-source, build]
|
||||
name: Publish immutable validation DEB
|
||||
# Candidate-controlled build scripts never run on this fresh hosted runner.
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }}
|
||||
VALIDATION_SOURCE_REF: ${{ needs.resolve-source.outputs.source_ref }}
|
||||
steps:
|
||||
- name: Download opaque validation package
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: validation-package-bytes-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/validation-package-input
|
||||
|
||||
- name: Validate and stage opaque package bytes
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 -I - <<'PYTHON'
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import stat
|
||||
|
||||
root = Path(os.environ["RUNNER_TEMP"])
|
||||
incoming = root / "validation-package-input"
|
||||
if incoming.is_symlink() or not incoming.is_dir() or sorted(p.name for p in incoming.iterdir()) != ["rustfs.deb"]:
|
||||
raise SystemExit("Validation artifact must contain exactly one rustfs.deb file")
|
||||
fd = os.open(incoming / "rustfs.deb", os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
||||
with os.fdopen(fd, "rb") as source:
|
||||
info = os.fstat(source.fileno())
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_size == 0:
|
||||
raise SystemExit("Validation package must be a nonempty regular file")
|
||||
staged = root / "validation-publication"
|
||||
staged.mkdir(mode=0o700)
|
||||
with (staged / "rustfs.deb").open("xb") as target:
|
||||
shutil.copyfileobj(source, target)
|
||||
PYTHON
|
||||
|
||||
- name: Prepare trusted publication tools
|
||||
id: tools
|
||||
env:
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
AWS_CLI=aws
|
||||
if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y -qq python3-venv
|
||||
AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/validation-awscli.XXXXXX")"
|
||||
python3 -m venv "${AWS_CLI_DIR}"
|
||||
"${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
|
||||
AWS_CLI="${AWS_CLI_DIR}/bin/aws"
|
||||
fi
|
||||
"${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
|
||||
echo "aws_cli=${AWS_CLI}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Upload validation package to immutable R2 path
|
||||
id: publish
|
||||
env:
|
||||
PACKAGE_FILE: ${{ runner.temp }}/validation-publication/rustfs.deb
|
||||
AWS_CLI: ${{ steps.tools.outputs.aws_cli }}
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ -z "${R2_ACCESS_KEY_ID}" || -z "${R2_SECRET_ACCESS_KEY}" || -z "${R2_ENDPOINT}" || -z "${R2_BUCKET}" ]]; then
|
||||
echo "Validation package publication requires complete R2 configuration" >&2
|
||||
exit 1
|
||||
fi
|
||||
[[ "${VALIDATION_SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]]
|
||||
test "${VALIDATION_SOURCE_REF}" = "${VALIDATION_SOURCE_SHA}"
|
||||
test -s "${PACKAGE_FILE}" && test ! -L "${PACKAGE_FILE}"
|
||||
export AWS_ACCESS_KEY_ID="${R2_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY}" AWS_DEFAULT_REGION=auto
|
||||
package_sha256="$(sha256sum "${PACKAGE_FILE}" | cut -d ' ' -f 1)"
|
||||
key="artifacts/rustfs/packages/validation/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${package_sha256}/rustfs.deb"
|
||||
url="https://dl.rustfs.com/${key}"
|
||||
"${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${key}" --body "${PACKAGE_FILE}" \
|
||||
--if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
|
||||
downloaded_sha256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${url}" | sha256sum | cut -d ' ' -f 1)"
|
||||
test "${downloaded_sha256}" = "${package_sha256}"
|
||||
manifest="${RUNNER_TEMP}/validation-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --arg workflow_sha "${GITHUB_SHA}" --arg source_ref "${VALIDATION_SOURCE_REF}" \
|
||||
--arg source_sha "${VALIDATION_SOURCE_SHA}" --arg channel validation \
|
||||
--argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--arg package_url "${url}" --arg package_sha256 "${package_sha256}" \
|
||||
'{schema: 3, workflow_sha: $workflow_sha, source_ref: $source_ref, source_sha: $source_sha, channel: $channel, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' > "${manifest}"
|
||||
echo "candidate_file=${manifest}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Upload validation candidate manifest
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: validation-candidate-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ steps.publish.outputs.candidate_file }}
|
||||
if-no-files-found: error
|
||||
@@ -0,0 +1,343 @@
|
||||
# Build source-bound helpers separately so live validation needs only artifacts.
|
||||
name: RustFS Validation Tools
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
candidate_sha:
|
||||
type: string
|
||||
required: true
|
||||
auto_testing_sha:
|
||||
type: string
|
||||
required: true
|
||||
build_native_heal:
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
build_shared_heal:
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
build_manager_native:
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
secrets:
|
||||
PF_TESTING_GH_TOKEN:
|
||||
required: true
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
candidate_sha:
|
||||
description: Exact RustFS source SHA from the validation candidate manifest
|
||||
type: string
|
||||
required: true
|
||||
auto_testing_sha:
|
||||
description: Exact auto-testing SHA containing the Heal metadata inspector
|
||||
type: string
|
||||
required: true
|
||||
build_native_heal:
|
||||
description: Also compile and run separate exact-source native Heal regressions
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
build_shared_heal:
|
||||
description: Build the controlled C10 shared-reference fixture encoder
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
build_manager_native:
|
||||
description: Build and run exact candidate checkpoint/MRF/incarnation regressions
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
manager-native-regressions:
|
||||
if: ${{ inputs.build_manager_native }}
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 180
|
||||
env:
|
||||
CANDIDATE_SHA: ${{ inputs.candidate_sha }}
|
||||
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
|
||||
steps:
|
||||
- name: Validate immutable manager native inputs
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
|
||||
|
||||
- name: Checkout exact manager candidate source
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: ${{ inputs.candidate_sha }}
|
||||
path: candidate
|
||||
persist-credentials: false
|
||||
|
||||
- name: Checkout exact manager collector source
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ inputs.auto_testing_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install manager native build dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y clang lld cmake pkg-config libssl-dev protobuf-compiler
|
||||
df -h .
|
||||
|
||||
- name: Build exact unmodified candidate manager test executable
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
|
||||
python3 -B auto-testing/scripts/validation_heal/manager_native.py build \
|
||||
--source candidate --commit "${CANDIDATE_SHA}" --out manager-tools
|
||||
|
||||
- name: Run exact manager regressions independently
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 -B auto-testing/scripts/validation_heal/manager_native.py run \
|
||||
--directory manager-tools --out manager-evidence
|
||||
|
||||
- name: Upload manager executable identities and original diagnostics
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-validation-manager-native-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
manager-tools/heal-manager-native-tests
|
||||
manager-tools/*.json
|
||||
manager-tools/*.jsonl
|
||||
manager-tools/*.stderr
|
||||
manager-tools/*.log
|
||||
manager-evidence/
|
||||
if-no-files-found: warn
|
||||
|
||||
native-heal-regressions:
|
||||
if: ${{ inputs.build_native_heal }}
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 180
|
||||
env:
|
||||
CANDIDATE_SHA: ${{ inputs.candidate_sha }}
|
||||
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
|
||||
steps:
|
||||
- name: Validate immutable native source inputs
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
|
||||
|
||||
- name: Checkout exact native candidate source
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: ${{ inputs.candidate_sha }}
|
||||
path: candidate
|
||||
persist-credentials: false
|
||||
|
||||
- name: Checkout exact native collector source
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ inputs.auto_testing_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install native build dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y clang lld cmake pkg-config libssl-dev protobuf-compiler
|
||||
df -h .
|
||||
|
||||
- name: Build and enumerate the unmodified candidate test executable
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
|
||||
use_race_native=false
|
||||
if [[ -f auto-testing/scripts/validation_heal/race_native.py ]]; then
|
||||
support_rc=0
|
||||
python3 -B auto-testing/scripts/validation_heal/race_native.py supports \
|
||||
--commit "${CANDIDATE_SHA}" || support_rc=$?
|
||||
case "${support_rc}" in
|
||||
0) use_race_native=true ;;
|
||||
2) ;; # The original exact-source collector handles legacy candidates.
|
||||
*) exit "${support_rc}" ;;
|
||||
esac
|
||||
fi
|
||||
if [[ "${use_race_native}" == true ]]; then
|
||||
python3 -B auto-testing/scripts/validation_heal/race_native.py build \
|
||||
--source candidate --commit "${CANDIDATE_SHA}" --testing-commit "${AUTO_TESTING_SHA}" --out native-tools
|
||||
else
|
||||
python3 -B auto-testing/scripts/validation_heal/native_regressions.py build \
|
||||
--source candidate --commit "${CANDIDATE_SHA}" --out native-tools
|
||||
fi
|
||||
|
||||
- name: Run exact native regressions separately from fleet validation
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ -f native-tools/race-native.json ]]; then
|
||||
test ! -e native-tools/native-regressions.json
|
||||
python3 -B auto-testing/scripts/validation_heal/race_native.py run \
|
||||
--directory native-tools --out native-evidence
|
||||
else
|
||||
python3 -B auto-testing/scripts/validation_heal/native_regressions.py run \
|
||||
--directory native-tools --out native-evidence
|
||||
fi
|
||||
|
||||
- name: Run existing F07 directory fsync failure regressions
|
||||
if: ${{ inputs.candidate_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Older immutable harness pins retain their original native suite;
|
||||
# absence of this additive collector must never fabricate its proof.
|
||||
if [[ ! -f auto-testing/scripts/validation_heal/persistence_native.py ]]; then
|
||||
mkdir native-persistence-evidence
|
||||
echo '{"complete":false,"reason":"pinned harness predates F07 directory-fsync collector","requiredNativeCoverageComplete":false,"canonicalCasePassClaimed":false}' \
|
||||
> native-persistence-evidence/not-available.json
|
||||
exit 0
|
||||
fi
|
||||
python3 -B auto-testing/scripts/validation_heal/persistence_native.py run \
|
||||
--directory native-tools --native-evidence native-evidence --out native-persistence-evidence \
|
||||
--testing-commit "${AUTO_TESTING_SHA}"
|
||||
python3 -B auto-testing/scripts/validation_heal/persistence_native.py verify \
|
||||
--directory native-tools --native-evidence native-evidence --out native-persistence-evidence \
|
||||
--testing-commit "${AUTO_TESTING_SHA}"
|
||||
|
||||
- name: Upload native executable identities and original diagnostics
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-validation-native-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
native-tools/ecstore-native-tests
|
||||
native-tools/ecstore-race-native-tests
|
||||
native-tools/*.json
|
||||
native-tools/*.jsonl
|
||||
native-tools/*.stderr
|
||||
native-tools/*.log
|
||||
native-evidence/
|
||||
native-persistence-evidence/
|
||||
if-no-files-found: warn
|
||||
|
||||
heal-metadata-inspector:
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CANDIDATE_SHA: ${{ inputs.candidate_sha }}
|
||||
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
|
||||
steps:
|
||||
- name: Validate immutable source inputs
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
|
||||
|
||||
- name: Checkout exact candidate source
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: ${{ inputs.candidate_sha }}
|
||||
path: candidate
|
||||
persist-credentials: false
|
||||
|
||||
- name: Checkout exact inspector source
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ inputs.auto_testing_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Build the read-only candidate metadata decoder
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "$(git -C candidate rev-parse HEAD)" = "${CANDIDATE_SHA}"
|
||||
test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
|
||||
df -h .
|
||||
python3 auto-testing/scripts/validation_heal/prepare_heal_meta_inspector.py \
|
||||
--source candidate --commit "${CANDIDATE_SHA}" --out inspector-build
|
||||
# Seed the helper's separate graph from the candidate's frozen graph.
|
||||
# Cargo adds the helper package; both lock identities are retained.
|
||||
cp candidate/Cargo.lock inspector-build/Cargo.lock
|
||||
cargo build --release --manifest-path inspector-build/Cargo.toml
|
||||
python3 auto-testing/scripts/validation_heal/check_bitrot_tool.py \
|
||||
--binary inspector-build/target/release/heal-meta-inspect --source candidate
|
||||
mkdir validation-tools
|
||||
cp inspector-build/target/release/heal-meta-inspect validation-tools/
|
||||
cp inspector-build/Cargo.lock validation-tools/inspector-Cargo.lock
|
||||
python3 - <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
source = json.loads(Path("inspector-build/source.json").read_text())
|
||||
binary = Path("validation-tools/heal-meta-inspect")
|
||||
source.update(built=True, testingCommit=os.environ["AUTO_TESTING_SHA"],
|
||||
binarySha256=hashlib.sha256(binary.read_bytes()).hexdigest(),
|
||||
inspectorLockSha256=hashlib.sha256(Path("validation-tools/inspector-Cargo.lock").read_bytes()).hexdigest(),
|
||||
toolchain=subprocess.check_output(["rustc", "-vV"], text=True),
|
||||
target="x86_64-unknown-linux-gnu", runId=os.environ["GITHUB_RUN_ID"],
|
||||
runAttempt=os.environ["GITHUB_RUN_ATTEMPT"])
|
||||
Path("validation-tools/heal-meta-inspect.json").write_text(json.dumps(source, indent=2) + "\n")
|
||||
PY
|
||||
git -C candidate diff --exit-code
|
||||
|
||||
- name: Build and test the controlled C10 metadata encoder
|
||||
if: ${{ inputs.build_shared_heal }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 -B auto-testing/scripts/validation_heal/prepare_heal_shared_fixture.py \
|
||||
--source candidate --commit "${CANDIDATE_SHA}" --out shared-build
|
||||
cp candidate/Cargo.lock shared-build/Cargo.lock
|
||||
export CARGO_TARGET_DIR="${GITHUB_WORKSPACE}/inspector-build/target"
|
||||
cargo build --release --manifest-path shared-build/Cargo.toml
|
||||
mkdir shared-tools
|
||||
if ! cargo test --release --manifest-path shared-build/Cargo.toml -- --test-threads=1 \
|
||||
> shared-tools/shared-fixture-tests.log 2>&1; then
|
||||
cat shared-tools/shared-fixture-tests.log
|
||||
exit 1
|
||||
fi
|
||||
cp "${CARGO_TARGET_DIR}/release/heal-shared-fixture" shared-tools/
|
||||
cp shared-build/Cargo.lock shared-tools/shared-Cargo.lock
|
||||
python3 - <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
source = json.loads(Path("shared-build/source.json").read_text())
|
||||
source.update(schemaVersion=1, built=True, sourceUnmodified=True,
|
||||
candidateSourceTree=subprocess.check_output(["git", "-C", "candidate", "rev-parse", "HEAD^{tree}"], text=True).strip(),
|
||||
testingCommit=os.environ["AUTO_TESTING_SHA"],
|
||||
binarySha256=hashlib.sha256(Path("shared-tools/heal-shared-fixture").read_bytes()).hexdigest(),
|
||||
sharedLockSha256=hashlib.sha256(Path("shared-tools/shared-Cargo.lock").read_bytes()).hexdigest(),
|
||||
toolchain=subprocess.check_output(["rustc", "-vV"], text=True), target="x86_64-unknown-linux-gnu",
|
||||
runId=os.environ["GITHUB_RUN_ID"], runAttempt=os.environ["GITHUB_RUN_ATTEMPT"])
|
||||
Path("shared-tools/heal-shared-fixture.json").write_text(json.dumps(source, indent=2) + "\n")
|
||||
PY
|
||||
git -C candidate diff --exit-code
|
||||
|
||||
- name: Upload controlled C10 fixture encoder evidence
|
||||
if: ${{ always() && inputs.build_shared_heal }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-validation-shared-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: shared-tools/
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload immutable validation tools
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-validation-tools-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: validation-tools/
|
||||
if-no-files-found: error
|
||||
@@ -16,8 +16,8 @@
|
||||
#
|
||||
# Forces a job failure and then runs .github/actions/schedule-failure-issue
|
||||
# through the exact `needs` + `always() && contains(needs.*.result,
|
||||
# 'failure')` wiring used by the real consumers (e2e-s3tests, mint, fuzz).
|
||||
# Dispatch it twice to verify both the issue-creation and
|
||||
# 'failure')` wiring used by the real consumers (e2e-s3tests, mint, fuzz,
|
||||
# performance-ab). Dispatch it twice to verify both the issue-creation and
|
||||
# the dedupe-comment paths, then close the resulting
|
||||
# "[scheduled-failure] Schedule Failure Alert Drill" issue in rustfs/backlog
|
||||
# (the default target repository). Requires the BACKLOG_ISSUE_TOKEN secret
|
||||
|
||||
@@ -28,6 +28,7 @@ on:
|
||||
- "mint"
|
||||
- "minio-interop"
|
||||
- "Nightly GNU Build"
|
||||
- "Performance A/B"
|
||||
- "Runner Hygiene"
|
||||
types: [completed]
|
||||
|
||||
|
||||
Generated
+86
-169
@@ -365,9 +365,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
|
||||
|
||||
[[package]]
|
||||
name = "arrow"
|
||||
version = "59.3.2"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f04e0a89cc175eaf3740282537851e6fd9d7276f97730b9d0ea2859c11a85df"
|
||||
checksum = "4c25317ebddb8f2e6f006759336c95e5ded847a446f3524c27853bfe7d77a69b"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -386,9 +386,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-arith"
|
||||
version = "59.3.2"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "157943c720919e8f5b6731407ce5cd4c9a5ef47a75bf10edaf24b7081b5763ef"
|
||||
checksum = "2ab555c0ffe9cb817041c0cee80baa9663a22225d0a035693579771f2693cadc"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -400,9 +400,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-array"
|
||||
version = "59.3.2"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b5b1682924f11ecf6929689c16cbc69efbb6dd141b06ad98ee1cb10d858c43"
|
||||
checksum = "2dae1d91737421e92c6dd7c7f9d0f2d25198fd6fc539738b628f4dad10adf533"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow-buffer",
|
||||
@@ -419,9 +419,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-buffer"
|
||||
version = "59.3.2"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5d687afc6c927965a8a759f3b68de91933d9dfa2e1bb6e156bac84b85665e92"
|
||||
checksum = "1fe9cd0064f2d7cf8ecfaf77cd70e67a980bc71aea92b63102bd37a7d3379a09"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"half",
|
||||
@@ -431,9 +431,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-cast"
|
||||
version = "59.3.2"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f23c3379feede42135ef27ca60e67f6e643cd4343ae74e4c8b16c83f55afd85"
|
||||
checksum = "ee510bc252781d700400979a96bd1eadebeae23901132bbabd92118da1b376c5"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -453,9 +453,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-csv"
|
||||
version = "59.3.2"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85b1984628b091b8319b3c305fce21d04f9e9b2578bb733b0d0108462ae1d698"
|
||||
checksum = "ace83d5935421b950fea3a0a3f9972c0837c297ef98ade57127a5661b681112c"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-cast",
|
||||
@@ -468,9 +468,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-data"
|
||||
version = "59.3.2"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb0a031c7ff5e2bdb443117054fb466af1d4df5b7f0ecbe197713dc8b113b1ba"
|
||||
checksum = "54f128c73bb7fe9f7137218f910edd5b436f390e0f040b1956f5028390787897"
|
||||
dependencies = [
|
||||
"arrow-buffer",
|
||||
"arrow-schema",
|
||||
@@ -481,9 +481,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-ipc"
|
||||
version = "59.3.2"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f5dd1afadda2eb821aaf851bb4c0adef60c30cc0f1a29bd02fb99dd07fb9f58d"
|
||||
checksum = "8e244f39b82fa7a823e8b5984a0bd988234f9c92c1df79378fb85090e6e4c821"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -497,9 +497,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-json"
|
||||
version = "59.3.2"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2bc4d56a0dad84cc51c5812ccff48b642047a1f3be80349cd84318983e79edb"
|
||||
checksum = "727b21ebf8894dcd9504d6ab66bffdac338391e1f85cde594526a0210299ea40"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -522,9 +522,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-ord"
|
||||
version = "59.3.2"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "561b8209faa2cd489764b7fd07d7cb9e8d274d78048c38de59f77c41140cb834"
|
||||
checksum = "70d376b9b883239ae0f9963cdeeb59126ccaac08d60adf208a6ad3e2dd4048c3"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -535,9 +535,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-row"
|
||||
version = "59.3.2"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d8c68b87a569f92b90c46cb0a09d86bd9609ddc9b1bf1fc5a503c8c6907c14e"
|
||||
checksum = "48096811f92b5c3a0c59d33cfecd7d7d2ed037d81e54a27a42d232e7700e6de2"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -548,9 +548,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-schema"
|
||||
version = "59.3.2"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e87acb0c6bf6237fca7a50d7b51e801c98a18a5fe6f3bb34f7d77b85ca165388"
|
||||
checksum = "37b6c8a6b12cf3869d299ce36ed8e1cd6fe9836264b68fb6c6538473a36124a1"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_json",
|
||||
@@ -558,9 +558,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-select"
|
||||
version = "59.3.2"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c844141958d805b47d1208e0d12aac53fcf388ce8011639fe82aa492adde0bb4"
|
||||
checksum = "e523e85fa9fb3181a391bf0edfa6a1ae0dc46df8cbf7f6b96034dcb1e2a7a04a"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow-array",
|
||||
@@ -572,9 +572,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-string"
|
||||
version = "59.3.2"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc130892b1a9552da77971924b390e06cf375d1f7e4344a7b7b40ca4426fa470"
|
||||
checksum = "b931a905a0e4b8832d9f54fb369cf6e545f9dc03cb3d9611216a7bd2653df60a"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -1514,23 +1514,6 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bindgen"
|
||||
version = "0.73.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "787ef8ef523575546b106a58213d6e6b06198a05c2f757258c68a74273670cfa"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"cexpr",
|
||||
"clang-sys",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
"shlex",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "binrw"
|
||||
version = "0.15.2"
|
||||
@@ -1743,13 +1726,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brotli"
|
||||
version = "9.1.0"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6737ce8b81460bf9407909f73ff50bf13f7cb8577e1e76715f57ed0eba58ba44"
|
||||
checksum = "f8b851b75c23ca7873623d612fe49bd1989aeb03d08fb9432187eb253d3d4c6b"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib 3.0.0",
|
||||
"alloc-stdlib 0.3.0",
|
||||
"brotli-decompressor 6.2.0",
|
||||
"brotli-decompressor 6.0.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1764,9 +1747,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "brotli-decompressor"
|
||||
version = "6.2.0"
|
||||
version = "6.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34c6783f42c21c952ad7112b354f0ba2f10760e9decea5b1b60523d6e270faaf"
|
||||
checksum = "941cd9bd4ddab83cb46fa5a2d428f1c857b24ac78cb876cf7beb710840934bd7"
|
||||
dependencies = [
|
||||
"alloc-no-stdlib 3.0.0",
|
||||
"alloc-stdlib 0.3.0",
|
||||
@@ -1984,15 +1967,6 @@ dependencies = [
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cexpr"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
|
||||
dependencies = [
|
||||
"nom 7.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.5"
|
||||
@@ -2109,17 +2083,6 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clang-sys"
|
||||
version = "1.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a"
|
||||
dependencies = [
|
||||
"glob",
|
||||
"libc",
|
||||
"libloading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.7"
|
||||
@@ -2236,7 +2199,7 @@ version = "0.4.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe"
|
||||
dependencies = [
|
||||
"brotli 9.1.0",
|
||||
"brotli 9.0.0",
|
||||
"bzip2",
|
||||
"compression-core",
|
||||
"flate2",
|
||||
@@ -2244,7 +2207,7 @@ dependencies = [
|
||||
"lz4",
|
||||
"memchr",
|
||||
"zstd 0.14.3",
|
||||
"zstd-safe",
|
||||
"zstd-safe 8.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2579,9 +2542,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-bigint"
|
||||
version = "0.7.10"
|
||||
version = "0.7.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c035cb64cba03dad2fb921f3e218785e8cf9a41790da5eb3a1400f62fa2be11"
|
||||
checksum = "cc907c45bb7e9ef8fc3778defd0f3cd8c81288f1b4e748eba064948f20c47122"
|
||||
dependencies = [
|
||||
"cpubits",
|
||||
"ctutils",
|
||||
@@ -2657,9 +2620,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ctutils"
|
||||
version = "0.4.6"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe3e9d21d39e6f39ff46e03d5b621424f88ee23d8a1672c6bfc9ccc7f2da4f6a"
|
||||
checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0"
|
||||
dependencies = [
|
||||
"cmov",
|
||||
"subtle",
|
||||
@@ -3669,9 +3632,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "der"
|
||||
version = "0.8.3"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a996a108feb8b427a6e12fcf6af4aa83409b79d72c15337d7741368131ee004"
|
||||
checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a"
|
||||
dependencies = [
|
||||
"const-oid 0.10.2",
|
||||
"pem-rfc7468 1.0.0",
|
||||
@@ -4077,7 +4040,6 @@ dependencies = [
|
||||
"rustfs-utils",
|
||||
"rustls",
|
||||
"s3s",
|
||||
"same-file",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
@@ -4104,7 +4066,7 @@ version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0"
|
||||
dependencies = [
|
||||
"der 0.8.3",
|
||||
"der 0.8.2",
|
||||
"digest 0.11.3",
|
||||
"elliptic-curve",
|
||||
"rfc6979",
|
||||
@@ -4169,9 +4131,9 @@ checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
|
||||
|
||||
[[package]]
|
||||
name = "elliptic-curve"
|
||||
version = "0.14.3"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "900f37e7e18d74a2347389af6cd69e3a30d5cc9e697f40004865679aa75e9c7c"
|
||||
checksum = "0ad4496221fcfc21015fd8c0e26fcfab00504832dc0604f52acdd7bad2a8dc81"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"crypto-bigint",
|
||||
@@ -5273,9 +5235,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath"
|
||||
version = "0.29.2"
|
||||
version = "0.29.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b4af2b5ecf2650278fe33b0e49b68c8c484d2306475c127628316dea740cf5c"
|
||||
checksum = "62f2351902fa259b758c7303505273393f6b08172ab5bdd5739a2d2bc6d554e8"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -5308,15 +5270,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-drain"
|
||||
version = "0.29.2"
|
||||
version = "0.29.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a5cceb6c15fbaaeb00b51acc7be7fad533d0b56cb422b1cfc16cf9ca1624d33a"
|
||||
checksum = "83dab2340134d9976d62f9f5172f9c6584c7f521b092f258e0e58e412e160b53"
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros"
|
||||
version = "0.29.2"
|
||||
version = "0.29.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08c0a42eb299ebb7ec6b9161489e5bf132603a34ca12cecadb8490937aa0492a"
|
||||
checksum = "a04bbe5701b097f224a08187a78b75bb9d9b4bbaff6a279c2a50ba73edbb7aac"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -6436,7 +6398,7 @@ dependencies = [
|
||||
"hashbrown 0.16.1",
|
||||
"indexmap 2.14.2",
|
||||
"metrics",
|
||||
"ordered-float 5.5.1",
|
||||
"ordered-float 5.5.0",
|
||||
"quanta",
|
||||
"radix_trie",
|
||||
"rand 0.9.5",
|
||||
@@ -7370,9 +7332,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ordered-float"
|
||||
version = "5.5.1"
|
||||
version = "5.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f59006ce1e204c30956638535b74d317bca99477b9356dfaec54c37f6f600ab"
|
||||
checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
@@ -7489,9 +7451,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pageant"
|
||||
version = "0.2.5"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0bc922d9aa8cfcf598de27b3c6122cb2eeda9d6fe71527a22ebcb7f633540ff"
|
||||
checksum = "f0786409b5aaa884655c9b4065681db0835db0cc185a6fb19fb17a5d38656866"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"byteorder",
|
||||
@@ -7560,9 +7522,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "parquet"
|
||||
version = "59.3.2"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8eb260d566f2fbdabe356d26aaa5d13403aac29b5b4e44463b8f9bd7b2b12680"
|
||||
checksum = "8b3d2ef0fb136e9341570f219ecb168902e76079c6d81ad555f0a4840b1e08a3"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow-array",
|
||||
@@ -7847,7 +7809,7 @@ version = "0.8.0-rc.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "986d2e952779af96ea048f160fd9194e1751b4faea78bcf3ceb456efe008088e"
|
||||
dependencies = [
|
||||
"der 0.8.3",
|
||||
"der 0.8.2",
|
||||
"spki 0.8.1",
|
||||
]
|
||||
|
||||
@@ -7890,7 +7852,7 @@ dependencies = [
|
||||
"aes 0.9.3",
|
||||
"aes-gcm",
|
||||
"cbc 0.2.1",
|
||||
"der 0.8.3",
|
||||
"der 0.8.2",
|
||||
"pbkdf2 0.13.0",
|
||||
"rand_core 0.10.1",
|
||||
"scrypt 0.12.0",
|
||||
@@ -7914,7 +7876,7 @@ version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7"
|
||||
dependencies = [
|
||||
"der 0.8.3",
|
||||
"der 0.8.2",
|
||||
"pkcs5 0.8.1",
|
||||
"rand_core 0.10.1",
|
||||
"spki 0.8.1",
|
||||
@@ -9176,9 +9138,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "russh"
|
||||
version = "0.65.0"
|
||||
version = "0.64.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3bb77a4b259312b2abbc541bca1c13cb4e9b61c560a7fd894068c3b92e0c9dc3"
|
||||
checksum = "ba61e87b9ec9a39a59a6bbed4c0b8ff7fb07073405b24a767a1f2c8560ac8f58"
|
||||
dependencies = [
|
||||
"aes 0.9.3",
|
||||
"aws-lc-rs",
|
||||
@@ -9193,7 +9155,7 @@ dependencies = [
|
||||
"curve25519-dalek 5.0.0",
|
||||
"data-encoding",
|
||||
"delegate",
|
||||
"der 0.8.3",
|
||||
"der 0.8.2",
|
||||
"digest 0.11.3",
|
||||
"ecdsa",
|
||||
"ed25519-dalek 3.0.0",
|
||||
@@ -10884,7 +10846,7 @@ version = "1.0.2"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"blake2",
|
||||
"brotli 9.1.0",
|
||||
"brotli 9.0.0",
|
||||
"bytes",
|
||||
"convert_case 0.12.0",
|
||||
"crc-fast",
|
||||
@@ -11345,7 +11307,7 @@ checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d"
|
||||
dependencies = [
|
||||
"base16ct",
|
||||
"ctutils",
|
||||
"der 0.8.3",
|
||||
"der 0.8.2",
|
||||
"hybrid-array",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -11511,15 +11473,6 @@ dependencies = [
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_spanned"
|
||||
version = "1.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_urlencoded"
|
||||
version = "0.7.1"
|
||||
@@ -11799,9 +11752,9 @@ checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256"
|
||||
|
||||
[[package]]
|
||||
name = "sketches-ddsketch"
|
||||
version = "0.3.2"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "45f82fa313d625dd72aaa65ec0c9f8c95407add0510c1bef3c18ea7257ec34ce"
|
||||
checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
@@ -11930,7 +11883,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der 0.8.3",
|
||||
"der 0.8.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12679,30 +12632,6 @@ dependencies = [
|
||||
"webpki-roots 0.26.11",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml"
|
||||
version = "0.9.12+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"
|
||||
dependencies = [
|
||||
"indexmap 2.14.2",
|
||||
"serde_core",
|
||||
"serde_spanned",
|
||||
"toml_datetime 0.7.5+spec-1.1.0",
|
||||
"toml_parser",
|
||||
"toml_writer",
|
||||
"winnow 0.7.15",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_datetime"
|
||||
version = "0.7.5+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_datetime"
|
||||
version = "1.1.2+spec-1.1.0"
|
||||
@@ -12719,9 +12648,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c"
|
||||
dependencies = [
|
||||
"indexmap 2.14.2",
|
||||
"toml_datetime 1.1.2+spec-1.1.0",
|
||||
"toml_datetime",
|
||||
"toml_parser",
|
||||
"winnow 1.0.4",
|
||||
"winnow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12730,15 +12659,9 @@ version = "1.1.5+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c"
|
||||
dependencies = [
|
||||
"winnow 1.0.4",
|
||||
"winnow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_writer"
|
||||
version = "1.1.3+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770"
|
||||
|
||||
[[package]]
|
||||
name = "tonic"
|
||||
version = "0.14.6"
|
||||
@@ -13765,12 +13688,6 @@ version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "0.7.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "1.0.4"
|
||||
@@ -14037,9 +13954,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.1.2"
|
||||
version = "9.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec2e3161de27091f18a831d9494e9c53378a34833af5c1dda35ad85849da18d6"
|
||||
checksum = "2886b00003a72c4ccee3f42cfc34908d33b2508a74d827dc2a4244706475c704"
|
||||
dependencies = [
|
||||
"aes 0.9.3",
|
||||
"bzip2",
|
||||
@@ -14091,18 +14008,10 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
|
||||
dependencies = [
|
||||
"apache-avro",
|
||||
"arrow-array",
|
||||
"arrow-ipc",
|
||||
"arrow-schema",
|
||||
"bytes",
|
||||
"kafka-protocol",
|
||||
"parquet",
|
||||
"semver",
|
||||
"toml",
|
||||
"zstd 0.14.3",
|
||||
"zstd-safe",
|
||||
"zstd-safe 7.3.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -14111,7 +14020,16 @@ version = "0.14.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70"
|
||||
dependencies = [
|
||||
"zstd-safe",
|
||||
"zstd-safe 8.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-safe"
|
||||
version = "7.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882"
|
||||
dependencies = [
|
||||
"zstd-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -14129,7 +14047,6 @@ version = "2.1.2+zstd.1.5.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
+4
-7
@@ -66,7 +66,6 @@ members = [
|
||||
"crates/io-metrics", # I/O metrics collection for performance analysis
|
||||
"crates/io-core", # Buffered Bytes, mmap-then-copy, and aligned pread I/O helpers
|
||||
"crates/zip", # ZIP file handling and compression
|
||||
"crates/zstd-compat", # Fixed codec for dependencies requiring zstd 0.13
|
||||
]
|
||||
resolver = "3"
|
||||
|
||||
@@ -261,7 +260,7 @@ aws-smithy-http-client = { default-features = false, version = "1.5.1" }
|
||||
aws-smithy-runtime-api = { version = "1.19.1" }
|
||||
aws-smithy-types = { version = "1.9.0" }
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "9.1.0"
|
||||
brotli = "9.0.0"
|
||||
clap = { version = "4.6.7" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.12.0"
|
||||
@@ -303,7 +302,6 @@ num_cpus = { version = "1.17.0" }
|
||||
nvml-wrapper = "0.13.0"
|
||||
object = { version = "=0.40.0", default-features = false }
|
||||
parking_lot = "0.13.0"
|
||||
same-file = "1.0.6"
|
||||
path-absolutize = "4.0.1"
|
||||
percent-encoding = "2.3.2"
|
||||
# Server-side QR rendering for TOTP enrollment, so neither the console nor the
|
||||
@@ -356,7 +354,7 @@ winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.1.2"
|
||||
zip = "9.1.0"
|
||||
zstd = "0.14.3"
|
||||
|
||||
# Observability and Metrics
|
||||
@@ -377,7 +375,7 @@ libunftp = { version = "0.23.1" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.0" }
|
||||
russh = { version = "0.64.1" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
@@ -386,7 +384,7 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.2" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.29.2", default-features = false }
|
||||
hotpath = { version = "0.29.1", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
@@ -426,7 +424,6 @@ strip = "none"
|
||||
|
||||
# Pin upstream fixes until compatible crates.io releases are available.
|
||||
[patch.crates-io]
|
||||
zstd = { path = "crates/zstd-compat" }
|
||||
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
|
||||
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
|
||||
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
|
||||
|
||||
@@ -146,7 +146,6 @@ zip.workspace = true
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
|
||||
[dev-dependencies]
|
||||
same-file.workspace = true
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["test-util"] }
|
||||
rustfs-heal.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
||||
@@ -20,14 +20,15 @@
|
||||
//! offline before heal converges, a strict latest-listing quorum of 3 can only
|
||||
//! ever observe 2 copies, so ListObjectsV2 silently dropped the object even
|
||||
//! though GetObject (read quorum 2) still succeeded. Exposed by the flaky
|
||||
//! "Mixed-version rolling upgrade from rc.2" CI lane (run 33478999853). The
|
||||
//! listing now follows MinIO's rule: an entry needs `ceil(asked/2)` copies
|
||||
//! among the online drives it asked, so three online drives need two copies
|
||||
//! (see `SetDisks::list_path` in `crates/ecstore/src/store/list_objects.rs`).
|
||||
//! "Mixed-version rolling upgrade from rc.2" CI lane (run 33478999853); the
|
||||
//! product fix relaxes the listing's required object quorum by the number of
|
||||
//! set drives the listing could not consult (see
|
||||
//! `latest_listing_required_object_quorum` in
|
||||
//! `crates/ecstore/src/store/list_objects.rs`).
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment, init_logging};
|
||||
use crate::common::{RustFSTestClusterEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use bytes::Bytes;
|
||||
use std::collections::HashSet;
|
||||
@@ -50,13 +51,13 @@ mod tests {
|
||||
format!("degraded-object-{idx:02}")
|
||||
}
|
||||
|
||||
async fn list_all_keys(client: &Client, bucket: &str) -> Result<HashSet<String>, Box<dyn Error + Send + Sync>> {
|
||||
async fn list_all_keys(client: &Client) -> Result<HashSet<String>, Box<dyn Error + Send + Sync>> {
|
||||
let mut keys = HashSet::new();
|
||||
let mut continuation_token: Option<String> = None;
|
||||
loop {
|
||||
let response = client
|
||||
.list_objects_v2()
|
||||
.bucket(bucket)
|
||||
.bucket(BUCKET)
|
||||
.set_continuation_token(continuation_token.clone())
|
||||
.send()
|
||||
.await?;
|
||||
@@ -146,7 +147,7 @@ mod tests {
|
||||
let expected: HashSet<String> = (0..OBJECT_COUNT).map(object_key).collect();
|
||||
let deadline = Instant::now() + LISTING_DEADLINE;
|
||||
let listed = loop {
|
||||
let listed = match list_all_keys(&client, BUCKET).await {
|
||||
let listed = match list_all_keys(&client).await {
|
||||
Ok(keys) => keys,
|
||||
Err(error) if Instant::now() < deadline => {
|
||||
info!("retrying degraded listing: {error}");
|
||||
@@ -170,97 +171,4 @@ mod tests {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 4-node x 4-drive cluster (one 16-drive set), #8828 shape:
|
||||
/// 1. PUT objects while the storage class is EC:8, so each is 8+8.
|
||||
/// 2. Restart with the EC:4 default, as after a storage-class change.
|
||||
/// 3. Stop two nodes (8 drives). Each object keeps eight data shards, so
|
||||
/// GET succeeds and ListObjectsV2 must list every key: the listing
|
||||
/// quorum is half of the eight drives it can ask.
|
||||
/// 4. Stop a third node. Four online drives are below the set's read
|
||||
/// quorum, so LIST must report 503 instead of a partial page.
|
||||
#[tokio::test]
|
||||
async fn ec8_objects_stay_listable_with_half_of_an_ec4_set_offline() -> TestResult {
|
||||
const EC8_BUCKET: &str = "degraded-listing-ec8";
|
||||
init_logging();
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, 4)).await?;
|
||||
cluster.set_env("RUSTFS_HEAL_ENABLED", "false");
|
||||
cluster.set_env("RUSTFS_SCANNER_ENABLED", "false");
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:8");
|
||||
cluster.start().await?;
|
||||
cluster.create_test_bucket(EC8_BUCKET).await?;
|
||||
let client = cluster.create_s3_client(0)?;
|
||||
for idx in 0..OBJECT_COUNT {
|
||||
let key = object_key(idx);
|
||||
client
|
||||
.put_object()
|
||||
.bucket(EC8_BUCKET)
|
||||
.key(&key)
|
||||
.body(Bytes::from(format!("ec8 listing payload {idx}")).into())
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format!("EC:8 PUT for {key} failed: {error}"))?;
|
||||
}
|
||||
|
||||
info!("restarting with the EC:4 default storage class");
|
||||
cluster.stop();
|
||||
// A later entry for the same variable wins when the node process starts.
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4");
|
||||
cluster.start().await?;
|
||||
let client = cluster.create_s3_client(0)?;
|
||||
|
||||
info!("stopping nodes 2 and 3 (eight of sixteen drives)");
|
||||
cluster.stop_node(2)?;
|
||||
cluster.stop_node(3)?;
|
||||
for idx in 0..OBJECT_COUNT {
|
||||
let key = object_key(idx);
|
||||
let deadline = Instant::now() + GET_RETRY_DEADLINE;
|
||||
loop {
|
||||
match client.get_object().bucket(EC8_BUCKET).key(&key).send().await {
|
||||
Ok(response) => {
|
||||
let body = response.body.collect().await?.into_bytes();
|
||||
assert_eq!(body.as_ref(), format!("ec8 listing payload {idx}").as_bytes());
|
||||
break;
|
||||
}
|
||||
Err(error) if Instant::now() < deadline => {
|
||||
info!("retrying EC:8 GET for {key}: {error}");
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
Err(error) => return Err(format!("EC:8 object {key} failed GET with eight drives: {error}").into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let expected: HashSet<String> = (0..OBJECT_COUNT).map(object_key).collect();
|
||||
let deadline = Instant::now() + LISTING_DEADLINE;
|
||||
loop {
|
||||
match list_all_keys(&client, EC8_BUCKET).await {
|
||||
Ok(listed) if listed == expected => break,
|
||||
Ok(listed) => assert!(
|
||||
Instant::now() < deadline,
|
||||
"EC:8 objects readable with eight drives stayed missing from ListObjectsV2: listed={listed:?}"
|
||||
),
|
||||
Err(error) => assert!(
|
||||
Instant::now() < deadline,
|
||||
"ListObjectsV2 kept failing with eight of sixteen drives online: {error}"
|
||||
),
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
|
||||
info!("stopping node 1 (four of sixteen drives remain)");
|
||||
cluster.stop_node(1)?;
|
||||
let deadline = Instant::now() + LISTING_DEADLINE;
|
||||
loop {
|
||||
match client.list_objects_v2().bucket(EC8_BUCKET).send().await {
|
||||
Err(error) if error.raw_response().is_some_and(|response| response.status().as_u16() == 503) => break,
|
||||
Err(error) => assert!(Instant::now() < deadline, "below the set read quorum LIST must report 503, got {error}"),
|
||||
Ok(response) => panic!("below the set read quorum LIST must fail, but listed {} keys", response.contents().len()),
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,20 +13,18 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, get_object_bytes, payload_for, put_object,
|
||||
retrying_get_equals, retrying_put, s3_probe_client, s3_probe_error_code, unique_bucket, wait_for_ready, wait_until,
|
||||
DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, retrying_get_equals, unique_bucket,
|
||||
wait_for_ready, wait_until,
|
||||
};
|
||||
use crate::chaos::{census_object_version_on_disk, signed_admin_post};
|
||||
use crate::common::{build_test_s3_config, init_logging};
|
||||
use crate::fault_proxy::FaultMode;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::operation::{get_object::GetObjectError, put_object::PutObjectError};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{Barrier, mpsc};
|
||||
use tokio::time::{Instant, timeout};
|
||||
use tokio::time::timeout;
|
||||
|
||||
#[tokio::test]
|
||||
async fn kill_and_restart_node_preserves_objects() -> TestResult {
|
||||
@@ -46,133 +44,6 @@ async fn kill_and_restart_node_preserves_objects() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ec4_partial_node_return_restores_read_write_quorum() -> TestResult {
|
||||
init_logging();
|
||||
let mut dist = DistCluster::start_with_env(
|
||||
DistLayout::FourByFour,
|
||||
&[
|
||||
("RUSTFS_ERASURE_SET_DRIVE_COUNT", "16"),
|
||||
("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
let storage = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
|
||||
let storage: serde_json::Value = serde_json::from_str(&storage)?;
|
||||
let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
|
||||
assert_eq!(storage.disks.len(), 16);
|
||||
assert_eq!(storage.backend.total_sets, vec![1]);
|
||||
assert_eq!(storage.backend.drives_per_set, vec![16]);
|
||||
assert_eq!(storage.backend.standard_sc_parity, Some(4));
|
||||
|
||||
let bucket = unique_bucket("partial-ec4");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let client = s3_probe_client(&dist.client(0)?);
|
||||
let baseline = payload_for("partial-recovery/baseline", 8 * 1024 * 1024);
|
||||
put_object(&client, &bucket, "baseline.bin", baseline.clone()).await?;
|
||||
wait_until(
|
||||
Duration::from_secs(30),
|
||||
|| async {
|
||||
for node in &dist.cluster.nodes {
|
||||
for directory in &node.data_dirs {
|
||||
let census = census_object_version_on_disk(std::path::Path::new(directory), &bucket, "baseline.bin", None)?;
|
||||
if !census.is_complete() {
|
||||
return Ok(false);
|
||||
}
|
||||
assert_eq!(census.data_blocks, Some(12), "the object must require twelve data shards");
|
||||
assert_eq!(census.parity_blocks, Some(4), "the object must have four parity shards");
|
||||
assert!(!census.expected_part_numbers.is_empty(), "the baseline must use physical shards");
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
"the baseline is complete on all sixteen EC:4 drives",
|
||||
)
|
||||
.await?;
|
||||
|
||||
dist.cluster.stop_node(2)?;
|
||||
dist.cluster.stop_node(3)?;
|
||||
let read_error = timeout(Duration::from_secs(60), get_object_bytes(&client, &bucket, "baseline.bin"))
|
||||
.await?
|
||||
.expect_err("eight unavailable drives must prevent the baseline GET");
|
||||
assert!(
|
||||
matches!(
|
||||
s3_probe_error_code(read_error.as_ref()),
|
||||
Some("SlowDown" | "SlowDownRead" | "ServiceUnavailable")
|
||||
),
|
||||
"quorum loss must produce an availability error: {read_error:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
read_error
|
||||
.downcast_ref::<SdkError<GetObjectError>>()
|
||||
.and_then(SdkError::raw_response)
|
||||
.map(|response| response.status().as_u16()),
|
||||
Some(503),
|
||||
"read quorum loss must remain a service availability failure"
|
||||
);
|
||||
let write_error = timeout(Duration::from_secs(60), put_object(&client, &bucket, "unavailable.bin", baseline.clone()))
|
||||
.await?
|
||||
.expect_err("eight unavailable drives must prevent a successful PUT");
|
||||
assert!(
|
||||
matches!(s3_probe_error_code(write_error.as_ref()), Some("SlowDown" | "ServiceUnavailable")),
|
||||
"quorum loss must produce an availability error: {write_error:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
write_error
|
||||
.downcast_ref::<SdkError<PutObjectError>>()
|
||||
.and_then(SdkError::raw_response)
|
||||
.map(|response| response.status().as_u16()),
|
||||
Some(503),
|
||||
"write quorum loss must remain a service availability failure"
|
||||
);
|
||||
|
||||
// Recover only four drives. The remaining node must stay stopped while
|
||||
// every live coordinator serves old data and acknowledges new writes.
|
||||
dist.cluster.start_node_process(2).await?;
|
||||
let recovery_deadline = Instant::now() + Duration::from_secs(90);
|
||||
let mut recovered_objects = Vec::new();
|
||||
for node in 0..3 {
|
||||
assert!(dist.cluster.nodes[3].process.is_none(), "the fourth node must remain unavailable");
|
||||
let client = s3_probe_client(&dist.client(node)?);
|
||||
retrying_get_equals(
|
||||
&client,
|
||||
&bucket,
|
||||
"baseline.bin",
|
||||
&baseline,
|
||||
recovery_deadline.saturating_duration_since(Instant::now()),
|
||||
)
|
||||
.await?;
|
||||
let key = format!("partial-{node}.bin");
|
||||
let body = payload_for(&key, 1024 * 1024);
|
||||
retrying_put(
|
||||
&client,
|
||||
&bucket,
|
||||
&key,
|
||||
body.clone(),
|
||||
recovery_deadline.saturating_duration_since(Instant::now()),
|
||||
)
|
||||
.await?;
|
||||
recovered_objects.push((key, body));
|
||||
}
|
||||
for node in 0..3 {
|
||||
assert!(dist.cluster.nodes[3].process.is_none(), "reads must complete before all drives return");
|
||||
let client = s3_probe_client(&dist.client(node)?);
|
||||
assert_object_bytes(&client, &bucket, "baseline.bin", &baseline).await?;
|
||||
for (key, body) in &recovered_objects {
|
||||
assert_object_bytes(&client, &bucket, key, body).await?;
|
||||
}
|
||||
}
|
||||
|
||||
dist.cluster.start_node(3).await?;
|
||||
wait_for_ready(&dist.cluster).await?;
|
||||
let client = s3_probe_client(&dist.client(3)?);
|
||||
assert_object_bytes(&client, &bucket, "baseline.bin", &baseline).await?;
|
||||
for (key, body) in &recovered_objects {
|
||||
assert_object_bytes(&client, &bucket, key, body).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_cluster_restart_preserves_objects() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
@@ -1181,16 +1181,11 @@ pub(crate) async fn retrying_get_equals(
|
||||
)
|
||||
.into()),
|
||||
Err(error) => {
|
||||
let retryable = match s3_probe_error_code(error.as_ref()) {
|
||||
Some("SlowDown" | "ServiceUnavailable") => true,
|
||||
// GET preserves this compatibility code for a lost data quorum.
|
||||
Some("SlowDownRead") => error
|
||||
.downcast_ref::<SdkError<GetObjectError>>()
|
||||
.and_then(SdkError::raw_response)
|
||||
.is_some_and(|response| response.status().as_u16() == 503),
|
||||
_ => false,
|
||||
};
|
||||
if retryable { Ok(false) } else { Err(error) }
|
||||
if matches!(s3_probe_error_code(error.as_ref()), Some("SlowDown" | "ServiceUnavailable")) {
|
||||
Ok(false)
|
||||
} else {
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -1329,62 +1324,6 @@ mod retry_tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn retrying_get_allows_read_quorum_recovery() -> TestResult {
|
||||
use aws_sdk_s3::config::Region;
|
||||
use aws_smithy_http_client::test_util::{ReplayEvent, StaticReplayClient};
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
|
||||
let endpoint = "http://read-quorum.example.invalid";
|
||||
let event = |status: u16, body: &'static str| {
|
||||
ReplayEvent::new(
|
||||
http::Request::builder()
|
||||
.method(Method::GET)
|
||||
.uri(format!("{endpoint}/retry-probe/key?x-id=GetObject"))
|
||||
.body(SdkBody::empty())
|
||||
.expect("build replay request"),
|
||||
http::Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len())
|
||||
.body(SdkBody::from(body))
|
||||
.expect("build replay response"),
|
||||
)
|
||||
};
|
||||
for status in [503, 403, 500] {
|
||||
let transport = StaticReplayClient::new(vec![
|
||||
event(status, "<Error><Code>SlowDownRead</Code></Error>"),
|
||||
event(200, "expected bytes"),
|
||||
]);
|
||||
let client = Client::from_conf(
|
||||
aws_sdk_s3::Config::builder()
|
||||
.behavior_version_latest()
|
||||
.credentials_provider(Credentials::new(FAKE_ACCESS_KEY, FAKE_SECRET_KEY, None, None, "read-quorum-test"))
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(endpoint)
|
||||
.force_path_style(true)
|
||||
.retry_config(RetryConfig::disabled())
|
||||
.http_client(transport.clone())
|
||||
.build(),
|
||||
);
|
||||
let result = retrying_get_equals(&client, "retry-probe", "key", b"expected bytes", Duration::from_secs(5)).await;
|
||||
if status == 503 {
|
||||
result?;
|
||||
assert_eq!(
|
||||
transport.actual_requests().count(),
|
||||
2,
|
||||
"the probe must retry the read-quorum error exactly once"
|
||||
);
|
||||
transport.assert_requests_match(&[]);
|
||||
} else {
|
||||
let error = result.expect_err("the compatibility code must not hide a non-503 failure");
|
||||
assert_eq!(s3_probe_error_code(error.as_ref()), Some("SlowDownRead"));
|
||||
assert_eq!(transport.actual_requests().count(), 1, "non-availability errors must fail immediately");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_polls_distinguish_committed_and_replicated_keys() -> TestResult {
|
||||
let target = FakeS3Target::start().await?;
|
||||
|
||||
@@ -31,7 +31,6 @@ mod multipart_fence_acceptance_test;
|
||||
mod multipart_rpc_delay_test;
|
||||
mod object_lock_test;
|
||||
mod observability_test;
|
||||
mod publication_log;
|
||||
mod replication_delete_marker_test;
|
||||
mod replication_quota_test;
|
||||
mod s3_basic_test;
|
||||
|
||||
@@ -1638,191 +1638,6 @@ async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation(
|
||||
"refused late packets must not recreate staging"
|
||||
);
|
||||
}
|
||||
late_part_packets_preserve_retired_and_fresh_uploads().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
|
||||
for (method, settles) in [("PreparePartTransaction", false), ("SettlePartTransaction", true)] {
|
||||
for recreate_bucket in [false, true] {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let key = "late-part-transaction";
|
||||
let previous = payload_for("committed-before-late-part", 4096);
|
||||
clients[0]
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(previous.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let upload = clients[0].create_multipart_upload().bucket(&bucket).key(key).send().await?;
|
||||
let upload_id = upload.upload_id().ok_or("missing original upload ID")?;
|
||||
clients[0]
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(vec![0x41; 9 * 1024 * 1024]))
|
||||
.send()
|
||||
.await?;
|
||||
let path = upload_path_scope(&bucket, key, upload_id)?;
|
||||
let hash = sha256_hex(format!("{bucket}/{key}").as_bytes());
|
||||
let roots = cluster
|
||||
.nodes
|
||||
.iter()
|
||||
.flat_map(|node| &node.data_dirs)
|
||||
.map(|root| Path::new(root).join(RUSTFS_META_MULTIPART_BUCKET).join(&path))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(roots.len(), 4);
|
||||
let original = multipart_sources(&cluster, Some(&hash))?;
|
||||
assert_eq!(original.len(), 4);
|
||||
assert!(
|
||||
original.values().all(|source| source
|
||||
.files
|
||||
.keys()
|
||||
.any(|path| path.file_name().and_then(|name| name.to_str()) == Some("part.1"))),
|
||||
"all selected peers originally own the acknowledged part"
|
||||
);
|
||||
let scope = upload_scope(upload_id)?;
|
||||
// Retain only the original attempt; recovery of the same upload
|
||||
// must be able to issue fresh settlement requests through the gate.
|
||||
let pauses = proxies[1..3]
|
||||
.iter()
|
||||
.map(|proxy| proxy.pause_late(&[method], 1, &scope))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let replacement = clients[0]
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(vec![0x62; 8 * 1024 * 1024]));
|
||||
let caller = tokio::spawn(async move { replacement.send().await });
|
||||
wait_for_faults(&pauses, &[method]).await?;
|
||||
let failure = tokio::time::timeout(DEADLINE, caller)
|
||||
.await??
|
||||
.expect_err("two retained peers cannot acknowledge part replacement");
|
||||
assert_unavailable(&failure, "storage_write_quorum", None);
|
||||
assert!(pauses.iter().all(|pause| pause.hits() == 1 && pause.responses() == 0));
|
||||
let expected_actions = if settles { (1, 0) } else { (0, 0) };
|
||||
assert!(
|
||||
pauses.iter().all(|pause| pause.retained_settle_actions() == expected_actions),
|
||||
"settlement capture must be a real Commit, not Rollback"
|
||||
);
|
||||
abort_after_fault(&clients[0], &bucket, key, upload_id, false).await?;
|
||||
assert!(
|
||||
roots.iter().all(|root| !root.exists()),
|
||||
"ordinary Abort physically retires all original roots"
|
||||
);
|
||||
assert!(multipart_sources(&cluster, Some(&hash))?.is_empty());
|
||||
for client in &clients {
|
||||
assert_object_bytes_after_fault(client, &bucket, key, &previous).await?;
|
||||
}
|
||||
let protected = if recreate_bucket {
|
||||
clients[0].delete_object().bucket(&bucket).key(key).send().await?;
|
||||
clients[0].delete_bucket().bucket(&bucket).send().await?;
|
||||
clients[1].create_bucket().bucket(&bucket).send().await?;
|
||||
let fresh = payload_for("recreated-before-late-part", 4096);
|
||||
clients[1]
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(fresh.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
fresh
|
||||
} else {
|
||||
previous
|
||||
};
|
||||
let fresh = clients[2].create_multipart_upload().bucket(&bucket).key(key).send().await?;
|
||||
let fresh_id = fresh.upload_id().ok_or("missing fresh upload ID")?;
|
||||
let fresh_body = payload_for("fresh-active-part", 4096);
|
||||
let fresh_part = clients[2]
|
||||
.upload_part()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(fresh_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from(fresh_body.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let before = multipart_sources(&cluster, Some(&hash))?;
|
||||
assert_eq!(before.len(), 4, "the new upload exists on every original participant");
|
||||
for pause in &pauses {
|
||||
pause.release();
|
||||
}
|
||||
wait_until(
|
||||
DEADLINE,
|
||||
|| async {
|
||||
Ok(pauses.iter().all(|pause| {
|
||||
pause.responses() == pause.hits()
|
||||
&& if settles {
|
||||
pause.successful_mutation_replies() == pause.hits() && pause.refused_mutations() == 0
|
||||
} else {
|
||||
pause.refused_mutations() == pause.hits() && pause.successful_mutation_replies() == 0
|
||||
}
|
||||
}))
|
||||
},
|
||||
"every original part packet returns its typed authenticated terminal outcome",
|
||||
)
|
||||
.await?;
|
||||
assert!(roots.iter().all(|root| !root.exists()), "late part packets cannot recreate retired roots");
|
||||
assert!(
|
||||
multipart_sources(&cluster, Some(&hash))? == before,
|
||||
"late packets cannot change a fresh upload's shard or control bytes"
|
||||
);
|
||||
for client in &clients {
|
||||
assert_object_bytes_after_fault(client, &bucket, key, &protected).await?;
|
||||
let listed = client
|
||||
.list_parts()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(fresh_id)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(listed.parts().len(), 1);
|
||||
assert_eq!(listed.parts()[0].e_tag(), fresh_part.e_tag());
|
||||
let missing = client
|
||||
.list_parts()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("old upload remains retired");
|
||||
assert_eq!(missing.raw_response().expect("old upload service response").status().as_u16(), 404);
|
||||
assert_eq!(missing.as_service_error().and_then(|error| error.meta().code()), Some("NoSuchUpload"));
|
||||
}
|
||||
clients[2]
|
||||
.complete_multipart_upload()
|
||||
.bucket(&bucket)
|
||||
.key(key)
|
||||
.upload_id(fresh_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.e_tag(fresh_part.e_tag().ok_or("missing fresh part ETag")?)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
for client in &clients {
|
||||
assert_object_bytes_after_fault(client, &bucket, key, &fresh_body).await?;
|
||||
}
|
||||
assert!(multipart_sources(&cluster, Some(&hash))?.is_empty());
|
||||
assert_no_uploads_after_fault(&clients, &bucket).await?;
|
||||
eprintln!(
|
||||
"MPU late part: method={method}, recreate_bucket={recreate_bucket}, retired_roots=4, delivered=2, refused={}, successful_replies={}, fresh_sources_unchanged=4, verified_readers=4",
|
||||
pauses.iter().map(RpcPause::refused_mutations).sum::<usize>(),
|
||||
pauses.iter().map(RpcPause::successful_mutation_replies).sum::<usize>()
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -1,413 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Follow the same append-only log files without rescanning previous JSON rows.
|
||||
|
||||
use same_file::Handle;
|
||||
use serde::{Deserialize, Deserializer};
|
||||
use serde_json::Value;
|
||||
use std::fs::File;
|
||||
use std::io::{self, Read};
|
||||
use std::path::PathBuf;
|
||||
|
||||
// Captured RPC spans already produce valid rows above 2 MiB. Keep complete
|
||||
// rows bounded independently from the much smaller selected-event history.
|
||||
const MAX_LINE_BYTES: usize = 4 << 20;
|
||||
const MAX_EVENTS: usize = 128;
|
||||
const MAX_EVENT_BYTES: usize = 1 << 20;
|
||||
|
||||
fn invalid(reason: &'static str) -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, reason)
|
||||
}
|
||||
|
||||
#[derive(Default, Deserialize)]
|
||||
struct Selector {
|
||||
#[serde(default)]
|
||||
event: Value,
|
||||
#[serde(default)]
|
||||
op: Value,
|
||||
#[serde(default)]
|
||||
bucket: Value,
|
||||
#[serde(default)]
|
||||
object: Value,
|
||||
}
|
||||
|
||||
impl Selector {
|
||||
fn matches(&self, bucket: &str, object: &str) -> bool {
|
||||
self.event.as_str() == Some("object_namespace_lock")
|
||||
&& self.op.as_str() == Some("data_movement_publication")
|
||||
&& self.bucket.as_str() == Some(bucket)
|
||||
&& self.object.as_str() == Some(object)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Probe {
|
||||
#[serde(default)]
|
||||
event: Value,
|
||||
#[serde(default)]
|
||||
op: Value,
|
||||
#[serde(default)]
|
||||
bucket: Value,
|
||||
#[serde(default)]
|
||||
object: Value,
|
||||
// A present null is not a missing fields object. The old reader uses the
|
||||
// root only when this key is absent, including for non-object JSON values.
|
||||
#[serde(default, deserialize_with = "present_fields")]
|
||||
fields: Option<Option<Selector>>,
|
||||
}
|
||||
|
||||
fn present_fields<'de, D: Deserializer<'de>>(d: D) -> Result<Option<Option<Selector>>, D::Error> {
|
||||
Option::<Selector>::deserialize(d).map(Some)
|
||||
}
|
||||
|
||||
impl Probe {
|
||||
fn matches(&self, bucket: &str, object: &str) -> bool {
|
||||
match &self.fields {
|
||||
Some(Some(fields)) => fields.matches(bucket, object),
|
||||
Some(None) => false,
|
||||
None => {
|
||||
self.event.as_str() == Some("object_namespace_lock")
|
||||
&& self.op.as_str() == Some("data_movement_publication")
|
||||
&& self.bucket.as_str() == Some(bucket)
|
||||
&& self.object.as_str() == Some(object)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn select(line: &[u8], bucket: &str, object: &str, full_decodes: &mut u64) -> io::Result<Option<Value>> {
|
||||
let text = std::str::from_utf8(line).map_err(|_| invalid("invalid UTF-8 in a completed log row"))?;
|
||||
// Skip unknown fields without allocating a Value tree for every RPC trace.
|
||||
// A shape/duplicate-field error falls back to the original Value parser;
|
||||
// it must not discard valid escaped strings or last-key-wins JSON.
|
||||
if let Ok(probe) = serde_json::from_str::<Probe>(text)
|
||||
&& !probe.matches(bucket, object)
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
*full_decodes += 1;
|
||||
let Ok(row) = serde_json::from_str::<Value>(text) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let fields = row.get("fields").unwrap_or(&row);
|
||||
if fields.get("event").and_then(Value::as_str) != Some("object_namespace_lock")
|
||||
|| fields.get("op").and_then(Value::as_str) != Some("data_movement_publication")
|
||||
|| fields.get("bucket").and_then(Value::as_str) != Some(bucket)
|
||||
|| fields.get("object").and_then(Value::as_str) != Some(object)
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let mut event = fields.clone();
|
||||
event["timestamp"] = row["timestamp"].clone();
|
||||
Ok(Some(event))
|
||||
}
|
||||
|
||||
struct Tail {
|
||||
path: PathBuf,
|
||||
handle: Handle,
|
||||
offset: u64,
|
||||
pending: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Tail {
|
||||
fn open(path: PathBuf) -> io::Result<Self> {
|
||||
let file = File::open(&path)?;
|
||||
if !file.metadata()?.is_file() {
|
||||
return Err(invalid("publication log is not a regular file"));
|
||||
}
|
||||
Ok(Self {
|
||||
path,
|
||||
handle: Handle::from_file(file)?,
|
||||
offset: 0,
|
||||
pending: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn check_identity(&self) -> io::Result<()> {
|
||||
if Handle::from_path(&self.path)? != self.handle {
|
||||
return Err(invalid("publication log file was replaced"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn poll(&mut self, bytes_read: &mut u64, mut accept: impl FnMut(&[u8]) -> io::Result<()>) -> io::Result<()> {
|
||||
self.check_identity()?;
|
||||
let end = self.handle.as_file().metadata()?.len();
|
||||
if end < self.offset {
|
||||
return Err(invalid("publication log file was truncated"));
|
||||
}
|
||||
// A fixed high-water snapshot bounds each poll even if writers append
|
||||
// faster than the reader. Bytes after it remain for the next poll.
|
||||
let mut chunk = [0_u8; 64 * 1024];
|
||||
while self.offset < end {
|
||||
let size = usize::try_from((end - self.offset).min(64 * 1024)).expect("a log chunk fits usize");
|
||||
self.handle.as_file_mut().read_exact(&mut chunk[..size])?;
|
||||
self.offset += u64::try_from(size).expect("a bounded log read fits u64");
|
||||
*bytes_read += u64::try_from(size).expect("a bounded log read fits u64");
|
||||
let appended_at = self.pending.len();
|
||||
self.pending.extend_from_slice(&chunk[..size]);
|
||||
let mut start = 0;
|
||||
for end in self.pending[appended_at..]
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(i, byte)| (*byte == b'\n').then_some(appended_at + i + 1))
|
||||
{
|
||||
if end - start > MAX_LINE_BYTES {
|
||||
return Err(invalid("publication log row exceeds its fixed bound"));
|
||||
}
|
||||
accept(&self.pending[start..end])?;
|
||||
start = end;
|
||||
}
|
||||
self.pending.drain(..start);
|
||||
if self.pending.len() > MAX_LINE_BYTES {
|
||||
return Err(invalid("unfinished publication log row exceeds its fixed bound"));
|
||||
}
|
||||
}
|
||||
self.check_identity()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct PublicationEvents {
|
||||
tails: Vec<Tail>,
|
||||
bucket: String,
|
||||
object: String,
|
||||
events: Vec<Value>,
|
||||
per_file: Vec<Vec<Value>>,
|
||||
event_count: usize,
|
||||
event_bytes: usize,
|
||||
bytes_read: u64,
|
||||
full_decodes: u64,
|
||||
failed: bool,
|
||||
}
|
||||
|
||||
impl PublicationEvents {
|
||||
pub(super) fn new(paths: Vec<PathBuf>, bucket: &str, object: &str) -> io::Result<Self> {
|
||||
if paths.is_empty() || paths.len() > 4 {
|
||||
return Err(invalid("publication capture requires one to four log files"));
|
||||
}
|
||||
let tails: Vec<Tail> = paths.into_iter().map(Tail::open).collect::<io::Result<_>>()?;
|
||||
let mut reader = Self {
|
||||
per_file: vec![Vec::new(); tails.len()],
|
||||
tails,
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
events: Vec::new(),
|
||||
event_bytes: 0,
|
||||
event_count: 0,
|
||||
bytes_read: 0,
|
||||
full_decodes: 0,
|
||||
failed: false,
|
||||
};
|
||||
// Prime before holding remote DELETE or starting a movement attempt.
|
||||
// Retain earlier matching events too: the caller still applies its
|
||||
// original timestamp boundary, and evidence keeps the complete history.
|
||||
reader.poll()?;
|
||||
Ok(reader)
|
||||
}
|
||||
|
||||
pub(super) fn poll(&mut self) -> io::Result<&[Value]> {
|
||||
if self.failed {
|
||||
return Err(invalid("publication log reader was already invalidated"));
|
||||
}
|
||||
let result = (|| {
|
||||
for (tail, history) in self.tails.iter_mut().zip(&mut self.per_file) {
|
||||
tail.poll(&mut self.bytes_read, |line| {
|
||||
if let Some(event) = select(line, &self.bucket, &self.object, &mut self.full_decodes)? {
|
||||
let bytes = serde_json::to_vec(&event).map_err(io::Error::other)?.len();
|
||||
if self.event_count >= MAX_EVENTS || bytes > MAX_EVENT_BYTES - self.event_bytes {
|
||||
return Err(invalid("publication event history exceeds its fixed bound"));
|
||||
}
|
||||
self.event_bytes += bytes;
|
||||
self.event_count += 1;
|
||||
history.push(event);
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
// Preserve the old file-then-row order, even when a later poll appends
|
||||
// to an earlier file. Only bounded selected history is copied here.
|
||||
self.events = self.per_file.iter().flatten().cloned().collect();
|
||||
if let Err(error) = result {
|
||||
self.failed = true;
|
||||
return Err(error);
|
||||
}
|
||||
Ok(&self.events)
|
||||
}
|
||||
|
||||
/// Diagnostics only: after a read error this history can be incomplete.
|
||||
pub(super) fn cached(&self) -> &[Value] {
|
||||
&self.events
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
use std::io::Write;
|
||||
|
||||
fn event(id: &str) -> Value {
|
||||
json!({"timestamp":"2026-10-26T00:00:00Z","event":"object_namespace_lock","op":"data_movement_publication","bucket":"bucket","object":"object","lock_attempt_id":id,"state":"acquiring","extra":{"kept":[1,2]}})
|
||||
}
|
||||
fn append(path: &std::path::Path, raw: &[u8]) {
|
||||
std::fs::OpenOptions::new()
|
||||
.append(true)
|
||||
.open(path)
|
||||
.expect("open append log")
|
||||
.write_all(raw)
|
||||
.expect("append log bytes");
|
||||
}
|
||||
fn legacy(raw: &[u8]) -> Vec<Value> {
|
||||
std::str::from_utf8(raw)
|
||||
.expect("fixture UTF-8")
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
let row: Value = serde_json::from_str(line).ok()?;
|
||||
let fields = row.get("fields").unwrap_or(&row);
|
||||
if fields.get("event").and_then(Value::as_str) == Some("object_namespace_lock")
|
||||
&& fields.get("op").and_then(Value::as_str) == Some("data_movement_publication")
|
||||
&& fields.get("bucket").and_then(Value::as_str) == Some("bucket")
|
||||
&& fields.get("object").and_then(Value::as_str) == Some("object")
|
||||
{
|
||||
let mut result = fields.clone();
|
||||
result["timestamp"] = row["timestamp"].clone();
|
||||
Some(result)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn appended_events_keep_full_fields_without_history_rescans() {
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
let mut raw = (json!({"message":"unrelated trace","payload":"x".repeat(4096)}).to_string() + "\n")
|
||||
.repeat(64)
|
||||
.into_bytes();
|
||||
raw.extend_from_slice(
|
||||
format!("{}\n", json!({"event":"object_namespace_lock","op":"get_object","bucket":"bucket","object":"object","spans":"x".repeat(2401832)})).as_bytes(),
|
||||
);
|
||||
raw.extend_from_slice(format!("{}\n", event("first")).as_bytes());
|
||||
std::fs::write(file.path(), &raw).expect("seed log");
|
||||
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("prime log");
|
||||
assert_eq!(reader.cached(), legacy(&raw));
|
||||
assert_eq!(reader.full_decodes, 1);
|
||||
let bytes = reader.bytes_read;
|
||||
assert_eq!(reader.poll().expect("unchanged poll"), legacy(&raw));
|
||||
assert_eq!(reader.bytes_read, bytes);
|
||||
let next = format!("{}\n", event("second"));
|
||||
append(file.path(), next.as_bytes());
|
||||
raw.extend_from_slice(next.as_bytes());
|
||||
assert_eq!(reader.poll().expect("new rows"), legacy(&raw));
|
||||
assert_eq!(reader.bytes_read, u64::try_from(raw.len()).expect("fixture length"));
|
||||
assert_eq!(reader.full_decodes, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn later_appends_preserve_the_old_file_then_row_order() {
|
||||
let first = tempfile::NamedTempFile::new().expect("first log");
|
||||
let second = tempfile::NamedTempFile::new().expect("second log");
|
||||
std::fs::write(first.path(), format!("{}\n", event("a1"))).expect("seed first");
|
||||
std::fs::write(second.path(), format!("{}\n", event("b1"))).expect("seed second");
|
||||
let mut reader = PublicationEvents::new(vec![first.path().to_owned(), second.path().to_owned()], "bucket", "object")
|
||||
.expect("prime both");
|
||||
append(first.path(), format!("{}\n", event("a2")).as_bytes());
|
||||
let mut expected = legacy(&std::fs::read(first.path()).expect("first bytes"));
|
||||
expected.extend(legacy(&std::fs::read(second.path()).expect("second bytes")));
|
||||
assert_eq!(reader.poll().expect("append first after second was read"), expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn root_nested_escaped_and_duplicate_fields_match_the_old_reader() {
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
let root = event("root");
|
||||
let mut nested = json!({"timestamp":root["timestamp"],"fields":root});
|
||||
nested["fields"]["extra"] = json!({"unicode":"é"});
|
||||
let null = json!({"fields":null,"event":"object_namespace_lock","op":"data_movement_publication","bucket":"bucket","object":"object"});
|
||||
let escaped = root
|
||||
.to_string()
|
||||
.replace("object_namespace_lock", "\\u006fbject_namespace_lock");
|
||||
let duplicate = root.to_string().replacen("{", "{\"event\":\"earlier-value\",", 1);
|
||||
let raw = format!("{root}\n{nested}\n{null}\n{escaped}\n{duplicate}\nnot json\n{{\"fields\":[]}}\n");
|
||||
std::fs::write(file.path(), &raw).expect("mixed log");
|
||||
let reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("probe and fallback");
|
||||
assert_eq!(reader.cached(), legacy(raw.as_bytes()));
|
||||
assert_eq!(reader.cached().len(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incomplete_rows_and_split_utf8_wait_for_the_newline() {
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
let mut row = event("partial");
|
||||
row["extra"] = json!("é");
|
||||
let raw = format!("{row}\n").into_bytes();
|
||||
let cut = raw.iter().position(|b| *b == 0xc3).expect("UTF-8 prefix") + 1;
|
||||
std::fs::write(file.path(), &raw[..cut]).expect("partial code point");
|
||||
let mut reader =
|
||||
PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("incomplete row is pending");
|
||||
assert!(reader.cached().is_empty());
|
||||
append(file.path(), &raw[cut..raw.len() - 1]);
|
||||
assert!(reader.poll().expect("complete JSON without delimiter").is_empty());
|
||||
append(file.path(), b"\n");
|
||||
assert_eq!(reader.poll().expect("published row"), legacy(&raw));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncation_is_sticky_even_after_the_file_grows_again() {
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
std::fs::write(file.path(), format!("{}\n", event("first"))).expect("seed log");
|
||||
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("prime log");
|
||||
file.as_file().set_len(0).expect("truncate");
|
||||
assert!(reader.poll().is_err());
|
||||
append(file.path(), format!("{}\n", event("longer replacement")).as_bytes());
|
||||
assert!(reader.poll().is_err());
|
||||
assert_eq!(reader.cached().len(), 1);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn a_replaced_path_does_not_switch_to_the_new_log() {
|
||||
let dir = tempfile::tempdir().expect("log dir");
|
||||
let path = dir.path().join("node.log");
|
||||
std::fs::write(&path, b"").expect("log file");
|
||||
let mut reader = PublicationEvents::new(vec![path.clone()], "bucket", "object").expect("open log");
|
||||
std::fs::rename(&path, dir.path().join("retired.log")).expect("replace log");
|
||||
std::fs::write(&path, format!("{}\n", event("other-generation"))).expect("replacement");
|
||||
assert!(reader.poll().is_err());
|
||||
assert!(reader.cached().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_utf8_and_oversized_or_excessive_records_fail_closed() {
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("empty log");
|
||||
append(file.path(), b"\xff\n");
|
||||
assert!(reader.poll().is_err());
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("empty log");
|
||||
append(file.path(), &vec![b'x'; MAX_LINE_BYTES + 1]);
|
||||
assert!(reader.poll().is_err());
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("empty log");
|
||||
for _ in 0..=MAX_EVENTS {
|
||||
append(file.path(), format!("{}\n", event("attempt")).as_bytes());
|
||||
}
|
||||
assert!(reader.poll().is_err());
|
||||
assert_eq!(reader.cached().len(), MAX_EVENTS);
|
||||
assert!(reader.poll().is_err());
|
||||
let file = tempfile::NamedTempFile::new().expect("log file");
|
||||
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("empty log");
|
||||
let mut large = event("large");
|
||||
large["extra"] = json!("x".repeat(600 * 1024));
|
||||
for _ in 0..2 {
|
||||
append(file.path(), format!("{large}\n").as_bytes());
|
||||
}
|
||||
assert!(reader.poll().is_err());
|
||||
assert_eq!(reader.cached().len(), 1);
|
||||
assert!(reader.poll().is_err());
|
||||
}
|
||||
}
|
||||
@@ -21,7 +21,6 @@ use super::harness::{
|
||||
rebalance_status_json, sha256_hex, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
|
||||
wait_for_rebalance_complete,
|
||||
};
|
||||
use super::publication_log::PublicationEvents;
|
||||
use super::scanner_heal_topology_fixture::{
|
||||
MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL, STAGE_AFTER_RESPONSIBILITY_PERSISTED, STAGE_BEFORE_SOURCE_CLEANUP,
|
||||
TopologyRun, assert_bucket_durability_strict, assert_process_durability_strict, require_test_hooks,
|
||||
@@ -175,6 +174,29 @@ async fn transition_version(
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_events(run: &TopologyRun, bucket: &str) -> TestResult<Vec<Value>> {
|
||||
let mut events = Vec::new();
|
||||
for path in run.node_log_paths() {
|
||||
let log = std::fs::read_to_string(path)?;
|
||||
for line in log.lines() {
|
||||
let Ok(row) = serde_json::from_str::<Value>(line) else {
|
||||
continue;
|
||||
};
|
||||
let fields = row.get("fields").unwrap_or(&row);
|
||||
if fields.get("event").and_then(Value::as_str) == Some("object_namespace_lock")
|
||||
&& fields.get("op").and_then(Value::as_str) == Some("data_movement_publication")
|
||||
&& fields.get("bucket").and_then(Value::as_str) == Some(bucket)
|
||||
&& fields.get("object").and_then(Value::as_str) == Some(KEY)
|
||||
{
|
||||
let mut event = fields.clone();
|
||||
event["timestamp"] = row["timestamp"].clone();
|
||||
events.push(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(events)
|
||||
}
|
||||
|
||||
fn event_time(event: &Value) -> TestResult<OffsetDateTime> {
|
||||
Ok(OffsetDateTime::parse(
|
||||
event["timestamp"].as_str().ok_or("lock event omitted its timestamp")?,
|
||||
@@ -182,10 +204,10 @@ fn event_time(event: &Value) -> TestResult<OffsetDateTime> {
|
||||
)?)
|
||||
}
|
||||
|
||||
async fn pending_publication(events: &mut PublicationEvents, started: OffsetDateTime) -> TestResult<Value> {
|
||||
async fn pending_publication(run: &TopologyRun, bucket: &str, started: OffsetDateTime) -> TestResult<Value> {
|
||||
loop {
|
||||
let observed = events.poll()?;
|
||||
for event in observed {
|
||||
let events = lock_events(run, bucket)?;
|
||||
for event in &events {
|
||||
if event_time(event)? < started {
|
||||
continue;
|
||||
}
|
||||
@@ -194,7 +216,7 @@ async fn pending_publication(events: &mut PublicationEvents, started: OffsetDate
|
||||
"mover acquired its publication lock while remote DELETE was held"
|
||||
);
|
||||
if event["state"] == "acquiring"
|
||||
&& !observed
|
||||
&& !events
|
||||
.iter()
|
||||
.any(|later| later["lock_attempt_id"] == event["lock_attempt_id"] && later["state"] != "acquiring")
|
||||
{
|
||||
@@ -206,14 +228,14 @@ async fn pending_publication(events: &mut PublicationEvents, started: OffsetDate
|
||||
}
|
||||
}
|
||||
|
||||
async fn acquired_publication(events: &mut PublicationEvents, pending: &Value) -> TestResult<Value> {
|
||||
async fn acquired_publication(run: &TopologyRun, bucket: &str, pending: &Value) -> TestResult<Value> {
|
||||
loop {
|
||||
for event in events.poll()? {
|
||||
for event in lock_events(run, bucket)? {
|
||||
if event["lock_attempt_id"] != pending["lock_attempt_id"] {
|
||||
continue;
|
||||
}
|
||||
match event["state"].as_str() {
|
||||
Some("acquired") => return Ok(event.clone()),
|
||||
Some("acquired") => return Ok(event),
|
||||
Some("failed" | "cancelled") => return Err(format!("held publication attempt did not acquire: {event}").into()),
|
||||
_ => {}
|
||||
}
|
||||
@@ -381,7 +403,6 @@ async fn tier_delete_during_movement(decommission: bool) -> TestResult {
|
||||
// started a worker. Install a separate release nonce before holding DELETE.
|
||||
let cleanup_barrier = run.movement_barrier.next_phase();
|
||||
cleanup_barrier.arm_movement(operation, &bucket, KEY, STAGE_BEFORE_SOURCE_CLEANUP)?;
|
||||
let mut publication_events = PublicationEvents::new(run.node_log_paths(), &bucket, KEY)?;
|
||||
let mut delete_gate = target.hold_delete_object(&remote_bucket, &old_remote.key);
|
||||
// The public DELETE persists a free owner and enqueues its remote cleanup.
|
||||
let client = run.dist.client(0)?;
|
||||
@@ -449,11 +470,11 @@ async fn tier_delete_during_movement(decommission: bool) -> TestResult {
|
||||
})?;
|
||||
drop(decommission_start);
|
||||
let movement_acknowledged_at = OffsetDateTime::now_utc();
|
||||
let pending = timeout_at(overlap_deadline, pending_publication(&mut publication_events, movement_started))
|
||||
let pending = timeout_at(overlap_deadline, pending_publication(&run, &bucket, movement_started))
|
||||
.await
|
||||
.map_err(|error| format!(
|
||||
"G-TIER {operation} publication pending timed out within DELETE overlap: {error}; startup={startup}; lock_events={:?}; cleanup_barrier={:?}",
|
||||
publication_events.cached(), cleanup_barrier.reached_observations()
|
||||
lock_events(&run, &bucket), cleanup_barrier.reached_observations()
|
||||
))?
|
||||
.map_err(|error| format!("G-TIER {operation} publication pending failed within DELETE overlap: {error}"))?;
|
||||
// Positive gate entry, exact physical ownership and the pending lock attempt
|
||||
@@ -463,11 +484,11 @@ async fn tier_delete_during_movement(decommission: bool) -> TestResult {
|
||||
let released_at = OffsetDateTime::now_utc();
|
||||
let delete_overlap_elapsed_ms = overlap_started.elapsed().as_millis();
|
||||
drop(delete_gate);
|
||||
let acquired = timeout(DELETE_OVERLAP_TIMEOUT, acquired_publication(&mut publication_events, &pending))
|
||||
let acquired = timeout(DELETE_OVERLAP_TIMEOUT, acquired_publication(&run, &bucket, &pending))
|
||||
.await
|
||||
.map_err(|error| format!(
|
||||
"G-TIER {operation} publication acquired after DELETE release timed out: {error}; pending={pending}; lock_events={:?}; cleanup_barrier={:?}",
|
||||
publication_events.cached(), cleanup_barrier.reached_observations()
|
||||
lock_events(&run, &bucket), cleanup_barrier.reached_observations()
|
||||
))?
|
||||
.map_err(|error| format!("G-TIER {operation} publication acquired after DELETE release failed: {error}"))?;
|
||||
assert!(event_time(&pending)? >= movement_started && event_time(&pending)? < released_at);
|
||||
@@ -475,7 +496,6 @@ async fn tier_delete_during_movement(decommission: bool) -> TestResult {
|
||||
event_time(&acquired)? >= released_at,
|
||||
"the same publication attempt must acquire after DELETE is released"
|
||||
);
|
||||
drop(publication_events);
|
||||
let reached = cleanup_barrier
|
||||
.wait_reached_during(
|
||||
&run.dist.cluster,
|
||||
|
||||
@@ -1129,14 +1129,12 @@ mod tests {
|
||||
/// and the marker plus the cursor tag.
|
||||
///
|
||||
/// Bug Reference: backlog#1047
|
||||
/// Earlier releases appended a `[rustfs_cache:...]` cursor tag to the last
|
||||
/// returned key inside the V2 continuation token. The orchestration layer
|
||||
/// passed that raw string to `forward_past`, so any key whose byte after the
|
||||
/// shared stem sorts below `[` (0x5B) — `.`, `/`, `-`, digits, uppercase —
|
||||
/// was silently dropped on the next page: with keys `a`, `a.txt`, `zz` and
|
||||
/// max_keys=1, page 2 returned `zz` and `a.txt` was never listed. The token
|
||||
/// is now base64 of the plain key, and this test keeps guarding the
|
||||
/// continuation boundary.
|
||||
/// The V2 continuation token appends a `[rustfs_cache:...]` cursor tag to the
|
||||
/// last returned key. The orchestration layer passed that raw string to
|
||||
/// `forward_past`, so any key whose byte after the shared stem sorts below
|
||||
/// `[` (0x5B) — `.`, `/`, `-`, digits, uppercase — was silently dropped on
|
||||
/// the next page: with keys `a`, `a.txt`, `zz` and max_keys=1, page 2
|
||||
/// returned `zz` and `a.txt` was never listed.
|
||||
#[tokio::test]
|
||||
async fn test_list_objects_v2_continuation_keeps_keys_after_marker_stem() {
|
||||
init_logging();
|
||||
|
||||
@@ -19,19 +19,6 @@ use http::{Method, StatusCode};
|
||||
use tokio::time::{Duration, sleep, timeout};
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// A successful reporting response can still have unknown scanner usage.
|
||||
fn reported_quota_usage(response: &serde_json::Value) -> std::io::Result<Option<u64>> {
|
||||
let value = response
|
||||
.get("current_usage")
|
||||
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "quota response is missing current_usage"))?;
|
||||
if value.is_null() {
|
||||
return Ok(None);
|
||||
}
|
||||
value.as_u64().map(Some).ok_or_else(|| {
|
||||
std::io::Error::new(std::io::ErrorKind::InvalidData, "quota current_usage is not an unsigned integer or null")
|
||||
})
|
||||
}
|
||||
|
||||
/// Test environment setup for quota tests
|
||||
pub struct QuotaTestEnv {
|
||||
pub env: RustFSTestEnvironment,
|
||||
@@ -100,69 +87,35 @@ impl QuotaTestEnv {
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_usage_response(
|
||||
&self,
|
||||
method: Method,
|
||||
path: &str,
|
||||
body: Option<String>,
|
||||
expected_usage: u64,
|
||||
) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let convergence = async {
|
||||
loop {
|
||||
let (status, response) = admin_request(
|
||||
&self.env.url,
|
||||
method.clone(),
|
||||
path,
|
||||
body.clone(),
|
||||
&self.env.access_key,
|
||||
&self.env.secret_key,
|
||||
)
|
||||
.await?;
|
||||
if status.is_success() {
|
||||
let response: serde_json::Value = serde_json::from_str(&response)?;
|
||||
if let Some(usage) = reported_quota_usage(&response)?
|
||||
&& usage == expected_usage
|
||||
{
|
||||
return Ok::<_, Box<dyn std::error::Error + Send + Sync>>(response);
|
||||
}
|
||||
} else if status != StatusCode::SERVICE_UNAVAILABLE {
|
||||
return Err(format!("quota usage request failed for {path}: {status} {response}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
};
|
||||
match timeout(Duration::from_secs(30), convergence).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(format!("quota usage did not converge to {expected_usage} for {path} within 30 seconds").into()),
|
||||
pub async fn get_bucket_quota_stats(&self) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!("{}/rustfs/admin/v3/quota-stats/{}", self.env.url, self.bucket_name);
|
||||
let response = awscurl_get(&url, &self.env.access_key, &self.env.secret_key).await?;
|
||||
|
||||
if response.contains("error") {
|
||||
Err(format!("Failed to get quota stats: {}", response).into())
|
||||
} else {
|
||||
Ok(serde_json::from_str(&response)?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_bucket_quota_stats(
|
||||
&self,
|
||||
bucket: &str,
|
||||
expected_usage: u64,
|
||||
) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.wait_for_usage_response(Method::GET, &format!("/rustfs/admin/v3/quota-stats/{bucket}"), None, expected_usage)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn wait_for_bucket_quota_check(
|
||||
pub async fn check_bucket_quota(
|
||||
&self,
|
||||
operation_type: &str,
|
||||
operation_size: u64,
|
||||
expected_usage: u64,
|
||||
) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!("{}/rustfs/admin/v3/quota-check/{}", self.env.url, self.bucket_name);
|
||||
let check_request = serde_json::json!({
|
||||
"operation_type": operation_type,
|
||||
"operation_size": operation_size
|
||||
});
|
||||
self.wait_for_usage_response(
|
||||
Method::POST,
|
||||
&format!("/rustfs/admin/v3/quota-check/{}", self.bucket_name),
|
||||
Some(check_request.to_string()),
|
||||
expected_usage,
|
||||
)
|
||||
.await
|
||||
|
||||
let response = awscurl_post(&url, &check_request.to_string(), &self.env.access_key, &self.env.secret_key).await?;
|
||||
|
||||
if response.contains("error") {
|
||||
Err(format!("Failed to check quota: {}", response).into())
|
||||
} else {
|
||||
Ok(serde_json::from_str(&response)?)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn upload_object(&self, key: &str, size_bytes: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
@@ -192,9 +145,30 @@ impl QuotaTestEnv {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_bucket_usage(&self) -> Result<u64, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let stats = self.get_bucket_quota_stats().await?;
|
||||
stats
|
||||
.get("current_usage")
|
||||
.and_then(|value| value.as_u64())
|
||||
.ok_or_else(|| format!("bucket quota usage is not authoritative for {}", self.bucket_name).into())
|
||||
}
|
||||
|
||||
async fn wait_for_bucket_usage(&self, expected: u64) -> Result<u64, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let stats = self.wait_for_bucket_quota_stats(&self.bucket_name, expected).await?;
|
||||
reported_quota_usage(&stats)?.ok_or_else(|| "converged quota usage is missing".into())
|
||||
let convergence = async {
|
||||
loop {
|
||||
let stats = self.get_bucket_quota_stats().await?;
|
||||
if let Some(usage) = stats.get("current_usage").and_then(|value| value.as_u64())
|
||||
&& usage == expected
|
||||
{
|
||||
return Ok::<u64, Box<dyn std::error::Error + Send + Sync>>(usage);
|
||||
}
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
};
|
||||
match timeout(Duration::from_secs(30), convergence).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(format!("bucket usage did not converge to {expected} bytes within 30 seconds").into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn set_bucket_quota_for(
|
||||
@@ -202,6 +176,8 @@ impl QuotaTestEnv {
|
||||
bucket: &str,
|
||||
quota_bytes: u64,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.wait_for_quota_usage_for(bucket).await?;
|
||||
|
||||
let quota_path = format!("/rustfs/admin/v3/quota/{bucket}");
|
||||
let quota_config = serde_json::json!({
|
||||
"quota": quota_bytes,
|
||||
@@ -235,6 +211,48 @@ impl QuotaTestEnv {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn wait_for_quota_usage_for(&self, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let stats_path = format!("/rustfs/admin/v3/quota-stats/{bucket}");
|
||||
let readiness = async {
|
||||
loop {
|
||||
let (status, response) =
|
||||
admin_request(&self.env.url, Method::GET, &stats_path, None, &self.env.access_key, &self.env.secret_key)
|
||||
.await?;
|
||||
if status.is_success() {
|
||||
let stats: serde_json::Value = serde_json::from_str(&response)?;
|
||||
if stats.get("current_usage").and_then(|value| value.as_u64()).is_some() {
|
||||
return Ok::<(), Box<dyn std::error::Error + Send + Sync>>(());
|
||||
}
|
||||
} else if status != StatusCode::SERVICE_UNAVAILABLE {
|
||||
return Err(format!("quota usage readiness failed for {bucket}: {status} {response}").into());
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
};
|
||||
match timeout(Duration::from_secs(30), readiness).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(format!("quota usage did not become authoritative for {bucket} within 30 seconds").into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get bucket quota statistics for specific bucket
|
||||
pub async fn get_bucket_quota_stats_for(
|
||||
&self,
|
||||
bucket: &str,
|
||||
) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
|
||||
debug!("Getting quota stats for bucket: {}", bucket);
|
||||
|
||||
let url = format!("{}/rustfs/admin/v3/quota-stats/{}", self.env.url, bucket);
|
||||
let response = awscurl_get(&url, &self.env.access_key, &self.env.secret_key).await?;
|
||||
|
||||
if response.contains("error") {
|
||||
Err(format!("Failed to get quota stats: {}", response).into())
|
||||
} else {
|
||||
let stats: serde_json::Value = serde_json::from_str(&response)?;
|
||||
Ok(stats)
|
||||
}
|
||||
}
|
||||
|
||||
/// Upload an object to specific bucket
|
||||
pub async fn upload_object_to_bucket(
|
||||
&self,
|
||||
@@ -259,132 +277,6 @@ impl QuotaTestEnv {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod usage_response_tests {
|
||||
use super::*;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
#[test]
|
||||
fn unknown_usage_is_not_zero() {
|
||||
assert_eq!(
|
||||
reported_quota_usage(&serde_json::json!({"current_usage": null})).expect("valid unknown usage"),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
reported_quota_usage(&serde_json::json!({"current_usage": 0})).expect("known zero usage"),
|
||||
Some(0)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_or_invalid_usage_is_not_readiness() {
|
||||
for response in [
|
||||
serde_json::json!({}),
|
||||
serde_json::json!({"current_usage": -1}),
|
||||
serde_json::json!({"current_usage": "0"}),
|
||||
serde_json::json!({"current_usage": 1.5}),
|
||||
serde_json::json!({"current_usage": false}),
|
||||
] {
|
||||
let error = reported_quota_usage(&response).expect_err("malformed usage must fail");
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
|
||||
}
|
||||
}
|
||||
|
||||
async fn mock_environment(
|
||||
responses: Vec<(StatusCode, &'static str)>,
|
||||
) -> Result<(QuotaTestEnv, JoinSet<()>), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let listener = TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)).await?;
|
||||
let address = listener.local_addr()?;
|
||||
let mut tasks = JoinSet::new();
|
||||
tasks.spawn(async move {
|
||||
for (status, body) in responses {
|
||||
let (mut stream, _) = listener.accept().await.expect("accept quota reporting request");
|
||||
let mut request = Vec::new();
|
||||
while !request.windows(4).any(|part| part == b"\r\n\r\n") {
|
||||
let mut buffer = [0_u8; 1024];
|
||||
let read = stream.read(&mut buffer).await.expect("read reporting request");
|
||||
assert!(read > 0 && request.len() + read <= 8192, "bounded HTTP request headers");
|
||||
request.extend_from_slice(&buffer[..read]);
|
||||
}
|
||||
let response = format!(
|
||||
"HTTP/1.1 {} Test\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
|
||||
status.as_u16(),
|
||||
body.len(),
|
||||
body
|
||||
);
|
||||
stream
|
||||
.write_all(response.as_bytes())
|
||||
.await
|
||||
.expect("write quota reporting response");
|
||||
stream.shutdown().await.expect("close reporting response");
|
||||
}
|
||||
});
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.url = format!("http://{address}");
|
||||
let client = env.create_s3_client();
|
||||
Ok((
|
||||
QuotaTestEnv {
|
||||
env,
|
||||
client,
|
||||
bucket_name: "quota-reporting-fixture".to_owned(),
|
||||
},
|
||||
tasks,
|
||||
))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reporting_waits_for_known_matching_usage_without_selecting_allowed()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let (env, mut tasks) = mock_environment(vec![
|
||||
(StatusCode::SERVICE_UNAVAILABLE, "unavailable"),
|
||||
(StatusCode::OK, r#"{"current_usage":null}"#),
|
||||
(StatusCode::OK, r#"{"current_usage":1}"#),
|
||||
(StatusCode::OK, r#"{"current_usage":0,"allowed":false}"#),
|
||||
])
|
||||
.await?;
|
||||
let response = env.wait_for_usage_response(Method::GET, "/reporting", None, 0).await?;
|
||||
assert_eq!(response["current_usage"], 0);
|
||||
assert_eq!(response["allowed"], false, "readiness must not select a desired admission outcome");
|
||||
timeout(Duration::from_secs(1), tasks.join_next())
|
||||
.await?
|
||||
.expect("reporting mock task")
|
||||
.expect("all responses consumed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reporting_rejects_malformed_success_instead_of_retrying() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let (env, mut tasks) = mock_environment(vec![(StatusCode::OK, r#"{"current_usage":"unknown"}"#)]).await?;
|
||||
let error = env
|
||||
.wait_for_usage_response(Method::GET, "/reporting", None, 0)
|
||||
.await
|
||||
.expect_err("malformed usage must fail");
|
||||
assert!(error.to_string().contains("not an unsigned integer or null"));
|
||||
timeout(Duration::from_secs(1), tasks.join_next())
|
||||
.await?
|
||||
.expect("reporting mock task")
|
||||
.expect("response consumed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reporting_does_not_retry_authorization_failure() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let (env, mut tasks) = mock_environment(vec![(StatusCode::FORBIDDEN, "denied")]).await?;
|
||||
let error = env
|
||||
.wait_for_usage_response(Method::GET, "/reporting", None, 0)
|
||||
.await
|
||||
.expect_err("authorization error must fail");
|
||||
assert!(error.to_string().contains("403"));
|
||||
timeout(Duration::from_secs(1), tasks.join_next())
|
||||
.await?
|
||||
.expect("reporting mock task")
|
||||
.expect("response consumed");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod integration_tests {
|
||||
use super::*;
|
||||
@@ -597,7 +489,7 @@ mod integration_tests {
|
||||
env.upload_object("file2.txt", 256 * 1024).await?;
|
||||
|
||||
// Check usage
|
||||
let usage = env.wait_for_bucket_usage((512 + 256) * 1024).await?;
|
||||
let usage = env.get_bucket_usage().await?;
|
||||
assert_eq!(usage, (512 + 256) * 1024);
|
||||
|
||||
// Delete a file
|
||||
@@ -632,7 +524,7 @@ mod integration_tests {
|
||||
env.upload_object("file2.txt", 512 * 1024).await?;
|
||||
|
||||
// Get detailed quota statistics
|
||||
let stats = env.wait_for_bucket_quota_stats(&env.bucket_name, (1024 + 512) * 1024).await?;
|
||||
let stats = env.get_bucket_quota_stats().await?;
|
||||
|
||||
assert_eq!(stats.get("bucket").unwrap().as_str().unwrap(), env.bucket_name);
|
||||
assert_eq!(stats.get("quota_limit").unwrap().as_u64().unwrap(), 2 * 1024 * 1024);
|
||||
@@ -661,22 +553,22 @@ mod integration_tests {
|
||||
env.upload_object("existing_file.txt", 512 * 1024).await?;
|
||||
|
||||
// Check if we can upload another 512KB (should succeed, exactly fill quota)
|
||||
let check_result = env.wait_for_bucket_quota_check("PUT", 512 * 1024, 512 * 1024).await?;
|
||||
let check_result = env.check_bucket_quota("PUT", 512 * 1024).await?;
|
||||
assert!(check_result.get("allowed").unwrap().as_bool().unwrap());
|
||||
assert_eq!(check_result.get("remaining_quota").unwrap().as_u64().unwrap(), 0);
|
||||
|
||||
// Note: we haven't actually uploaded the second file yet, so current_usage is still 512KB
|
||||
// Check if we can upload 1KB (should succeed - we haven't used the full quota yet)
|
||||
let check_result = env.wait_for_bucket_quota_check("PUT", 1024, 512 * 1024).await?;
|
||||
let check_result = env.check_bucket_quota("PUT", 1024).await?;
|
||||
assert!(check_result.get("allowed").unwrap().as_bool().unwrap());
|
||||
assert_eq!(check_result.get("remaining_quota").unwrap().as_u64().unwrap(), 523264); // 511 * 1024
|
||||
|
||||
// Check if we can upload 600KB (should fail - would exceed quota)
|
||||
let check_result = env.wait_for_bucket_quota_check("PUT", 600 * 1024, 512 * 1024).await?;
|
||||
let check_result = env.check_bucket_quota("PUT", 600 * 1024).await?;
|
||||
assert!(!check_result.get("allowed").unwrap().as_bool().unwrap());
|
||||
|
||||
// Check delete operation (should always be allowed)
|
||||
let check_result = env.wait_for_bucket_quota_check("DELETE", 512 * 1024, 512 * 1024).await?;
|
||||
let check_result = env.check_bucket_quota("DELETE", 512 * 1024).await?;
|
||||
assert!(check_result.get("allowed").unwrap().as_bool().unwrap());
|
||||
|
||||
env.cleanup_bucket().await?;
|
||||
@@ -708,8 +600,8 @@ mod integration_tests {
|
||||
env.upload_object_to_bucket(&bucket2, "another_file.txt", 512 * 1024).await?;
|
||||
|
||||
// Verify statistics are independent
|
||||
let stats1 = env.wait_for_bucket_quota_stats(&bucket1, 1024 * 1024).await?;
|
||||
let stats2 = env.wait_for_bucket_quota_stats(&bucket2, (1024 + 512) * 1024).await?;
|
||||
let stats1 = env.get_bucket_quota_stats_for(&bucket1).await?;
|
||||
let stats2 = env.get_bucket_quota_stats_for(&bucket2).await?;
|
||||
|
||||
assert_eq!(stats1.get("current_usage").unwrap().as_u64().unwrap(), 1024 * 1024);
|
||||
assert_eq!(stats2.get("current_usage").unwrap().as_u64().unwrap(), (1024 + 512) * 1024);
|
||||
@@ -770,6 +662,7 @@ mod integration_tests {
|
||||
let env = QuotaTestEnv::new().await?;
|
||||
|
||||
env.create_bucket().await?;
|
||||
env.wait_for_quota_usage_for(&env.bucket_name).await?;
|
||||
|
||||
// Test 1: GET quota for bucket without quota config
|
||||
let url = format!("{}/rustfs/admin/v3/quota/{}", env.env.url, env.bucket_name);
|
||||
|
||||
@@ -27,8 +27,8 @@ use hyper_util::{
|
||||
};
|
||||
use prost::Message as _;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
BatchGenerallyLockRequest, DeleteRequest, GenerallyLockRequest, PreparePartTransactionResponse, ReadVersionRequest,
|
||||
RecoverMultipartSourceRequest, RenameDataResponse, SettlePartTransactionRequest, SettlePartTransactionResponse,
|
||||
BatchGenerallyLockRequest, DeleteRequest, GenerallyLockRequest, ReadVersionRequest, RecoverMultipartSourceRequest,
|
||||
RenameDataResponse,
|
||||
};
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
@@ -56,35 +56,6 @@ enum Disposition {
|
||||
Discard,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum MutationReply {
|
||||
None,
|
||||
Rename,
|
||||
PreparePart,
|
||||
SettlePart,
|
||||
}
|
||||
|
||||
impl MutationReply {
|
||||
fn outcome(self, bytes: &[u8]) -> io::Result<Option<(bool, bool)>> {
|
||||
let result = match self {
|
||||
Self::None => return Ok(None),
|
||||
Self::Rename => {
|
||||
let reply = RenameDataResponse::decode(unary_payload(bytes)?).map_err(io::Error::other)?;
|
||||
(reply.success, reply.error.is_some())
|
||||
}
|
||||
Self::PreparePart => {
|
||||
let reply = PreparePartTransactionResponse::decode(unary_payload(bytes)?).map_err(io::Error::other)?;
|
||||
(reply.success, reply.error.is_some())
|
||||
}
|
||||
Self::SettlePart => {
|
||||
let reply = SettlePartTransactionResponse::decode(unary_payload(bytes)?).map_err(io::Error::other)?;
|
||||
(reply.success, reply.error.is_some())
|
||||
}
|
||||
};
|
||||
Ok(Some(result))
|
||||
}
|
||||
}
|
||||
|
||||
struct PauseState {
|
||||
methods: Vec<&'static str>,
|
||||
scope: Option<Vec<u8>>,
|
||||
@@ -93,9 +64,6 @@ struct PauseState {
|
||||
limit: usize,
|
||||
retain_after_disconnect: bool,
|
||||
refused_mutations: AtomicUsize,
|
||||
successful_mutation_replies: AtomicUsize,
|
||||
settle_commits: AtomicUsize,
|
||||
settle_rollbacks: AtomicUsize,
|
||||
responses: AtomicUsize,
|
||||
matched: AtomicUsize,
|
||||
blocked: AtomicUsize,
|
||||
@@ -154,17 +122,6 @@ impl RpcPause {
|
||||
self.state.refused_mutations.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn successful_mutation_replies(&self) -> usize {
|
||||
self.state.successful_mutation_replies.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
pub(crate) fn retained_settle_actions(&self) -> (usize, usize) {
|
||||
(
|
||||
self.state.settle_commits.load(Ordering::SeqCst),
|
||||
self.state.settle_rollbacks.load(Ordering::SeqCst),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn unsupported_source_inspection_routes(&self) -> HashSet<String> {
|
||||
self.state
|
||||
.unsupported_inspection_routes
|
||||
@@ -437,9 +394,6 @@ impl RpcFaultProxy {
|
||||
limit,
|
||||
retain_after_disconnect,
|
||||
refused_mutations: AtomicUsize::new(0),
|
||||
successful_mutation_replies: AtomicUsize::new(0),
|
||||
settle_commits: AtomicUsize::new(0),
|
||||
settle_rollbacks: AtomicUsize::new(0),
|
||||
responses: AtomicUsize::new(0),
|
||||
matched: AtomicUsize::new(0),
|
||||
blocked: AtomicUsize::new(0),
|
||||
@@ -555,7 +509,6 @@ async fn forward_request(
|
||||
if let Some(method) = method.filter(|name| pause.methods.contains(name)) {
|
||||
let mut route = None;
|
||||
let mut lock_owners = Vec::new();
|
||||
let mut settle_action = None;
|
||||
let matches = if let Some(scope) = pause.scope.as_deref() {
|
||||
// Inspect bounded control routing while preserving the exact
|
||||
// signed bytes and trailers forwarded to the backend.
|
||||
@@ -569,19 +522,12 @@ async fn forward_request(
|
||||
inspection = Some((pause.clone(), disk));
|
||||
}
|
||||
let mut matches = scope_matches(method, &bytes, scope)?;
|
||||
if matches && method == "SettlePartTransaction" {
|
||||
settle_action = Some(
|
||||
SettlePartTransactionRequest::decode(unary_payload(&bytes)?)
|
||||
.map_err(io::Error::other)?
|
||||
.rollback,
|
||||
);
|
||||
}
|
||||
if matches && (pause.record_lock_owners || pause.lock_owner.is_some()) {
|
||||
lock_owners = scoped_lock_owners(method, &bytes, scope)?;
|
||||
if let Some(owner) = &pause.lock_owner {
|
||||
matches = lock_owners.iter().any(|candidate| candidate == owner);
|
||||
if !matches && pause.record_lock_owners {
|
||||
observed = Some((pause.clone(), None, std::mem::take(&mut lock_owners), MutationReply::None));
|
||||
observed = Some((pause.clone(), None, std::mem::take(&mut lock_owners), false));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -603,14 +549,6 @@ async fn forward_request(
|
||||
true
|
||||
};
|
||||
if matches && pause.matched.fetch_add(1, Ordering::SeqCst) < pause.limit {
|
||||
if let Some(rollback) = settle_action {
|
||||
let counter = if rollback {
|
||||
&pause.settle_rollbacks
|
||||
} else {
|
||||
&pause.settle_commits
|
||||
};
|
||||
counter.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
let mut released = pause.release.subscribe();
|
||||
pause.blocked.fetch_add(1, Ordering::SeqCst);
|
||||
pause.reached.notify_one();
|
||||
@@ -630,14 +568,10 @@ async fn forward_request(
|
||||
pause,
|
||||
route,
|
||||
lock_owners,
|
||||
match method {
|
||||
"RenameData" | "RenameMultipartData" | "RenameDataAtIncarnation" | "RenameDataWithQuotaFence" => {
|
||||
MutationReply::Rename
|
||||
}
|
||||
"PreparePartTransaction" => MutationReply::PreparePart,
|
||||
"SettlePartTransaction" => MutationReply::SettlePart,
|
||||
_ => MutationReply::None,
|
||||
},
|
||||
matches!(
|
||||
method,
|
||||
"RenameData" | "RenameMultipartData" | "RenameDataAtIncarnation" | "RenameDataWithQuotaFence"
|
||||
),
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -670,7 +604,7 @@ async fn forward_request(
|
||||
.expect("inspection response routes")
|
||||
.insert(disk);
|
||||
}
|
||||
if let Some((pause, route, lock_owners, mutation_reply)) = observed {
|
||||
if let Some((pause, route, lock_owners, rename)) = observed {
|
||||
pause.responses.fetch_add(1, Ordering::SeqCst);
|
||||
if response.status().is_success() && response.headers().get("grpc-status").is_none_or(|status| status == "0") {
|
||||
pause.forwarded.fetch_add(1, Ordering::SeqCst);
|
||||
@@ -692,14 +626,10 @@ async fn forward_request(
|
||||
.headers
|
||||
.get("grpc-status")
|
||||
.or_else(|| trailers.as_ref()?.get("grpc-status"));
|
||||
if parts.status.is_success()
|
||||
&& status.is_some_and(|status| status == "0")
|
||||
&& let Some((success, has_error)) = mutation_reply.outcome(&bytes)?
|
||||
{
|
||||
if !success && has_error {
|
||||
if rename && parts.status.is_success() && status.is_none_or(|status| status == "0") {
|
||||
let result = RenameDataResponse::decode(unary_payload(&bytes)?).map_err(io::Error::other)?;
|
||||
if !result.success && result.error.is_some() {
|
||||
pause.refused_mutations.fetch_add(1, Ordering::SeqCst);
|
||||
} else if success && !has_error {
|
||||
pause.successful_mutation_replies.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
let body = Full::new(bytes)
|
||||
@@ -746,10 +676,6 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn backend_with_status(status: Option<&'static str>) -> io::Result<Backend> {
|
||||
backend_with_reply(status, None).await
|
||||
}
|
||||
|
||||
async fn backend_with_reply(status: Option<&'static str>, reply: Option<Bytes>) -> io::Result<Backend> {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||
let address = listener.local_addr()?;
|
||||
let received = Arc::new(AtomicUsize::new(0));
|
||||
@@ -761,11 +687,9 @@ mod tests {
|
||||
accepted = listener.accept() => {
|
||||
let Ok((socket, _)) = accepted else { break };
|
||||
let received = received_task.clone();
|
||||
let reply = reply.clone();
|
||||
children.spawn(async move {
|
||||
let service = service_fn(move |request: Request<Incoming>| {
|
||||
let received = received.clone();
|
||||
let reply = reply.clone();
|
||||
async move {
|
||||
received.fetch_add(1, Ordering::SeqCst);
|
||||
let authority = request.headers().get(http::header::HOST).cloned().or_else(|| {
|
||||
@@ -774,7 +698,6 @@ mod tests {
|
||||
})
|
||||
});
|
||||
let body = request.into_body().collect().await?.to_bytes();
|
||||
let body = reply.unwrap_or(body);
|
||||
let mut trailers = http::HeaderMap::new();
|
||||
trailers.insert("grpc-status", http::HeaderValue::from_static(status.unwrap_or("0")));
|
||||
trailers.insert("x-proof", http::HeaderValue::from_static("unchanged-proof"));
|
||||
@@ -1511,93 +1434,6 @@ mod tests {
|
||||
.await
|
||||
.expect("scoped discard deadline");
|
||||
assert_late_packet_ownership().await;
|
||||
assert_late_part_reply_refusal().await;
|
||||
}
|
||||
|
||||
async fn assert_late_part_reply_refusal() {
|
||||
use rustfs_protos::proto_gen::node_service::{Error, PreparePartTransactionRequest};
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
// method, success, error, gRPC status, rollback, expected successes/refusals
|
||||
for (method, success, has_error, status, rollback, expected) in [
|
||||
("PreparePartTransaction", false, true, None, false, (0, 1)),
|
||||
("PreparePartTransaction", true, false, None, false, (1, 0)),
|
||||
("PreparePartTransaction", false, false, None, false, (0, 0)),
|
||||
("SettlePartTransaction", true, false, None, false, (1, 0)),
|
||||
("SettlePartTransaction", false, true, None, true, (0, 1)),
|
||||
("SettlePartTransaction", true, true, None, false, (0, 0)),
|
||||
("SettlePartTransaction", false, true, Some("16"), false, (0, 0)),
|
||||
("PreparePartTransaction", false, true, Some("16"), false, (0, 0)),
|
||||
("RenameMultipartData", false, true, None, false, (0, 1)),
|
||||
] {
|
||||
let error = has_error.then(|| Error {
|
||||
code: 1,
|
||||
error_info: "retired source".to_owned(),
|
||||
});
|
||||
let reply = match method {
|
||||
"PreparePartTransaction" => unary_frame(&PreparePartTransactionResponse { success, error }),
|
||||
"SettlePartTransaction" => unary_frame(&SettlePartTransactionResponse { success, error }),
|
||||
_ => unary_frame(&RenameDataResponse {
|
||||
success,
|
||||
error,
|
||||
..Default::default()
|
||||
}),
|
||||
};
|
||||
// An unknown field must survive typed inspection byte-for-byte.
|
||||
let mut framed = reply.to_vec();
|
||||
framed.extend_from_slice(&[0xfa, 0x03, 0x03, b'x', b'y', b'z']);
|
||||
let payload_len = u32::try_from(framed.len() - 5).expect("small reply");
|
||||
framed[1..5].copy_from_slice(&payload_len.to_be_bytes());
|
||||
let reply = Bytes::from(framed);
|
||||
let backend = backend_with_reply(status, Some(reply.clone()))
|
||||
.await
|
||||
.expect("typed reply backend");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy");
|
||||
let pause = proxy.pause_late(&[method], 1, "late-upload").expect("part gate");
|
||||
let request = match method {
|
||||
"PreparePartTransaction" => unary_frame(&PreparePartTransactionRequest {
|
||||
disk: "disk".to_owned(),
|
||||
dst_path: "late-upload/part.1".to_owned(),
|
||||
..Default::default()
|
||||
}),
|
||||
"SettlePartTransaction" => unary_frame(&SettlePartTransactionRequest {
|
||||
disk: "disk".to_owned(),
|
||||
path: "late-upload/part.1".to_owned(),
|
||||
rollback,
|
||||
..Default::default()
|
||||
}),
|
||||
_ => Bytes::from_static(b"late-upload"),
|
||||
};
|
||||
let address = proxy.local_addr();
|
||||
let caller = tokio::spawn(async move {
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let response = call_body(&client, address, method, request).await.expect("retained reply");
|
||||
response.into_body().collect().await.expect("typed response frames")
|
||||
});
|
||||
pause.wait_for_hits(1).await.expect("actual retained request");
|
||||
pause.release();
|
||||
let response = caller.await.expect("caller task");
|
||||
let trailers = response.trailers().expect("reply trailers");
|
||||
assert_eq!(trailers["x-proof"], "unchanged-proof");
|
||||
assert_eq!(trailers["grpc-status"], status.unwrap_or("0"));
|
||||
assert_eq!(response.to_bytes(), reply);
|
||||
assert_eq!(pause.responses(), 1);
|
||||
assert_eq!(
|
||||
(pause.successful_mutation_replies(), pause.refused_mutations()),
|
||||
expected,
|
||||
"typed {method} outcome"
|
||||
);
|
||||
let actions = if method == "SettlePartTransaction" {
|
||||
if rollback { (0, 1) } else { (1, 0) }
|
||||
} else {
|
||||
(0, 0)
|
||||
};
|
||||
assert_eq!(pause.retained_settle_actions(), actions);
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("part reply deadline");
|
||||
}
|
||||
async fn assert_late_packet_ownership() {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
|
||||
@@ -12,9 +12,6 @@ Applies to `crates/ecstore/`.
|
||||
|
||||
- Be careful with allocations and locking in hot paths.
|
||||
- Keep network and disk operations async-friendly; avoid introducing unnecessary blocking.
|
||||
- Coalesce related `open`, `read`/`write`, `fsync`, and `close` filesystem steps into a single `blocking_queue` or `run_blocking_namespace_operation` closure per disk rather than chaining multiple `tokio::fs` or `spawn_blocking` dispatches.
|
||||
- Pin a parent directory once (`openat2(BENEATH | NO_SYMLINKS)` on Linux) and reuse the pinned guard across sibling metadata reads (`read_optional_in_pinned`, `part.N.meta` loops) instead of re-walking ancestor components per file or re-opening when `source_parent == destination_parent`.
|
||||
- After `file.metadata()`, pre-size read buffers (`Vec::with_capacity(len + 1)`) before `read_to_end` to avoid `std::io::read_to_end`'s 32-byte exponential growth (`6–7` `read()` syscalls per small metadata file).
|
||||
- Benchmark-sensitive changes should include measurable rationale.
|
||||
|
||||
## Allocation Discipline in Hot Paths
|
||||
|
||||
@@ -11314,7 +11314,13 @@ mod tests {
|
||||
let marker = marker
|
||||
.as_deref()
|
||||
.expect("production checkpoint should persist an object marker");
|
||||
assert_eq!(marker, expected_marker, "production checkpoint should stop at the first page boundary");
|
||||
let marker_suffix = marker
|
||||
.strip_prefix(&expected_marker)
|
||||
.expect("production checkpoint should stop at the first page boundary");
|
||||
assert!(
|
||||
marker_suffix.is_empty() || marker_suffix.starts_with("[rustfs_cache:v2,"),
|
||||
"unexpected production checkpoint marker suffix: {marker_suffix}"
|
||||
);
|
||||
assert_eq!(version_marker.as_deref(), Some("null"));
|
||||
}
|
||||
|
||||
|
||||
@@ -3186,9 +3186,17 @@ pub mod test_support {
|
||||
use crate::store::init_local_disks_with_instance_ctx;
|
||||
|
||||
pub async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
|
||||
let mut dirs = Vec::with_capacity(4);
|
||||
let mut endpoints = Vec::with_capacity(4);
|
||||
for disk_idx in 0..4 {
|
||||
isolated_store_over_temp_disks_with_count(4).await
|
||||
}
|
||||
|
||||
pub async fn isolated_store_over_temp_disks_with_count(disk_count: usize) -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
|
||||
assert!(
|
||||
matches!(disk_count, 4 | 16),
|
||||
"isolated store fixture requires a registered logical set geometry"
|
||||
);
|
||||
let mut dirs = Vec::with_capacity(disk_count);
|
||||
let mut endpoints = Vec::with_capacity(disk_count);
|
||||
for disk_idx in 0..disk_count {
|
||||
let dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
|
||||
@@ -3201,7 +3209,7 @@ pub mod test_support {
|
||||
let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
drives_per_set: disk_count,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "metadata-sys-cache-test".to_string(),
|
||||
platform: "test".to_string(),
|
||||
|
||||
@@ -43,6 +43,11 @@ const EVENT_METACACHE_LISTING: &str = "metacache_listing";
|
||||
pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = Result<(), DiskError>> + Send>> + Send + 'static>;
|
||||
pub type PartialFn =
|
||||
Box<dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
|
||||
pub(crate) type PartialResultFn = Box<
|
||||
dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = disk::error::Result<()>> + Send>>
|
||||
+ Send
|
||||
+ 'static,
|
||||
>;
|
||||
type FinishedFn = Box<dyn Fn(&[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
@@ -55,7 +60,6 @@ impl FallbackClaimTracker {
|
||||
self.claimed.lock().await.insert(disk.endpoint().to_string());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn claimed_keys(&self) -> HashSet<String> {
|
||||
self.claimed.lock().await.clone()
|
||||
}
|
||||
@@ -313,17 +317,7 @@ pub struct ListPathRawOutcome {
|
||||
pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
|
||||
let rx = rx.child_token();
|
||||
let _cancel_guard = rx.clone().drop_guard();
|
||||
list_path_raw_inner(rx, opts, None).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// Like [`list_path_raw`], but reports how the merged stream ended.
|
||||
pub(crate) async fn list_path_raw_with_outcome(
|
||||
rx: CancellationToken,
|
||||
opts: ListPathRawOptions,
|
||||
) -> disk::error::Result<ListPathRawOutcome> {
|
||||
let rx = rx.child_token();
|
||||
let _cancel_guard = rx.clone().drop_guard();
|
||||
list_path_raw_inner(rx, opts, None).await
|
||||
list_path_raw_inner(rx, opts, None, None).await.map(|_| ())
|
||||
}
|
||||
|
||||
pub(crate) async fn list_path_raw_with_claim_tracker(
|
||||
@@ -333,13 +327,25 @@ pub(crate) async fn list_path_raw_with_claim_tracker(
|
||||
) -> disk::error::Result<ListPathRawOutcome> {
|
||||
let rx = rx.child_token();
|
||||
let _cancel_guard = rx.clone().drop_guard();
|
||||
list_path_raw_inner(rx, opts, Some(claim_tracker)).await
|
||||
list_path_raw_inner(rx, opts, Some(claim_tracker), None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn list_path_raw_with_partial_result(
|
||||
rx: CancellationToken,
|
||||
opts: ListPathRawOptions,
|
||||
claim_tracker: FallbackClaimTracker,
|
||||
partial_result: PartialResultFn,
|
||||
) -> disk::error::Result<ListPathRawOutcome> {
|
||||
let rx = rx.child_token();
|
||||
let _cancel_guard = rx.clone().drop_guard();
|
||||
list_path_raw_inner(rx, opts, Some(claim_tracker), Some(partial_result)).await
|
||||
}
|
||||
|
||||
async fn list_path_raw_inner(
|
||||
rx: CancellationToken,
|
||||
opts: ListPathRawOptions,
|
||||
fallback_claim_tracker: Option<FallbackClaimTracker>,
|
||||
partial_result: Option<PartialResultFn>,
|
||||
) -> disk::error::Result<ListPathRawOutcome> {
|
||||
if opts.disks.is_empty() {
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
@@ -1014,7 +1020,13 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(partial_fn) = opts.partial.as_ref() {
|
||||
if let Some(partial_fn) = partial_result.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
result = partial_fn(MetaCacheEntries(top_entries), &errs) => result?,
|
||||
}
|
||||
} else if let Some(partial_fn) = opts.partial.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
@@ -1296,6 +1308,7 @@ mod tests {
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("listing should complete");
|
||||
@@ -1320,6 +1333,7 @@ mod tests {
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("listing should complete");
|
||||
@@ -1813,6 +1827,39 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_propagates_partial_callback_failure() {
|
||||
let first = MetaCacheEntry {
|
||||
name: "bucket/object-a".to_string(),
|
||||
metadata: vec![1],
|
||||
..Default::default()
|
||||
};
|
||||
let second = MetaCacheEntry {
|
||||
name: "bucket/object-b".to_string(),
|
||||
metadata: vec![2],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let err = list_path_raw_with_partial_result(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None, None],
|
||||
min_disks: 2,
|
||||
test_reader_behaviors: vec![
|
||||
TestReaderBehavior::Entries(vec![first]),
|
||||
TestReaderBehavior::Entries(vec![second]),
|
||||
],
|
||||
..Default::default()
|
||||
},
|
||||
FallbackClaimTracker::default(),
|
||||
Box::new(move |_, _| Box::pin(async { Err(DiskError::ErasureReadQuorum) })),
|
||||
)
|
||||
.await
|
||||
.expect_err("a resolution failure after reader advancement must fail the listing");
|
||||
|
||||
assert_eq!(err, DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_cancels_a_blocked_partial_callback() {
|
||||
let cancel = CancellationToken::new();
|
||||
|
||||
@@ -8748,89 +8748,6 @@ mod tests {
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn endpoint_reconnect_admits_ready_drive_while_another_format_probe_is_blocked() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let reference = crate::layout::format::FormatV3::new(1, 2);
|
||||
let mut peers = Vec::new();
|
||||
let mut endpoints = Vec::new();
|
||||
for index in 0..2 {
|
||||
let mut format = reference.clone();
|
||||
format.erasure.this = reference.erasure.sets[0][index];
|
||||
let peer = TestGrpcPeer::spawn(
|
||||
Bytes::from(format.to_json().expect("format should serialize")),
|
||||
Bytes::from_static(b"recovered-peer-data"),
|
||||
)
|
||||
.await
|
||||
.expect("authenticated regression peer is required");
|
||||
endpoints.push(Endpoint {
|
||||
url: url::Url::parse(&format!("{}/data/rustfs{index}", peer.addr)).expect("endpoint should parse"),
|
||||
is_local: false,
|
||||
pool_idx: 0,
|
||||
set_idx: 0,
|
||||
disk_idx: i32::try_from(index).expect("fixture index fits"),
|
||||
});
|
||||
peers.push(peer);
|
||||
}
|
||||
let set = crate::set_disk::SetDisks::new(
|
||||
"independent-reconnect-test".to_owned(),
|
||||
Arc::new(tokio::sync::RwLock::new(vec![None; 2])),
|
||||
2,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
endpoints,
|
||||
reference.clone(),
|
||||
Vec::new(),
|
||||
)
|
||||
.await;
|
||||
let gate = peers[0].peer.pause_next_format_read();
|
||||
let pending_set = Arc::clone(&set);
|
||||
let pending = tokio::spawn(async move { pending_set.connect_disks().await });
|
||||
time::timeout(Duration::from_secs(10), gate.entered.notified())
|
||||
.await
|
||||
.expect("the first endpoint must reach the format barrier");
|
||||
|
||||
let ready = time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
if let Some(disk) = set.disks.read().await[1].clone() {
|
||||
return disk;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
if let Ok(disk) = &ready {
|
||||
assert!(set.disks.read().await[0].is_none(), "the blocked endpoint must remain unpublished");
|
||||
assert!(!pending.is_finished(), "the sweep still owns its blocked endpoint probe");
|
||||
assert_eq!(
|
||||
disk.read_all("bucket", "object")
|
||||
.await
|
||||
.expect("the ready endpoint must serve real I/O"),
|
||||
Bytes::from_static(b"recovered-peer-data")
|
||||
);
|
||||
assert_eq!(
|
||||
peers[1].peer.object_read_all_disks(),
|
||||
vec![reference.erasure.sets[0][1].to_string()],
|
||||
"the usable handle must carry its validated slot identity"
|
||||
);
|
||||
}
|
||||
gate.release.cancel();
|
||||
pending.await.expect("all endpoint probes must finish after release");
|
||||
for disk in set.disks.read().await.iter().flatten() {
|
||||
disk.close().await.expect("stop each published handle's monitoring");
|
||||
}
|
||||
for peer in peers {
|
||||
peer.stop().await;
|
||||
}
|
||||
ready.expect("a blocked earlier endpoint must not delay a ready drive in the same set");
|
||||
assert!(
|
||||
set.disks.read().await.iter().all(Option::is_some),
|
||||
"both endpoints must eventually be admitted"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn missing_read_disks_preserve_concurrently_published_handles() {
|
||||
|
||||
@@ -41,10 +41,6 @@ use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::{Request, Response};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_REMOTE_LOCK: &str = "remote_lock";
|
||||
|
||||
fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(request: &mut Request<T>) -> std::io::Result<()> {
|
||||
set_tonic_rolling_mutation_body_digest(request)
|
||||
}
|
||||
@@ -237,20 +233,6 @@ pub struct RemoteClient {
|
||||
impl RemoteClient {
|
||||
const ONLINE_CHECK_RESOURCE: &'static str = "health-lock-online";
|
||||
|
||||
fn record_release_ack(&self, lock_id: &LockId) {
|
||||
tracing::trace!(
|
||||
target: "rustfs_ecstore::object_lock_diag",
|
||||
event = EVENT_NAMESPACE_LOCK_RELEASE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "acknowledged",
|
||||
peer_kind = "remote",
|
||||
lock_id = %lock_id,
|
||||
peer = %self.addr,
|
||||
"Remote namespace lock release acknowledged"
|
||||
);
|
||||
}
|
||||
|
||||
pub fn new(endpoint: String) -> Self {
|
||||
Self { addr: endpoint }
|
||||
}
|
||||
@@ -1222,6 +1204,8 @@ impl LockClient for RemoteClient {
|
||||
}
|
||||
|
||||
async fn release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
info!("remote release for {}", lock_id);
|
||||
|
||||
let unlock_request = Self::create_unlock_request(lock_id);
|
||||
let request_string = serde_json::to_string(&unlock_request)
|
||||
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
|
||||
@@ -1242,9 +1226,6 @@ impl LockClient for RemoteClient {
|
||||
if let Some(error_info) = resp.error_info {
|
||||
return Err(LockError::internal(error_info));
|
||||
}
|
||||
if resp.success {
|
||||
self.record_release_ack(lock_id);
|
||||
}
|
||||
Ok(resp.success)
|
||||
}
|
||||
|
||||
@@ -1280,13 +1261,7 @@ impl LockClient for RemoteClient {
|
||||
Ok(lock_ids
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, lock_id)| {
|
||||
let result = resp.results.get(idx);
|
||||
if result.is_some_and(|result| result.success && result.error_info.is_none()) {
|
||||
self.record_release_ack(lock_id);
|
||||
}
|
||||
result.map(|result| result.success).unwrap_or(false)
|
||||
})
|
||||
.map(|(idx, _)| resp.results.get(idx).map(|result| result.success).unwrap_or(false))
|
||||
.collect())
|
||||
}
|
||||
|
||||
|
||||
@@ -317,17 +317,6 @@ impl Sets {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
|
||||
if let Some(exit_signal) = &self.exit_signal {
|
||||
let _ = exit_signal.send(());
|
||||
// Wait for an in-flight reconnect pass as well as its receiver to exit.
|
||||
while exit_signal.receiver_count() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep simulated peers' metadata ownership separate while sharing disks and lock clients.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
|
||||
|
||||
@@ -19,7 +19,6 @@ use tracing::Span;
|
||||
use uuid::Uuid;
|
||||
|
||||
const EVENT_OBJECT_NAMESPACE_LOCK: &str = "object_namespace_lock";
|
||||
const EVENT_OBJECT_NAMESPACE_LOCK_PEER: &str = "object_namespace_lock_peer";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
|
||||
|
||||
@@ -57,7 +56,6 @@ struct AttemptContext {
|
||||
requested_version_id: Option<String>,
|
||||
namespace: String,
|
||||
owner: String,
|
||||
lock_id: Option<String>,
|
||||
mode: &'static str,
|
||||
timeout: Duration,
|
||||
started_at: Instant,
|
||||
@@ -87,7 +85,6 @@ impl ObjectLockAttempt {
|
||||
requested_version_id: requested_version_id.map(str::to_owned),
|
||||
namespace: lock.namespace().to_owned(),
|
||||
owner: lock.owner().to_owned(),
|
||||
lock_id: None,
|
||||
mode,
|
||||
timeout,
|
||||
started_at: Instant::now(),
|
||||
@@ -108,29 +105,9 @@ impl ObjectLockAttempt {
|
||||
context.finished = true;
|
||||
context.trace("disabled");
|
||||
}
|
||||
Ok(guard) => {
|
||||
context.lock_id = guard.lock_id().map(ToString::to_string);
|
||||
Ok(_) => {
|
||||
context.acquired_at = Some(Instant::now());
|
||||
context.trace("acquired");
|
||||
if let NamespaceLockGuard::Standard(guard) = guard {
|
||||
for (peer_index, peer_lock_id) in guard.granted_lock_ids().enumerate() {
|
||||
context.parent.in_scope(|| {
|
||||
tracing::trace!(
|
||||
target: "rustfs_ecstore::object_lock_diag",
|
||||
parent: &context.parent,
|
||||
event = EVENT_OBJECT_NAMESPACE_LOCK_PEER,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_OBJECT_LOCK,
|
||||
state = "granted",
|
||||
lock_attempt_id = %context.id,
|
||||
lock_id = context.lock_id.as_deref(),
|
||||
peer_lock_id = %peer_lock_id,
|
||||
peer_index,
|
||||
"Object namespace peer owner granted"
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
context.finished = true;
|
||||
@@ -194,7 +171,6 @@ impl AttemptContext {
|
||||
mode = self.mode,
|
||||
owner = %self.owner,
|
||||
lock_attempt_id = %self.id,
|
||||
lock_id = self.lock_id.as_deref(),
|
||||
acquire_ms = duration_ms(acquired_at.duration_since(self.started_at)),
|
||||
hold_ms = duration_ms(now.duration_since(acquired_at)),
|
||||
timeout_ms = duration_ms(self.timeout),
|
||||
@@ -259,238 +235,15 @@ pub(crate) mod tests {
|
||||
}
|
||||
|
||||
pub(crate) fn rows(&self) -> Vec<serde_json::Value> {
|
||||
self.all_rows()
|
||||
.into_iter()
|
||||
.filter(|row| row["fields"]["event"] == EVENT_OBJECT_NAMESPACE_LOCK)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn all_rows(&self) -> Vec<serde_json::Value> {
|
||||
let bytes = self.0.lock().expect("lock diagnostic capture mutex");
|
||||
let text = std::str::from_utf8(&bytes).expect("UTF-8 diagnostic events");
|
||||
text.lines()
|
||||
.map(|line| serde_json::from_str::<serde_json::Value>(line).expect("JSON diagnostic event"))
|
||||
.filter(|row| row["fields"]["event"] == EVENT_OBJECT_NAMESPACE_LOCK)
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn object_lock_diagnostics_join_guard_identity_to_physical_release() {
|
||||
use rustfs_lock::{
|
||||
DisabledLockManager, FastObjectLockManager, GlobalLockManager, LocalClient, LockClient, LockRequest, LockType,
|
||||
NamespaceLock, ObjectKey,
|
||||
};
|
||||
|
||||
temp_env::with_vars([(rustfs_config::ENV_OBJECT_LOCK_DIAG_ENABLE, Some("true"))], || {
|
||||
let capture = CapturedLockEvents::default();
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("diagnostic release runtime");
|
||||
tracing::subscriber::with_default(capture.subscriber(), || {
|
||||
runtime.block_on(async {
|
||||
let clients = (0..4)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn LockClient>)
|
||||
.collect::<Vec<_>>();
|
||||
let lock = NamespaceLockWrapper::new(
|
||||
NamespaceLock::with_clients_and_quorum("diagnostics".to_owned(), clients.clone(), 4),
|
||||
ObjectKey::new("bucket", "object"),
|
||||
"coordinator".to_owned(),
|
||||
);
|
||||
let request = tracing::info_span!("request", request_id = "original-multipart-request");
|
||||
let mut attempt = request.in_scope(|| {
|
||||
ObjectLockAttempt::start(
|
||||
"abort_multipart_upload",
|
||||
"bucket",
|
||||
"object",
|
||||
None,
|
||||
&lock,
|
||||
"write",
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
});
|
||||
let result = lock.get_write_lock(Duration::from_secs(5)).await;
|
||||
attempt.observe(&result);
|
||||
let guard = result.expect("four real namespace owners");
|
||||
let id = guard.lock_id().expect("distributed guard identity").clone();
|
||||
let peer_ids = match &guard {
|
||||
NamespaceLockGuard::Standard(guard) => {
|
||||
assert_eq!(guard.granted_lock_ids().count(), 4);
|
||||
guard.granted_lock_ids().cloned().collect::<std::collections::HashSet<_>>()
|
||||
}
|
||||
NamespaceLockGuard::Fast(_) => panic!("real distributed namespace required"),
|
||||
};
|
||||
let rows = capture.rows();
|
||||
assert_eq!(rows.last().expect("acquired event")["fields"]["lock_id"], id.to_string());
|
||||
let rows = capture.all_rows();
|
||||
let mappings = rows
|
||||
.iter()
|
||||
.filter(|row| row["fields"]["event"] == EVENT_OBJECT_NAMESPACE_LOCK_PEER)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(mappings.len(), 4, "every granted peer must map to its logical guard");
|
||||
assert!(mappings.iter().all(|row| {
|
||||
row["fields"]["lock_id"] == id.to_string()
|
||||
&& peer_ids
|
||||
.iter()
|
||||
.any(|peer_id| row["fields"]["peer_lock_id"] == peer_id.to_string())
|
||||
}));
|
||||
for client in &clients {
|
||||
let mut owned = 0;
|
||||
for peer_id in &peer_ids {
|
||||
owned += usize::from(
|
||||
client
|
||||
.check_status(peer_id)
|
||||
.await
|
||||
.expect("actual peer owner status")
|
||||
.is_some(),
|
||||
);
|
||||
}
|
||||
assert_eq!(owned, 1, "each client must hold exactly one actual granted identity");
|
||||
}
|
||||
drop((guard, attempt));
|
||||
assert!(
|
||||
capture.all_rows().iter().all(|row| row["fields"]["state"] != "released"),
|
||||
"enqueue is not release acknowledgement"
|
||||
);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while capture
|
||||
.all_rows()
|
||||
.iter()
|
||||
.filter(|row| {
|
||||
row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "released"
|
||||
})
|
||||
.count()
|
||||
< 4
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all physical release callbacks finish");
|
||||
for client in &clients {
|
||||
for peer_id in &peer_ids {
|
||||
assert!(
|
||||
client
|
||||
.check_status(peer_id)
|
||||
.await
|
||||
.expect("released peer owner status")
|
||||
.is_none()
|
||||
);
|
||||
assert!(!client.release(peer_id).await.expect("duplicate release is absent"));
|
||||
}
|
||||
}
|
||||
let disabled = LocalClient::with_manager(Arc::new(GlobalLockManager::Disabled(DisabledLockManager::new())));
|
||||
let disabled_request =
|
||||
LockRequest::new(ObjectKey::new("bucket", "disabled"), LockType::Exclusive, "disabled-owner".to_owned());
|
||||
assert!(
|
||||
disabled
|
||||
.acquire_lock(&disabled_request)
|
||||
.await
|
||||
.expect("disabled acquire bypass")
|
||||
.success
|
||||
);
|
||||
assert!(
|
||||
disabled
|
||||
.release(&disabled_request.lock_id)
|
||||
.await
|
||||
.expect("disabled release bypass")
|
||||
);
|
||||
let rows = capture.all_rows();
|
||||
let releases = rows
|
||||
.iter()
|
||||
.filter(|row| row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "released")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
releases.len(),
|
||||
4,
|
||||
"duplicates and disabled guards cannot acknowledge another physical release"
|
||||
);
|
||||
let physical_mappings = rows
|
||||
.iter()
|
||||
.filter(|row| row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "mapped")
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(physical_mappings.len(), 4);
|
||||
assert!(releases.iter().all(|row| physical_mappings.iter().any(|mapped| {
|
||||
mapped["fields"]["physical_guard_id"] == row["fields"]["physical_guard_id"]
|
||||
&& mapped["fields"]["process_id"] == row["fields"]["process_id"]
|
||||
&& peer_ids
|
||||
.iter()
|
||||
.any(|peer_id| mapped["fields"]["lock_id"] == peer_id.to_string())
|
||||
})));
|
||||
let dropped = capture.rows().pop().expect("guard drop event");
|
||||
assert_eq!(dropped["fields"]["lock_id"], id.to_string());
|
||||
assert!(
|
||||
dropped["spans"]
|
||||
.as_array()
|
||||
.expect("retained request span")
|
||||
.iter()
|
||||
.any(|span| span["request_id"] == "original-multipart-request")
|
||||
);
|
||||
let manager = Arc::new(FastObjectLockManager::new());
|
||||
let stale_client = LocalClient::with_manager(Arc::new(GlobalLockManager::Enabled(manager.clone())));
|
||||
let stale_request = LockRequest::new(
|
||||
ObjectKey::new("bucket", "externally-released"),
|
||||
LockType::Exclusive,
|
||||
"stale-owner".to_owned(),
|
||||
);
|
||||
assert!(
|
||||
stale_client
|
||||
.acquire_lock(&stale_request)
|
||||
.await
|
||||
.expect("real stale owner")
|
||||
.success
|
||||
);
|
||||
assert_eq!(manager.force_unlock(&stale_request.resource), 1);
|
||||
let mut successor = manager
|
||||
.acquire_write_lock(stale_request.resource.clone(), "successor-owner")
|
||||
.await
|
||||
.expect("real successor guard");
|
||||
assert!(
|
||||
stale_client
|
||||
.release(&stale_request.lock_id)
|
||||
.await
|
||||
.expect("retire stale client record")
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.get_lock_info(&stale_request.resource)
|
||||
.expect("successor remains locked")
|
||||
.owner
|
||||
.as_ref(),
|
||||
"successor-owner"
|
||||
);
|
||||
assert_eq!(
|
||||
capture
|
||||
.all_rows()
|
||||
.iter()
|
||||
.filter(
|
||||
|row| row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "released"
|
||||
)
|
||||
.count(),
|
||||
4,
|
||||
"a stale guard's failed physical release must not produce an ACK"
|
||||
);
|
||||
let successor_id = successor.guard_id();
|
||||
assert!(successor.release(), "the actual successor release succeeds");
|
||||
assert!(!successor.release(), "a duplicate explicit release cannot complete again");
|
||||
drop(successor);
|
||||
assert_eq!(
|
||||
capture
|
||||
.all_rows()
|
||||
.iter()
|
||||
.filter(|row| row["fields"]["event"] == "namespace_lock_release"
|
||||
&& row["fields"]["state"] == "released"
|
||||
&& row["fields"]["physical_guard_id"] == successor_id)
|
||||
.count(),
|
||||
1,
|
||||
"explicit release followed by Drop records one physical completion"
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
fn namespace() -> NamespaceLockWrapper {
|
||||
let manager = rustfs_lock::GlobalLockManager::Enabled(Arc::new(rustfs_lock::fast_lock::FastObjectLockManager::new()));
|
||||
NamespaceLockWrapper::new(
|
||||
|
||||
@@ -1235,13 +1235,7 @@ impl DiskHealthTracker {
|
||||
}
|
||||
|
||||
pub(crate) fn metric_epoch_for_reconnect(&self) -> Self {
|
||||
let health = Self::new();
|
||||
// Explicit test faults stay attached to the drive across handle renewal.
|
||||
#[cfg(test)]
|
||||
if self.test_forced_offline.load(Ordering::Acquire) {
|
||||
health.force_offline_for_test();
|
||||
}
|
||||
health
|
||||
Self::new()
|
||||
}
|
||||
|
||||
pub fn last_capacity_snapshot(&self) -> Option<(u64, u64, u64, u64)> {
|
||||
@@ -2670,11 +2664,6 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn is_online(&self) -> bool {
|
||||
// A matching format does not make a handle usable while its I/O gate is faulty.
|
||||
// Reconnect must renew it even if its old recovery monitor has stopped.
|
||||
if self.health.is_faulty() {
|
||||
return false;
|
||||
}
|
||||
let Ok(Some(disk_id)) = self.disk.get_disk_id().await else {
|
||||
return false;
|
||||
};
|
||||
@@ -4706,38 +4695,6 @@ mod tests {
|
||||
assert_eq!(snapshot.total_errors_availability, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_online_status_matches_io_health_and_disk_identity() {
|
||||
let dir = tempfile::tempdir().expect("create a drive root");
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 root")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let meta_dir = dir.path().join(RUSTFS_META_BUCKET);
|
||||
tokio::fs::create_dir_all(&meta_dir).await.expect("create metadata directory");
|
||||
let mut format = FormatV3::new(1, 1);
|
||||
format.erasure.this = format.erasure.sets[0][0];
|
||||
tokio::fs::write(meta_dir.join(FORMAT_CONFIG_FILE), format.to_json().expect("serialize format"))
|
||||
.await
|
||||
.expect("persist original identity");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("open formatted drive"));
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
assert!(wrapper.is_online().await, "healthy formatted drive is online");
|
||||
wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Suspect);
|
||||
assert!(wrapper.is_online().await, "suspect drives still admit foreground I/O");
|
||||
wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
|
||||
assert!(!wrapper.is_online().await, "a readable format cannot make a faulty handle usable");
|
||||
assert!(!wrapper.health.mark_recovery_success(&endpoint, "test_probe"));
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Returning);
|
||||
assert!(
|
||||
!wrapper.is_online().await,
|
||||
"returning drives retain the faulty I/O gate until recovery completes"
|
||||
);
|
||||
wrapper.reset_health_for_store_init_retry();
|
||||
wrapper.set_disk_id_state(Some(Uuid::new_v4())).await;
|
||||
assert!(!wrapper.is_online().await, "healthy state must not bypass a mismatched disk identity");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fresh_capacity_keeps_disk_identity_precheck_without_requiring_id_in_snapshot() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
@@ -159,18 +159,8 @@ pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
|
||||
std::fs::symlink_metadata(path)
|
||||
}
|
||||
|
||||
pub fn create_dir_all_std(path: &Path) -> io::Result<()> {
|
||||
match std::fs::create_dir(path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => Ok(()),
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => std::fs::create_dir_all(path),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
tokio::task::spawn_blocking(move || create_dir_all_std(&path)).await?
|
||||
fs::create_dir_all(path.as_ref()).await
|
||||
}
|
||||
|
||||
fn is_dir_error(e: &io::Error) -> bool {
|
||||
@@ -220,7 +210,6 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::create_dir(path.as_ref()).await
|
||||
}
|
||||
@@ -250,7 +239,7 @@ impl BucketExistenceCache {
|
||||
}
|
||||
}
|
||||
|
||||
fn check_exists(&self, path: &Path) -> Option<bool> {
|
||||
fn check_exists(&self, path: &PathBuf) -> Option<bool> {
|
||||
let mut cache = self.cache.lock().ok()?;
|
||||
if let Some((timestamp, exists)) = cache.get(path) {
|
||||
if timestamp.elapsed() < self.ttl {
|
||||
@@ -267,7 +256,7 @@ impl BucketExistenceCache {
|
||||
}
|
||||
}
|
||||
|
||||
fn invalidate(&self, path: &Path) {
|
||||
fn invalidate(&self, path: &PathBuf) {
|
||||
if let Ok(mut cache) = self.cache.lock() {
|
||||
cache.remove(path);
|
||||
}
|
||||
@@ -279,16 +268,15 @@ static BUCKET_EXISTENCE_CACHE: std::sync::LazyLock<BucketExistenceCache> =
|
||||
|
||||
/// Cached access check - reduces statx syscalls
|
||||
pub async fn cached_access(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
let path = path.as_ref();
|
||||
let path_buf = path.as_ref().to_path_buf();
|
||||
|
||||
if let Some(exists) = BUCKET_EXISTENCE_CACHE.check_exists(path) {
|
||||
if let Some(exists) = BUCKET_EXISTENCE_CACHE.check_exists(&path_buf) {
|
||||
if exists {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(io::Error::new(io::ErrorKind::NotFound, "bucket not found (cached)"));
|
||||
}
|
||||
|
||||
let path_buf = path.to_path_buf();
|
||||
let result = fs::metadata(&path_buf).await;
|
||||
|
||||
match &result {
|
||||
@@ -304,7 +292,7 @@ pub async fn cached_access(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
|
||||
pub fn invalidate_bucket_cache(path: impl AsRef<Path>) {
|
||||
BUCKET_EXISTENCE_CACHE.invalidate(path.as_ref());
|
||||
BUCKET_EXISTENCE_CACHE.invalidate(&path.as_ref().to_path_buf());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -115,7 +115,7 @@ use time::OffsetDateTime;
|
||||
use tokio::fs::{self, File};
|
||||
#[cfg(not(unix))]
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::io::{AsyncRead, AsyncSeekExt, AsyncWrite, ErrorKind, ReadBuf};
|
||||
use tokio::io::{AsyncRead, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind, ReadBuf};
|
||||
use tokio::sync::{Notify, RwLock, Semaphore};
|
||||
use tokio::time::{Instant, Sleep, interval_at, timeout};
|
||||
use tracing::{debug, error, info, warn};
|
||||
@@ -301,10 +301,9 @@ fn read_all_data_std(
|
||||
let size = usize::try_from(metadata.len()).map_err(|err| ReadAllError::Disk(DiskError::other(err)))?;
|
||||
let mut bytes = Vec::new();
|
||||
bytes
|
||||
.try_reserve_exact(size.saturating_add(1))
|
||||
.try_reserve_exact(size)
|
||||
.map_err(|err| ReadAllError::Disk(Error::other(err)))?;
|
||||
std::io::Read::read_to_end(&mut std::io::Read::take(&mut file, metadata.len()), &mut bytes)
|
||||
.map_err(|err| ReadAllError::Disk(to_file_error(err).into()))?;
|
||||
std::io::Read::read_to_end(&mut file, &mut bytes).map_err(|err| ReadAllError::Disk(to_file_error(err).into()))?;
|
||||
|
||||
let modtime = metadata.modified().ok().map(OffsetDateTime::from);
|
||||
Ok((bytes, modtime))
|
||||
@@ -8380,7 +8379,7 @@ impl LocalDisk {
|
||||
return Err(DiskError::FileNotFound);
|
||||
}
|
||||
|
||||
let meta_path = file_path.as_ref().join(STORAGE_FORMAT_FILE);
|
||||
let meta_path = path_join(&[file_path.as_ref(), Path::new(STORAGE_FORMAT_FILE)]);
|
||||
|
||||
let res = {
|
||||
if read_data {
|
||||
@@ -8416,7 +8415,8 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
#[hotpath::measure(impl_type = "LocalDisk")]
|
||||
async fn read_metadata_with_dmtime(&self, file_path: PathBuf) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
|
||||
async fn read_metadata_with_dmtime(&self, file_path: impl AsRef<Path>) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
|
||||
check_path_length(file_path.as_ref().to_string_lossy().as_ref())?;
|
||||
let io_permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
|
||||
|
||||
// HP-12 item 1 (sub-change A): fold the open + fstat + bounded xl.meta
|
||||
@@ -8429,7 +8429,9 @@ impl LocalDisk {
|
||||
// - is_dir -> Error::FileNotFound (NOT to_file_error(EISDIR))
|
||||
// - metadata failure -> to_file_error
|
||||
// - parse failure -> propagated verbatim from read_xl_meta_no_data_sync (`?`)
|
||||
let (data, modtime) = self.run_metadata_read(io_permit, file_path, read_metadata_only_std).await??;
|
||||
let (data, modtime) = self
|
||||
.run_metadata_read(io_permit, file_path.as_ref().to_path_buf(), read_metadata_only_std)
|
||||
.await??;
|
||||
|
||||
Ok((data, modtime))
|
||||
}
|
||||
@@ -9460,64 +9462,101 @@ impl LocalDisk {
|
||||
skip_parent
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
let is_owned = matches!(data, InternalBuf::Owned(_));
|
||||
let buf = match data {
|
||||
InternalBuf::Ref(slice) => Bytes::copy_from_slice(slice),
|
||||
InternalBuf::Owned(bytes) => bytes,
|
||||
};
|
||||
|
||||
let path = file_path.to_path_buf();
|
||||
let create_parent = if let Some(parent) = path.parent()
|
||||
&& parent != skip_parent
|
||||
{
|
||||
os::check_path_length(parent.to_string_lossy().as_ref())?;
|
||||
!(!skip_parent.as_os_str().is_empty() && skip_parent.starts_with(parent))
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
let root = self.publication_root.clone();
|
||||
let operation = move |is_cancelled: &dyn Fn() -> bool| {
|
||||
if is_cancelled() {
|
||||
return Err(std::io::Error::other("blocking namespace operation cancelled"));
|
||||
}
|
||||
if create_parent && let Some(parent) = path.parent() {
|
||||
super::fs::create_dir_all_std(parent)?;
|
||||
}
|
||||
#[cfg(test)]
|
||||
if is_owned {
|
||||
run_owned_file_write_before_open(&path);
|
||||
}
|
||||
|
||||
let (mut file, parent_handle) = if sync == SyncMode::FileAndDir {
|
||||
root.create_truncate_with_parent(&path)?
|
||||
} else {
|
||||
(root.create_truncate(&path)?, None)
|
||||
};
|
||||
std::io::Write::write_all(&mut file, buf.as_ref())?;
|
||||
if sync != SyncMode::None {
|
||||
file.sync_data()?;
|
||||
// FileOnly callers rename the file away, so the tmp
|
||||
// directory entry never needs to become durable.
|
||||
if sync == SyncMode::FileAndDir
|
||||
&& let Some(parent) = path.parent()
|
||||
{
|
||||
os::fsync_directory_handle_std(parent, parent_handle.as_ref())?;
|
||||
match data {
|
||||
InternalBuf::Ref(buf) => {
|
||||
let mut f = self.open_file(file_path, O_CREATE | O_WRONLY | O_TRUNC, skip_parent).await?;
|
||||
f.write_all(buf).await.map_err(to_file_error)?;
|
||||
if sync != SyncMode::None {
|
||||
f.sync_data().await.map_err(to_file_error)?;
|
||||
// Persist the directory entry too, so a freshly created file
|
||||
// survives power loss along with its contents. Skipped for
|
||||
// FileOnly: the caller renames the file away immediately.
|
||||
if sync == SyncMode::FileAndDir
|
||||
&& let Some(parent) = file_path.parent()
|
||||
{
|
||||
os::fsync_dir(parent).await.map_err(to_file_error)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let result = if sync == SyncMode::None {
|
||||
self.blocking_queue.run_cancellable(operation).await
|
||||
} else {
|
||||
self.blocking_queue.run_fsync_cancellable(operation).await
|
||||
};
|
||||
result.map_err(to_file_error)?;
|
||||
InternalBuf::Owned(buf) => {
|
||||
let path = file_path.to_path_buf();
|
||||
if let Some(parent) = path.parent()
|
||||
&& parent != skip_parent
|
||||
{
|
||||
os::make_dir_all(parent, skip_parent).await?;
|
||||
}
|
||||
|
||||
let root = self.publication_root.clone();
|
||||
let operation = move |is_cancelled: &dyn Fn() -> bool| {
|
||||
if is_cancelled() {
|
||||
return Err(std::io::Error::other("blocking namespace operation cancelled"));
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_owned_file_write_before_open(&path);
|
||||
|
||||
let (mut file, parent_handle) = if sync == SyncMode::FileAndDir {
|
||||
root.create_truncate_with_parent(&path)?
|
||||
} else {
|
||||
(root.create_truncate(&path)?, None)
|
||||
};
|
||||
std::io::Write::write_all(&mut file, buf.as_ref())?;
|
||||
if sync != SyncMode::None {
|
||||
file.sync_data()?;
|
||||
// FileOnly callers rename the file away, so the tmp
|
||||
// directory entry never needs to become durable.
|
||||
if sync == SyncMode::FileAndDir
|
||||
&& let Some(parent) = path.parent()
|
||||
{
|
||||
os::fsync_directory_handle_std(parent, parent_handle.as_ref())?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let result = if sync == SyncMode::None {
|
||||
self.blocking_queue.run_cancellable(operation).await
|
||||
} else {
|
||||
self.blocking_queue.run_fsync_cancellable(operation).await
|
||||
};
|
||||
result.map_err(to_file_error)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn open_file(&self, path: impl AsRef<Path>, mode: usize, skip_parent: impl AsRef<Path>) -> Result<File> {
|
||||
let mut skip_parent = skip_parent.as_ref();
|
||||
if skip_parent.as_os_str().is_empty() {
|
||||
skip_parent = self.io_root();
|
||||
}
|
||||
|
||||
if let Some(parent) = path.as_ref().parent()
|
||||
&& parent != skip_parent
|
||||
{
|
||||
os::make_dir_all(parent, skip_parent).await?;
|
||||
}
|
||||
|
||||
let f = if mode == O_CREATE | O_WRONLY | O_TRUNC {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
let root = self.publication_root.clone();
|
||||
File::from_std(
|
||||
self.blocking_queue
|
||||
.run_cancellable(move |is_cancelled| {
|
||||
if is_cancelled() {
|
||||
return Err(std::io::Error::other("blocking namespace operation cancelled"));
|
||||
}
|
||||
root.create_truncate(&path)
|
||||
})
|
||||
.await
|
||||
.map_err(to_file_error)?,
|
||||
)
|
||||
} else {
|
||||
super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?
|
||||
};
|
||||
|
||||
Ok(f)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
async fn open_file_read_only(&self, path: impl AsRef<Path>) -> Result<File> {
|
||||
let f = super::fs::open_file(path.as_ref(), O_RDONLY).await.map_err(to_file_error)?;
|
||||
@@ -10477,11 +10516,10 @@ async fn read_file_all(path: impl AsRef<Path>, root: &os::PublicationRoot) -> Re
|
||||
let root = root.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let result = (|| {
|
||||
let file = root.open_readonly(&path)?;
|
||||
let mut file = root.open_readonly(&path)?;
|
||||
let metadata = file.metadata()?;
|
||||
let size = usize::try_from(metadata.len()).map_err(std::io::Error::other)?;
|
||||
let mut data = Vec::with_capacity(size.saturating_add(1));
|
||||
std::io::Read::read_to_end(&mut std::io::Read::take(file, metadata.len()), &mut data)?;
|
||||
let mut data = Vec::new();
|
||||
std::io::Read::read_to_end(&mut file, &mut data)?;
|
||||
Ok::<_, std::io::Error>((Bytes::from(data), metadata))
|
||||
})();
|
||||
result
|
||||
@@ -10537,21 +10575,7 @@ fn local_disk_object_path(root: &Path, bucket: &str, key: &str) -> Result<PathBu
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[inline]
|
||||
fn is_clean_relative_segment(s: &str) -> bool {
|
||||
let b = s.as_bytes();
|
||||
!matches!(b.first(), None | Some(b'/' | b'.')) && !b.ends_with(b"/") && !b.windows(2).any(|w| matches!(w, b"//" | b"/."))
|
||||
}
|
||||
|
||||
fn build_local_disk_object_path(root: &Path, bucket: &str, key: &str) -> (PathBuf, PathBuf) {
|
||||
#[cfg(not(windows))]
|
||||
if is_clean_relative_segment(bucket) && is_clean_relative_segment(key) {
|
||||
let bucket_path = root.join(bucket);
|
||||
let path = bucket_path.join(key);
|
||||
return (bucket_path, path);
|
||||
}
|
||||
|
||||
let cache_key = if key.is_empty() {
|
||||
bucket.to_string()
|
||||
} else {
|
||||
@@ -10716,10 +10740,9 @@ fn reject_local_disk_symlink_components(root: &Path, path: &Path) -> Result<()>
|
||||
fn normalize_path_cow(path: &Path) -> std::borrow::Cow<'_, Path> {
|
||||
// Windows prefixes retain the existing PathBuf reconstruction semantics.
|
||||
#[cfg(not(windows))]
|
||||
if !path.as_os_str().as_encoded_bytes().contains(&b'.')
|
||||
|| !path
|
||||
.components()
|
||||
.any(|c| matches!(c, std::path::Component::ParentDir | std::path::Component::CurDir))
|
||||
if !path
|
||||
.components()
|
||||
.any(|c| matches!(c, std::path::Component::ParentDir | std::path::Component::CurDir))
|
||||
{
|
||||
return std::borrow::Cow::Borrowed(path);
|
||||
}
|
||||
@@ -12439,12 +12462,12 @@ impl DiskAPI for LocalDisk {
|
||||
let src_volume_dir = self.io_get_bucket_path(src_volume)?;
|
||||
let dst_volume_dir = self.io_get_bucket_path(dst_volume)?;
|
||||
if !skip_access_checks(src_volume) {
|
||||
cached_access(&src_volume_dir)
|
||||
access(&src_volume_dir)
|
||||
.await
|
||||
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
|
||||
}
|
||||
if !skip_access_checks(dst_volume) {
|
||||
cached_access(&dst_volume_dir)
|
||||
access(&dst_volume_dir)
|
||||
.await
|
||||
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
|
||||
}
|
||||
@@ -12523,7 +12546,7 @@ impl DiskAPI for LocalDisk {
|
||||
if !origvolume.is_empty() {
|
||||
let origvolume_dir = self.io_get_bucket_path(origvolume)?;
|
||||
if !skip_access_checks(origvolume) {
|
||||
cached_access(origvolume_dir)
|
||||
access(origvolume_dir)
|
||||
.await
|
||||
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
|
||||
}
|
||||
@@ -13153,7 +13176,7 @@ impl DiskAPI for LocalDisk {
|
||||
if !org_volume.is_empty() {
|
||||
let org_volume_path = self.io_get_bucket_path(org_volume)?;
|
||||
if !skip_access_checks(org_volume) {
|
||||
cached_access(&org_volume_path)
|
||||
access(&org_volume_path)
|
||||
.await
|
||||
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ use super::{
|
||||
LocalDisk,
|
||||
multipart_source_state::{
|
||||
LoadedSource, SourceHead, SourceState, SourceStore, decode, encode, load_source, raw_sha, read_optional,
|
||||
read_optional_in_pinned,
|
||||
},
|
||||
};
|
||||
use crate::disk::{
|
||||
@@ -128,6 +127,20 @@ pub(super) fn validate_retired_head(
|
||||
}
|
||||
|
||||
impl SourceStore {
|
||||
pub async fn root_missing(&self) -> Result<bool> {
|
||||
let source = self.source.clone();
|
||||
let base = self.base.clone();
|
||||
let root = self.publication_root.clone();
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
match os::pin_multipart_source_directory(&source, &base, &root) {
|
||||
Ok(_) => Ok(false),
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => Ok(true),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
fn verify_abort_cohort(&self, certificate: &MultipartAbortCertificate) -> Result<()> {
|
||||
certificate.verify_with_authority(&self.authority)?;
|
||||
if certificate.disk_ids != self.disk_ids
|
||||
@@ -252,7 +265,7 @@ impl SourceStore {
|
||||
let certificate = self.read_abort_certificate(&loaded).await?;
|
||||
Ok(MultipartSourceInspection {
|
||||
disk_id: self.disk_id,
|
||||
root_missing: loaded.root_missing,
|
||||
root_missing: self.root_missing().await?,
|
||||
abort_round: loaded
|
||||
.state
|
||||
.as_ref()
|
||||
@@ -607,17 +620,12 @@ pub(super) struct OrdinaryMultipartDeleteCheck {
|
||||
}
|
||||
impl OrdinaryMultipartDeleteCheck {
|
||||
pub(super) fn validate(&self) -> Result<()> {
|
||||
let pinned = match os::pin_multipart_source_directory(&self.source, &self.base, &self.root) {
|
||||
Ok(pinned) => pinned,
|
||||
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
if let Some(raw) = read_optional_in_pinned(&pinned, &self.source, STATE, &self.base, &self.root)? {
|
||||
if let Some(raw) = read_optional(&self.source.join(STATE), &self.base, &self.root)? {
|
||||
let state: SourceState = decode(&raw)?;
|
||||
if state.abort.is_some() || state.phase == MultipartSourcePhase::Aborting {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
} else if os::multipart_source_has_native_controls_in_pinned(&self.source, &pinned)? {
|
||||
} else if os::multipart_source_has_native_controls(&self.source, &self.base, &self.root)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -170,7 +170,7 @@ impl LocalDisk {
|
||||
recovery: loaded.state.as_ref().and_then(|state| state.recovery.clone()),
|
||||
negative_certificate,
|
||||
ready_certificate,
|
||||
root_missing: loaded.root_missing,
|
||||
root_missing: store.root_missing().await?,
|
||||
abort_round: loaded
|
||||
.state
|
||||
.as_ref()
|
||||
|
||||
@@ -106,7 +106,6 @@ impl SourceHead {
|
||||
}
|
||||
|
||||
pub(super) struct LoadedSource {
|
||||
pub root_missing: bool,
|
||||
pub state: Option<SourceState>,
|
||||
pub head: Option<SourceHead>,
|
||||
pub original: Option<Bytes>,
|
||||
@@ -225,31 +224,11 @@ pub(super) fn read_optional(path: &Path, base: &Path, root: &os::PublicationRoot
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
}
|
||||
pub(super) fn read_optional_in_pinned(
|
||||
pinned: &os::ExistingBaseDirectoryGuard,
|
||||
parent: &Path,
|
||||
name: &str,
|
||||
base: &Path,
|
||||
root: &os::PublicationRoot,
|
||||
) -> Result<Option<Bytes>> {
|
||||
match os::read_multipart_source_metadata_in_pinned(pinned, parent, std::ffi::OsStr::new(name), base, root) {
|
||||
Ok(bytes) => Ok(Some(Bytes::from(bytes))),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
}
|
||||
/// Canonical source mutations are admitted under SourceRW at the syscall,
|
||||
/// not on the earlier async caller that may have timed out.
|
||||
pub(super) fn validate_source_mutation(source: &Path, base: &Path, root: &os::PublicationRoot, initial: bool) -> Result<()> {
|
||||
let pinned = match os::pin_multipart_source_directory(source, base, root) {
|
||||
Ok(pinned) => pinned,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
return if initial { Ok(()) } else { Err(DiskError::FileNotFound) };
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let state = read_optional_in_pinned(&pinned, source, STATE, base, root)?;
|
||||
let head = read_optional_in_pinned(&pinned, source, STORAGE_FORMAT_FILE, base, root)?;
|
||||
let state = read_optional(&source.join(STATE), base, root)?;
|
||||
let head = read_optional(&source.join(STORAGE_FORMAT_FILE), base, root)?;
|
||||
let Some(head) = head else {
|
||||
return if initial && state.is_none() {
|
||||
Ok(())
|
||||
@@ -369,9 +348,6 @@ impl SourceStore {
|
||||
.map_err(DiskError::from)?
|
||||
}
|
||||
pub async fn persist(&self, state: &SourceState) -> Result<()> {
|
||||
self.persist_with_source_sync(state, false).await
|
||||
}
|
||||
async fn persist_with_source_sync(&self, state: &SourceState, sync_source_before: bool) -> Result<()> {
|
||||
state.validate(self.disk_id, &self.path)?;
|
||||
let raw = encode(state)?;
|
||||
let source = self.source.clone();
|
||||
@@ -389,9 +365,6 @@ impl SourceStore {
|
||||
});
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
let _pin = os::pin_multipart_source_directory(&source, &base, &root)?;
|
||||
if sync_source_before {
|
||||
os::fsync_dir_std(&source)?;
|
||||
}
|
||||
if let Some(path) = terminal {
|
||||
os::replace_multipart_source_metadata(&path, &raw, &base, &root)?;
|
||||
}
|
||||
@@ -400,7 +373,6 @@ impl SourceStore {
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub async fn original(&self, state: &SourceState) -> Result<Bytes> {
|
||||
let path = original_path(&self.source, state.backup_owner);
|
||||
let base = self.base.clone();
|
||||
@@ -518,7 +490,7 @@ impl SourceStore {
|
||||
if encode(actual)? != encode(state)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let raw = loaded.original.as_ref().ok_or(DiskError::FileCorrupt)?;
|
||||
let raw = self.original(state).await?;
|
||||
let allowed = match loaded.head.as_ref() {
|
||||
None => {
|
||||
state.phase == MultipartSourcePhase::Recovering
|
||||
@@ -579,7 +551,7 @@ impl SourceStore {
|
||||
if !allowed {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let mut info = file_info(raw, RUSTFS_META_MULTIPART_BUCKET, &self.path, state.lineage.original_version_id)?;
|
||||
let mut info = file_info(&raw, RUSTFS_META_MULTIPART_BUCKET, &self.path, state.lineage.original_version_id)?;
|
||||
strip_native_controls(&mut info.metadata);
|
||||
mark_native_source(&mut info);
|
||||
set_source_status(
|
||||
@@ -800,32 +772,13 @@ pub(super) fn load_source(
|
||||
physical_slot: usize,
|
||||
disk_ids: &[Uuid],
|
||||
) -> Result<LoadedSource> {
|
||||
let pinned = match os::pin_multipart_source_directory(source, base, root) {
|
||||
Ok(pinned) => pinned,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
return Ok(LoadedSource {
|
||||
root_missing: true,
|
||||
state: None,
|
||||
head: None,
|
||||
original: None,
|
||||
info: None,
|
||||
snapshot: None,
|
||||
head_matches_state: false,
|
||||
head_matches_activation: false,
|
||||
});
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let state_raw = read_optional_in_pinned(&pinned, source, STATE, base, root)?;
|
||||
let state_raw = read_optional(&source.join(STATE), base, root)?;
|
||||
let state: Option<SourceState> = state_raw.as_deref().map(decode).transpose()?;
|
||||
if state.is_none() && os::multipart_source_has_native_controls_in_pinned(source, &pinned)? {
|
||||
if state.is_none() && os::multipart_source_has_native_controls(source, base, root)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let head = if let Some(state) = state
|
||||
.as_ref()
|
||||
.filter(|state| state.prepared_length.is_some_and(|len| len > 64 * 1024))
|
||||
{
|
||||
let head_path = source.join(STORAGE_FORMAT_FILE);
|
||||
let head_path = source.join(STORAGE_FORMAT_FILE);
|
||||
let head = if let Some(state) = state.as_ref() {
|
||||
let maximum = state
|
||||
.prepared_length
|
||||
.unwrap_or(0)
|
||||
@@ -836,8 +789,7 @@ pub(super) fn load_source(
|
||||
let raw = if is_proxy {
|
||||
None
|
||||
} else {
|
||||
let raw = read_optional_in_pinned(&pinned, source, STORAGE_FORMAT_FILE, base, root)?
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
let raw = read_optional(&head_path, base, root)?.ok_or(DiskError::FileCorrupt)?;
|
||||
if raw_sha(&raw) != sha256 {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
@@ -849,17 +801,10 @@ pub(super) fn load_source(
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
} else {
|
||||
read_optional_in_pinned(&pinned, source, STORAGE_FORMAT_FILE, base, root)?.map(|raw| {
|
||||
let sha256 = raw_sha(&raw);
|
||||
let length = raw.len() as u64;
|
||||
let is_proxy = state
|
||||
.as_ref()
|
||||
.is_some_and(|state| Some(sha256) == state.prepared_sha256 && Some(length) == state.prepared_length);
|
||||
SourceHead {
|
||||
sha256,
|
||||
length,
|
||||
raw: (!is_proxy).then_some(raw),
|
||||
}
|
||||
read_optional(&head_path, base, root)?.map(|raw| SourceHead {
|
||||
sha256: raw_sha(&raw),
|
||||
length: raw.len() as u64,
|
||||
raw: Some(raw),
|
||||
})
|
||||
};
|
||||
let mut original = None;
|
||||
@@ -1005,7 +950,6 @@ pub(super) fn load_source(
|
||||
head_matches_state = true;
|
||||
}
|
||||
Ok(LoadedSource {
|
||||
root_missing: false,
|
||||
state,
|
||||
head,
|
||||
original,
|
||||
@@ -1643,8 +1587,10 @@ impl NativeMultipartPublication {
|
||||
}
|
||||
}
|
||||
self.store.sync_target_with_lease(&self.publication, target_lease).await?;
|
||||
let source = self.store.source.clone();
|
||||
os::run_blocking_namespace_operation(self.store.lease.clone(), move || os::fsync_dir_std(&source)).await?;
|
||||
state.advance(MultipartSourcePhase::ConsumedUnsealed)?;
|
||||
self.store.persist_with_source_sync(&state, true).await?;
|
||||
self.store.persist(&state).await?;
|
||||
*self.state.lock() = state;
|
||||
let loaded = self.store.load().await?;
|
||||
let snapshot = loaded.snapshot.ok_or(DiskError::FileCorrupt)?;
|
||||
|
||||
@@ -411,23 +411,15 @@ impl LocalDisk {
|
||||
}
|
||||
let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
|
||||
if incoming.data.is_none() && !incoming.is_remote() {
|
||||
let source_data_dir = source_object.join(source_dir.to_string());
|
||||
let pinned_source_data = (!incoming.parts.is_empty())
|
||||
.then(|| os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root))
|
||||
.transpose()?;
|
||||
let mut names = Vec::with_capacity(incoming.parts.len().saturating_mul(3));
|
||||
for part in &incoming.parts {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let meta_name = format!("part.{}.meta", part.number);
|
||||
let bytes = os::read_multipart_source_metadata_in_pinned(
|
||||
pinned_source_data.as_ref().expect("pinned source data directory"),
|
||||
&source_data_dir,
|
||||
std::ffi::OsStr::new(&meta_name),
|
||||
&base,
|
||||
&publication_root,
|
||||
)?;
|
||||
let path = source_object
|
||||
.join(source_dir.to_string())
|
||||
.join(format!("part.{}.meta", part.number));
|
||||
let bytes = os::read_multipart_source_metadata(&path, &base, &publication_root)?;
|
||||
let actual = rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?;
|
||||
let normalized = i64::try_from(actual.size)
|
||||
.map_err(std::io::Error::other)?
|
||||
@@ -449,13 +441,13 @@ impl LocalDisk {
|
||||
// Preserve the native completed-MPU directory format. This
|
||||
// immutable, validated generation record is distinct from
|
||||
// the upload's mutable top-level xl.meta.
|
||||
names.push(meta_name);
|
||||
names.push(format!("part.{}.meta", part.number));
|
||||
if let Some(integrity) = &part.integrity {
|
||||
names.push(integrity.file_name());
|
||||
}
|
||||
}
|
||||
os::hardlink_multipart_source_files(
|
||||
&source_data_dir,
|
||||
&source_object.join(source_dir.to_string()),
|
||||
&private_object.join(final_dir.to_string()),
|
||||
&names,
|
||||
&base,
|
||||
|
||||
+64
-135
@@ -73,10 +73,6 @@ pub fn check_path_length(path_name: &str) -> Result<()> {
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
|
||||
if path_name.len() <= 255 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Check each path segment length is > 255 on all Unix
|
||||
// platforms, look for this value as NAME_MAX in
|
||||
// /usr/include/linux/limits.h
|
||||
@@ -3372,29 +3368,6 @@ fn open_existing_directory_below_base_std(
|
||||
use rustix::fs::{Mode, OFlags, openat};
|
||||
let relative = existing_base_directory_suffix(directory, base)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use rustix::fs::{ResolveFlags, openat2};
|
||||
use std::os::fd::AsFd;
|
||||
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
||||
let owned_base;
|
||||
let base_fd = if base == publication_root.io_path.as_path() || base == publication_root.path.as_path() {
|
||||
publication_root.directory.as_fd()
|
||||
} else {
|
||||
owned_base = std::os::fd::OwnedFd::from(publication_root.open_directory(base)?);
|
||||
owned_base.as_fd()
|
||||
};
|
||||
let target = if relative.as_os_str().is_empty() {
|
||||
Path::new(".")
|
||||
} else {
|
||||
relative
|
||||
};
|
||||
match openat2(base_fd, target, flags, Mode::empty(), ResolveFlags::BENEATH | ResolveFlags::NO_SYMLINKS) {
|
||||
Ok(leaf) => return Ok(vec![leaf]),
|
||||
Err(rustix::io::Errno::NOSYS) => {}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
let base = publication_root.open_directory(base)?.into();
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let base = {
|
||||
@@ -4948,16 +4921,12 @@ impl LinuxCommitDirectories {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
// A pinned directory may have been unlinked or replaced. Do not report a
|
||||
// successful publication into a detached object tree.
|
||||
let current = self.root.open_directory(&self.source_path)?.metadata()?;
|
||||
if (current.dev(), current.ino()) != (self.source.device, self.source.inode) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "commit directory identity changed"));
|
||||
}
|
||||
let destination = self.prepare_destination()?;
|
||||
if self.source_path != self.destination_path
|
||||
|| (self.source.device, self.source.inode) != (destination.device, destination.inode)
|
||||
{
|
||||
let current = self.root.open_directory(&self.destination_path)?.metadata()?;
|
||||
if (current.dev(), current.ino()) != (destination.device, destination.inode) {
|
||||
for (path, directory) in [
|
||||
(&self.source_path, &self.source),
|
||||
(&self.destination_path, self.prepare_destination()?),
|
||||
] {
|
||||
let current = self.root.open_directory(path)?.metadata()?;
|
||||
if (current.dev(), current.ino()) != (directory.device, directory.inode) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "commit directory identity changed"));
|
||||
}
|
||||
}
|
||||
@@ -5250,20 +5219,12 @@ pub(crate) fn prepare_rename_commit_guard(
|
||||
existing_base_directory_suffix(destination_parent, destination_base)?;
|
||||
let source = LinuxCommitDirectory::new(publication_root.open_directory(source_parent)?)?;
|
||||
let destination = std::sync::OnceLock::new();
|
||||
if source_parent == destination_parent {
|
||||
let _ = destination.set(LinuxCommitDirectory {
|
||||
file: Arc::clone(&source.file),
|
||||
device: source.device,
|
||||
inode: source.inode,
|
||||
});
|
||||
} else {
|
||||
match publication_root.open_directory(destination_parent) {
|
||||
Ok(directory) => {
|
||||
let _ = destination.set(LinuxCommitDirectory::new(directory)?);
|
||||
}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
match publication_root.open_directory(destination_parent) {
|
||||
Ok(directory) => {
|
||||
let _ = destination.set(LinuxCommitDirectory::new(directory)?);
|
||||
}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
Ok(RenameCommitGuard {
|
||||
directories: Arc::new(LinuxCommitDirectories {
|
||||
@@ -6123,20 +6084,12 @@ fn existing_base_directory_suffix<'a>(dir_path: &'a Path, base_dir: &Path) -> io
|
||||
|
||||
/// Missing state never downgrades surviving native control records to legacy.
|
||||
/// Inspect only this pinned upload root; parts and proof trees are not traversed.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn multipart_source_has_native_controls(source: &Path, base: &Path, root: &PublicationRoot) -> io::Result<bool> {
|
||||
let pin = match pin_multipart_source_directory(source, base, root) {
|
||||
let _pin = match pin_multipart_source_directory(source, base, root) {
|
||||
Ok(pin) => pin,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
multipart_source_has_native_controls_in_pinned(source, &pin)
|
||||
}
|
||||
|
||||
pub(crate) fn multipart_source_has_native_controls_in_pinned(
|
||||
source: &Path,
|
||||
_pin: &ExistingBaseDirectoryGuard,
|
||||
) -> io::Result<bool> {
|
||||
let mut entries = std::fs::read_dir(source)?;
|
||||
for _ in 0..READ_DIR_PROBE_RAW_LIMIT {
|
||||
let Some(entry) = entries.next() else {
|
||||
@@ -6440,10 +6393,15 @@ pub(crate) fn sync_multipart_owned_record_parents(
|
||||
if !directory.starts_with(source) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "native record barrier escaped source"));
|
||||
}
|
||||
let mut pinned = pin_multipart_source_directory(directory, base, publication_root)?;
|
||||
let pinned = pin_multipart_source_directory(directory, base, publication_root)?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let opened = std::fs::File::from(pinned.pop().ok_or_else(|| io::Error::other("empty native directory guard"))?);
|
||||
let opened = std::fs::File::from(
|
||||
pinned
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("empty native directory guard"))?
|
||||
.try_clone()?,
|
||||
);
|
||||
fsync_directory_handle_std(directory, Some(&opened))?;
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
@@ -6555,16 +6513,6 @@ pub(crate) fn read_multipart_source_metadata(
|
||||
read_multipart_metadata(path, base, publication_root, Some(16 * 1024 * 1024))
|
||||
}
|
||||
|
||||
pub(crate) fn read_multipart_source_metadata_in_pinned(
|
||||
pinned: &ExistingBaseDirectoryGuard,
|
||||
parent: &Path,
|
||||
name: &std::ffi::OsStr,
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
read_multipart_metadata_in_pinned(pinned, parent, name, base, publication_root, Some(16 * 1024 * 1024))
|
||||
}
|
||||
|
||||
/// Target FI histories have no source-control size limit. Keep the same
|
||||
/// pinned-parent and no-follow checks when inspecting every referenced DD.
|
||||
pub(crate) fn read_multipart_reclaim_target_metadata(
|
||||
@@ -6575,71 +6523,22 @@ pub(crate) fn read_multipart_reclaim_target_metadata(
|
||||
read_multipart_metadata(path, base, publication_root, None)
|
||||
}
|
||||
|
||||
fn read_multipart_metadata_bytes(
|
||||
mut reader: impl std::io::Read,
|
||||
size_hint: Option<u64>,
|
||||
maximum: Option<u64>,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
if let (Some(len), Some(maximum)) = (size_hint, maximum)
|
||||
&& len > maximum
|
||||
{
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart metadata exceeds its size limit"));
|
||||
}
|
||||
let cap = size_hint
|
||||
.and_then(|len| usize::try_from(len).ok())
|
||||
.map_or(0, |len| len.saturating_add(1));
|
||||
let mut bytes = Vec::with_capacity(cap);
|
||||
match (size_hint, maximum) {
|
||||
(Some(len), _) => {
|
||||
std::io::Read::read_to_end(&mut reader.take(len), &mut bytes)?;
|
||||
}
|
||||
(None, Some(maximum)) => {
|
||||
fn read_multipart_metadata_bytes(mut reader: impl std::io::Read, maximum: Option<u64>) -> io::Result<Vec<u8>> {
|
||||
let mut bytes = Vec::new();
|
||||
match maximum {
|
||||
Some(maximum) => {
|
||||
std::io::Read::read_to_end(&mut reader.take(maximum.saturating_add(1)), &mut bytes)?;
|
||||
if bytes.len() as u64 > maximum {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart metadata exceeds its size limit"));
|
||||
}
|
||||
}
|
||||
(None, None) => {
|
||||
None => {
|
||||
std::io::Read::read_to_end(&mut reader, &mut bytes)?;
|
||||
}
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn read_multipart_metadata_in_pinned(
|
||||
pinned: &ExistingBaseDirectoryGuard,
|
||||
parent: &Path,
|
||||
name: &std::ffi::OsStr,
|
||||
base: &Path,
|
||||
publication_root: &PublicationRoot,
|
||||
maximum: Option<u64>,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{Mode, OFlags, openat};
|
||||
let _ = (parent, base, publication_root);
|
||||
let directory = pinned
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("missing multipart source directory"))?;
|
||||
let mut file = std::fs::File::from(openat(
|
||||
directory,
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
let metadata = file.metadata()?;
|
||||
if !metadata.is_file() {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart metadata is not a regular file"));
|
||||
}
|
||||
read_multipart_metadata_bytes(&mut file, Some(metadata.len()), maximum)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = pinned;
|
||||
read_multipart_metadata(&parent.join(name), base, publication_root, maximum)
|
||||
}
|
||||
}
|
||||
|
||||
fn read_multipart_metadata(
|
||||
path: &Path,
|
||||
base: &Path,
|
||||
@@ -6648,14 +6547,27 @@ fn read_multipart_metadata(
|
||||
) -> io::Result<Vec<u8>> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{Mode, OFlags, openat};
|
||||
let parent = path
|
||||
.parent()
|
||||
.ok_or_else(|| io::Error::other("multipart metadata has no parent"))?;
|
||||
let guards = open_existing_directory_below_base_std(parent, base, publication_root)?;
|
||||
let directory = guards
|
||||
.last()
|
||||
.ok_or_else(|| io::Error::other("missing multipart source directory"))?;
|
||||
let name = path
|
||||
.file_name()
|
||||
.ok_or_else(|| io::Error::other("multipart metadata has no name"))?;
|
||||
let guards = open_existing_directory_below_base_std(parent, base, publication_root)?;
|
||||
read_multipart_metadata_in_pinned(&guards, parent, name, base, publication_root, maximum)
|
||||
let mut file = std::fs::File::from(openat(
|
||||
directory,
|
||||
name,
|
||||
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
if !file.metadata()?.is_file() {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart metadata is not a regular file"));
|
||||
}
|
||||
read_multipart_metadata_bytes(&mut file, maximum)
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
@@ -6686,7 +6598,7 @@ fn read_multipart_metadata(
|
||||
if windows_file_attribute_tag(&file)?.FileAttributes & (FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT) != 0 {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart metadata is not a regular file"));
|
||||
}
|
||||
read_multipart_metadata_bytes(file.as_file_mut(), None, maximum)
|
||||
read_multipart_metadata_bytes(file.as_file_mut(), maximum)
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
{
|
||||
@@ -7322,16 +7234,33 @@ pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Pat
|
||||
/// This function will not create directories under base_dir.
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
let dir_path = dir_path.as_ref();
|
||||
let base_dir = base_dir.as_ref();
|
||||
if !base_dir.as_os_str().is_empty() && base_dir.starts_with(dir_path) {
|
||||
if !base_dir.as_ref().to_string_lossy().is_empty() && base_dir.as_ref().starts_with(dir_path.as_ref()) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Err(err) = super::fs::make_dir_all(dir_path).await
|
||||
&& err.kind() != io::ErrorKind::AlreadyExists
|
||||
{
|
||||
return Err(err);
|
||||
if let Err(e) = super::fs::mkdir(dir_path.as_ref()).await {
|
||||
if e.kind() == io::ErrorKind::AlreadyExists {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if e.kind() != io::ErrorKind::NotFound {
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
if let Some(parent) = dir_path.as_ref().parent() {
|
||||
// Fall back to creating the missing parent chain only when the direct mkdir proves it is required.
|
||||
if let Err(parent_err) = super::fs::make_dir_all(parent).await
|
||||
&& parent_err.kind() != io::ErrorKind::AlreadyExists
|
||||
{
|
||||
return Err(parent_err);
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(retry_err) = super::fs::mkdir(dir_path.as_ref()).await
|
||||
&& retry_err.kind() != io::ErrorKind::AlreadyExists
|
||||
{
|
||||
return Err(retry_err);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -603,6 +603,10 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
|
||||
request.object_version_id = version_id.filter(|value| !value.is_empty()).map(str::to_string);
|
||||
// A degraded read's surviving minority is not a recreate instruction; the
|
||||
// locked storage read decides whether the target is absent (backlog#2900).
|
||||
// Reader failures include checksum damage that metadata-only healing cannot detect.
|
||||
if matches!(reason, "missing_shards" | "decode_error") {
|
||||
request.scan_mode = Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep);
|
||||
}
|
||||
|
||||
let request_id = request.id.clone();
|
||||
let bucket = bucket.to_string();
|
||||
@@ -5232,10 +5236,14 @@ impl SetDisks {
|
||||
let disks = self.get_disks_internal().await;
|
||||
let transaction_path = part_transaction_path(dst_object);
|
||||
let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
|
||||
let rollback_path = format!("{transaction_path}/{PART_TRANSACTION_ROLLBACK}");
|
||||
let current_meta_path = format!("{dst_object}.meta");
|
||||
|
||||
let reads = disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
let transaction_meta_path = transaction_meta_path.clone();
|
||||
let rollback_path = rollback_path.clone();
|
||||
let current_meta_path = current_meta_path.clone();
|
||||
async move {
|
||||
let Some(disk) = disk else {
|
||||
return PartTransactionObservation {
|
||||
@@ -5245,20 +5253,44 @@ impl SetDisks {
|
||||
err: Some(DiskError::DiskNotFound),
|
||||
};
|
||||
};
|
||||
let (transaction_meta, err) = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &transaction_meta_path).await {
|
||||
Ok(meta) => (Some(meta), None),
|
||||
Err(DiskError::FileNotFound) => (None, None),
|
||||
Err(err) => (None, Some(err)),
|
||||
let transaction_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &transaction_meta_path).await {
|
||||
Ok(meta) => Some(meta),
|
||||
Err(DiskError::FileNotFound) => None,
|
||||
Err(err) => {
|
||||
return PartTransactionObservation {
|
||||
transaction_meta: None,
|
||||
current_meta: None,
|
||||
rollback: false,
|
||||
err: Some(err),
|
||||
};
|
||||
}
|
||||
};
|
||||
let rollback = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &rollback_path).await {
|
||||
Ok(_) => true,
|
||||
Err(DiskError::FileNotFound) => false,
|
||||
Err(err) => {
|
||||
return PartTransactionObservation {
|
||||
transaction_meta,
|
||||
current_meta: None,
|
||||
rollback: false,
|
||||
err: Some(err),
|
||||
};
|
||||
}
|
||||
};
|
||||
let current_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, ¤t_meta_path).await {
|
||||
Ok(meta) => Some(meta),
|
||||
Err(DiskError::FileNotFound | DiskError::DiskNotFound) => None,
|
||||
Err(_) => None,
|
||||
};
|
||||
PartTransactionObservation {
|
||||
transaction_meta,
|
||||
current_meta: None,
|
||||
rollback: false,
|
||||
err,
|
||||
current_meta,
|
||||
rollback,
|
||||
err: None,
|
||||
}
|
||||
}
|
||||
});
|
||||
let mut observations = join_all(reads).await;
|
||||
let observations = join_all(reads).await;
|
||||
let read_errs = observations
|
||||
.iter()
|
||||
.map(|observation| observation.err.clone())
|
||||
@@ -5275,42 +5307,6 @@ impl SetDisks {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let rollback_path = format!("{transaction_path}/{PART_TRANSACTION_ROLLBACK}");
|
||||
let current_meta_path = format!("{dst_object}.meta");
|
||||
let details = disks.iter().zip(observations.iter_mut()).map(|(disk, observation)| {
|
||||
let disk = disk.clone();
|
||||
let rollback_path = rollback_path.clone();
|
||||
let current_meta_path = current_meta_path.clone();
|
||||
async move {
|
||||
if observation.err.is_some() {
|
||||
return;
|
||||
}
|
||||
let Some(disk) = disk else {
|
||||
observation.err = Some(DiskError::DiskNotFound);
|
||||
return;
|
||||
};
|
||||
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &rollback_path).await {
|
||||
Ok(_) => observation.rollback = true,
|
||||
Err(DiskError::FileNotFound) => {}
|
||||
Err(err) => {
|
||||
observation.err = Some(err);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if let Ok(meta) = disk.read_all(RUSTFS_META_MULTIPART_BUCKET, ¤t_meta_path).await {
|
||||
observation.current_meta = Some(meta);
|
||||
}
|
||||
}
|
||||
});
|
||||
join_all(details).await;
|
||||
let read_errs = observations
|
||||
.iter()
|
||||
.map(|observation| observation.err.clone())
|
||||
.collect::<Vec<_>>();
|
||||
if let Some(err) = reduce_write_quorum_errs(&read_errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let mut current_counts: HashMap<Bytes, usize> = HashMap::new();
|
||||
let mut transaction_meta_values = HashSet::new();
|
||||
for observation in observations.iter().filter(|observation| observation.err.is_none()) {
|
||||
@@ -13588,6 +13584,61 @@ mod tests {
|
||||
release_read_repair_heal_reservation(&key).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_repair_reader_failure_requests_deep_scan_for_the_exact_version() {
|
||||
fn captured() -> &'static std::sync::Mutex<Option<rustfs_heal_contracts::heal_channel::HealChannelRequest>> {
|
||||
static REQUEST: OnceLock<std::sync::Mutex<Option<rustfs_heal_contracts::heal_channel::HealChannelRequest>>> =
|
||||
OnceLock::new();
|
||||
REQUEST.get_or_init(|| std::sync::Mutex::new(None))
|
||||
}
|
||||
fn capture(request: rustfs_heal_contracts::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
|
||||
*captured().lock().expect("read-repair request capture must be available") = Some(request);
|
||||
Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Accepted) })
|
||||
}
|
||||
|
||||
for (reason, expected_scan) in [
|
||||
("missing_shards", Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep)),
|
||||
("decode_error", Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep)),
|
||||
("metadata_read_error", None),
|
||||
] {
|
||||
let object = format!("reader-failure-{}", Uuid::new_v4());
|
||||
let version = Uuid::new_v4().to_string();
|
||||
submit_read_repair_heal_with_submitter(
|
||||
ReadRepairHealSubmission {
|
||||
bucket: "reader-failure",
|
||||
object: &object,
|
||||
version_id: Some(&version),
|
||||
pool_index: 1,
|
||||
set_index: 2,
|
||||
part_number: Some(1),
|
||||
reason,
|
||||
mrf_intent: None,
|
||||
},
|
||||
capture,
|
||||
)
|
||||
.await;
|
||||
let request = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if let Some(request) = captured().lock().expect("captured request must be readable").take() {
|
||||
break request;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("the actual asynchronous admission must be captured");
|
||||
assert_eq!(request.scan_mode, expected_scan, "reader damage needs payload verification");
|
||||
assert_eq!(request.source, HealRequestSource::ReadRepair);
|
||||
assert_eq!(request.bucket, "reader-failure");
|
||||
assert_eq!(request.object_prefix.as_deref(), Some(object.as_str()));
|
||||
assert_eq!(request.object_version_id.as_deref(), Some(version.as_str()));
|
||||
assert_eq!((request.pool_index, request.set_index), (Some(1), Some(2)));
|
||||
assert_eq!(request.recreate_missing, None, "storage evidence decides absence");
|
||||
let key = ReadRepairHealCacheKey::new("reader-failure", &object, Some(&version), 1, 2);
|
||||
release_read_repair_heal_reservation(&key).await;
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// backlog#898 — groups A/C: pure old-data-dir cleanup classification.
|
||||
// ========================================================================
|
||||
|
||||
@@ -36,12 +36,13 @@ use crate::disk::{
|
||||
self, BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, Disk, DiskAPI,
|
||||
};
|
||||
use crate::set_disk::{
|
||||
DiskError, DiskStore, FileInfo, HashAlgorithm, ReadOptions, SetDisks, can_try_inline_data_shards_direct,
|
||||
codec_streaming_rollout_applies, collect_inline_data_shard_fileinfos_from_observations, file_info_is_valid_for_metadata,
|
||||
get_metadata_slowtail_fault_request, inline_erasure_shard_file_offset, inline_erasure_shard_size,
|
||||
is_get_metadata_data_read_early_stop_enabled, is_get_metadata_early_stop_bounded_fanout_enabled,
|
||||
is_get_metadata_early_stop_enabled, is_get_metadata_non_inline_data_read_early_stop_enabled, is_version_early_stop_enabled,
|
||||
object_fits_single_block,
|
||||
DiskError, DiskStore, FileInfo, HashAlgorithm, ReadOptions, SetDisks, build_inline_bitrot_readers_from_refs,
|
||||
can_try_inline_data_shards_direct, codec_streaming_rollout_applies, coding,
|
||||
collect_inline_data_shard_fileinfos_from_observations, file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request,
|
||||
inline_erasure_shard_file_offset, inline_erasure_shard_size, is_get_metadata_data_read_early_stop_enabled,
|
||||
is_get_metadata_early_stop_bounded_fanout_enabled, is_get_metadata_early_stop_enabled,
|
||||
is_get_metadata_non_inline_data_read_early_stop_enabled, is_version_early_stop_enabled, object_fits_single_block,
|
||||
try_read_inline_data_shards_direct,
|
||||
};
|
||||
use futures::future::join_all;
|
||||
use metrics::counter;
|
||||
@@ -124,7 +125,6 @@ fn metadata_metrics_path(bucket: &str) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn metadata_distribution_key(bucket: &str, object: &str) -> String {
|
||||
[bucket, object].join("/")
|
||||
}
|
||||
@@ -449,16 +449,36 @@ pub(in crate::set_disk) fn bounded_metadata_fanout_order(
|
||||
total_disks: usize,
|
||||
default_parity_count: usize,
|
||||
) -> Vec<usize> {
|
||||
let fallback_order = || (0..total_disks).collect::<Vec<_>>();
|
||||
if default_parity_count == 0 || default_parity_count >= total_disks {
|
||||
return (0..total_disks).collect();
|
||||
return fallback_order();
|
||||
}
|
||||
|
||||
let data_blocks = total_disks - default_parity_count;
|
||||
let start = FileInfo::distribution_start(bucket, object, total_disks);
|
||||
let offset = (start + 1) % total_disks;
|
||||
let distribution_key = metadata_distribution_key(bucket, object);
|
||||
let distribution = FileInfo::new(&distribution_key, data_blocks, default_parity_count)
|
||||
.erasure
|
||||
.distribution;
|
||||
if distribution.len() != total_disks {
|
||||
return fallback_order();
|
||||
}
|
||||
|
||||
let mut order = Vec::with_capacity(total_disks);
|
||||
order.extend((0..data_blocks).map(|i| (total_disks + i - offset) % total_disks));
|
||||
order.extend((0..total_disks).filter(|&disk_index| (start + disk_index + 1) % total_disks >= data_blocks));
|
||||
for block_index in 1..=data_blocks {
|
||||
let Some(disk_index) = distribution
|
||||
.iter()
|
||||
.position(|distributed_block| *distributed_block == block_index)
|
||||
else {
|
||||
return fallback_order();
|
||||
};
|
||||
order.push(disk_index);
|
||||
}
|
||||
order.extend(
|
||||
distribution
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(disk_index, block_index)| (*block_index > data_blocks).then_some(disk_index)),
|
||||
);
|
||||
order
|
||||
}
|
||||
|
||||
@@ -577,9 +597,6 @@ impl MetadataFanoutDiagnostics {
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn record(&self, path: &'static str) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
rustfs_io_metrics::record_get_object_metadata_fanout_duration(path, self.fanout_duration.as_secs_f64());
|
||||
if let Some(latency) = self.first_response_latency() {
|
||||
rustfs_io_metrics::record_get_object_first_metadata_response_latency(path, latency.as_secs_f64());
|
||||
@@ -603,9 +620,6 @@ impl MetadataFanoutDiagnostics {
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn record_quorum_candidate_latency(&self, path: &'static str, read_quorum: usize) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
if let Some(latency) = self.quorum_candidate_latency(read_quorum) {
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(path, "reached");
|
||||
rustfs_io_metrics::record_get_object_quorum_reached_latency(path, latency.as_secs_f64());
|
||||
@@ -615,14 +629,15 @@ impl MetadataFanoutDiagnostics {
|
||||
|
||||
#[cfg(test)]
|
||||
pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
candidate: &FileInfo,
|
||||
parts_metadata: &[FileInfo],
|
||||
disks: &[Option<DiskStore>],
|
||||
) -> Option<&'static str> {
|
||||
verify_inline_body_from_observations(
|
||||
_object,
|
||||
bucket,
|
||||
object,
|
||||
candidate,
|
||||
parts_metadata
|
||||
.iter()
|
||||
@@ -630,11 +645,13 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
|
||||
.map(|(index, metadata)| (index, Some(metadata))),
|
||||
disks,
|
||||
)
|
||||
.await
|
||||
.err()
|
||||
}
|
||||
|
||||
fn verify_inline_body_from_observations<'a>(
|
||||
_object: &str,
|
||||
async fn verify_inline_body_from_observations<'a>(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
candidate: &FileInfo,
|
||||
observations: impl IntoIterator<Item = (usize, Option<&'a FileInfo>)>,
|
||||
disks: &[Option<DiskStore>],
|
||||
@@ -643,18 +660,21 @@ fn verify_inline_body_from_observations<'a>(
|
||||
return Err(reason);
|
||||
}
|
||||
|
||||
let data_shards = candidate.erasure.data_blocks;
|
||||
if data_shards == 0
|
||||
|| candidate.erasure.block_size == 0
|
||||
|| (candidate.erasure.parity_blocks > 0
|
||||
&& candidate.uses_legacy_checksum
|
||||
&& !reed_solomon_simd::ReedSolomonEncoder::supports(data_shards, candidate.erasure.parity_blocks))
|
||||
{
|
||||
let Ok(erasure) = coding::Erasure::try_new_with_options(
|
||||
candidate.erasure.data_blocks,
|
||||
candidate.erasure.parity_blocks,
|
||||
candidate.erasure.block_size,
|
||||
candidate.uses_legacy_checksum,
|
||||
) else {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
|
||||
}
|
||||
let data_files = collect_inline_data_shard_fileinfos_from_observations(observations, candidate, data_shards, |index| {
|
||||
disks.get(index).is_some_and(Option::is_some)
|
||||
})?;
|
||||
};
|
||||
let data_files =
|
||||
match collect_inline_data_shard_fileinfos_from_observations(observations, candidate, erasure.data_shards, |index| {
|
||||
disks.get(index).is_some_and(Option::is_some)
|
||||
}) {
|
||||
Ok(data_files) => data_files,
|
||||
Err(reason) => return Err(reason),
|
||||
};
|
||||
|
||||
let Some(part) = candidate.parts.first() else {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
|
||||
@@ -673,51 +693,22 @@ fn verify_inline_body_from_observations<'a>(
|
||||
object_size,
|
||||
object_size,
|
||||
candidate.erasure.block_size,
|
||||
data_shards,
|
||||
erasure.data_shards,
|
||||
candidate.uses_legacy_checksum,
|
||||
);
|
||||
let shard_size = inline_erasure_shard_size(candidate.erasure.block_size, data_shards, candidate.uses_legacy_checksum);
|
||||
if object_size == 0 || read_length == 0 || read_length > shard_size || data_files.len() < data_shards {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||
}
|
||||
let shards_needed = object_size.div_ceil(read_length);
|
||||
if shards_needed > data_shards {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||
}
|
||||
let Some(encoded_capacity) = read_length.checked_mul(shards_needed) else {
|
||||
let shard_size = inline_erasure_shard_size(candidate.erasure.block_size, erasure.data_shards, candidate.uses_legacy_checksum);
|
||||
let Ok(mut readers) =
|
||||
build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
|
||||
else {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||
};
|
||||
#[cfg(test)]
|
||||
{
|
||||
for file in &data_files {
|
||||
if file.data.is_some() {
|
||||
crate::set_disk::disk_call_counters::record_shard_open(_object, true);
|
||||
}
|
||||
}
|
||||
crate::set_disk::disk_call_counters::record(_object, crate::set_disk::disk_call_counters::KIND_BODY_DECODE, 0);
|
||||
}
|
||||
|
||||
let hash_size = checksum_algo.size();
|
||||
let need = hash_size + read_length;
|
||||
let mut body = Vec::with_capacity(encoded_capacity);
|
||||
for file in data_files.iter().take(shards_needed) {
|
||||
let Some(raw) = file.data.as_deref() else {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||
};
|
||||
let Some(block) = raw.get(..need) else {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||
};
|
||||
let (hash, data) = block.split_at(hash_size);
|
||||
if hash_size > 0 && checksum_algo.hash_encode(data).as_ref() != hash {
|
||||
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
|
||||
}
|
||||
body.extend_from_slice(data);
|
||||
if body.len() >= object_size {
|
||||
body.truncate(object_size);
|
||||
return Ok(bytes::Bytes::from(body));
|
||||
}
|
||||
#[cfg(test)]
|
||||
crate::set_disk::disk_call_counters::record(object, crate::set_disk::disk_call_counters::KIND_BODY_DECODE, 0);
|
||||
match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await {
|
||||
Some(body) if body.len() == object_size => Ok(body),
|
||||
_ => Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
|
||||
}
|
||||
Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY)
|
||||
}
|
||||
|
||||
fn data_read_early_stop_inline_candidate_miss_reason(candidate: &FileInfo) -> Option<&'static str> {
|
||||
@@ -795,18 +786,16 @@ fn data_read_inline_missing_shards_are_pending(
|
||||
fanout_order: &[usize],
|
||||
scheduled_fanout_len: usize,
|
||||
) -> bool {
|
||||
let data_shards = candidate.erasure.data_blocks;
|
||||
if data_shards == 0
|
||||
|| candidate.erasure.block_size == 0
|
||||
|| !candidate.has_valid_erasure_geometry()
|
||||
|| (candidate.erasure.parity_blocks > 0
|
||||
&& candidate.uses_legacy_checksum
|
||||
&& !reed_solomon_simd::ReedSolomonEncoder::supports(data_shards, candidate.erasure.parity_blocks))
|
||||
{
|
||||
let Ok(erasure) = coding::Erasure::try_new_with_options(
|
||||
candidate.erasure.data_blocks,
|
||||
candidate.erasure.parity_blocks,
|
||||
candidate.erasure.block_size,
|
||||
candidate.uses_legacy_checksum,
|
||||
) else {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let distribution = &candidate.erasure.distribution;
|
||||
let mut data_shards_seen_or_pending = vec![false; data_shards];
|
||||
let mut data_shards_seen_or_pending = vec![false; erasure.data_shards];
|
||||
let mut missing_pending_data_shards = 0usize;
|
||||
|
||||
for slot in slots {
|
||||
@@ -814,7 +803,7 @@ fn data_read_inline_missing_shards_are_pending(
|
||||
let Some(&block_index) = distribution.get(disk_index) else {
|
||||
return false;
|
||||
};
|
||||
if block_index == 0 || block_index > data_shards {
|
||||
if block_index == 0 || block_index > erasure.data_shards {
|
||||
continue;
|
||||
}
|
||||
if !disks.get(disk_index).is_some_and(Option::is_some) {
|
||||
@@ -1181,8 +1170,7 @@ impl SetDisks {
|
||||
default_parity_count: usize,
|
||||
allow_coalescing: bool,
|
||||
) -> disk::error::Result<MetadataReadResult> {
|
||||
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
let fanout_start = stage_metrics_enabled.then(Instant::now);
|
||||
let fanout_start = Instant::now();
|
||||
let mut slots = MetadataReadResult::pending(disks.len());
|
||||
let mut task_ids = vec![None; disks.len()];
|
||||
let mut observations = Vec::with_capacity(disks.len());
|
||||
@@ -1232,7 +1220,7 @@ impl SetDisks {
|
||||
let diagnostic_context = diagnostic_context;
|
||||
let task = join_set.spawn(async move {
|
||||
let read = async move {
|
||||
let response_start = stage_metrics_enabled.then(Instant::now);
|
||||
let response_start = Instant::now();
|
||||
let result = if let Some(disk) = disk {
|
||||
#[cfg(test)]
|
||||
let _fanout_task_guard = rename_fanout_barrier::task_guard(&object);
|
||||
@@ -1255,7 +1243,7 @@ impl SetDisks {
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
};
|
||||
(index, result, response_start.map_or(Duration::ZERO, |s| s.elapsed()))
|
||||
(index, result, response_start.elapsed())
|
||||
};
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
return crate::cluster::rpc::runtime_sources::with_read_version_diagnostic_context(
|
||||
@@ -1360,10 +1348,7 @@ impl SetDisks {
|
||||
},
|
||||
Err(join_error) => {
|
||||
let err = DiskError::Unexpected;
|
||||
observations.push(MetadataFanoutObservation::from_error(
|
||||
&err,
|
||||
fanout_start.map_or(Duration::ZERO, |s| s.elapsed()),
|
||||
));
|
||||
observations.push(MetadataFanoutObservation::from_error(&err, fanout_start.elapsed()));
|
||||
if let Some(index) = task_ids.iter().position(|id| *id == Some(join_error.id())) {
|
||||
slots[index].result = Some(Err(err));
|
||||
accumulator.observe(&slots[index]);
|
||||
@@ -1399,11 +1384,14 @@ impl SetDisks {
|
||||
accumulator.candidate_has_read_reserve()
|
||||
}
|
||||
Some(candidate) => match verify_inline_body_from_observations(
|
||||
bucket.as_ref(),
|
||||
object.as_ref(),
|
||||
candidate,
|
||||
slots.iter().map(|slot| (slot.disk_index, slot.file_info())),
|
||||
disks,
|
||||
) {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(body) => {
|
||||
verified_inline_body = Some(body);
|
||||
true
|
||||
@@ -1461,8 +1449,7 @@ impl SetDisks {
|
||||
scheduled_count.saturating_sub(cancelled_count),
|
||||
cancelled_count,
|
||||
);
|
||||
let diagnostics =
|
||||
MetadataFanoutDiagnostics::new(fanout_start.map_or(Duration::ZERO, |s| s.elapsed()), observations);
|
||||
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
|
||||
return Ok(MetadataReadResult {
|
||||
slots,
|
||||
diagnostics,
|
||||
@@ -1545,7 +1532,7 @@ impl SetDisks {
|
||||
rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, final_miss_reason);
|
||||
rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0);
|
||||
rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0);
|
||||
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.map_or(Duration::ZERO, |s| s.elapsed()), observations);
|
||||
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
|
||||
Ok(MetadataReadResult {
|
||||
slots,
|
||||
diagnostics,
|
||||
|
||||
@@ -275,22 +275,6 @@ impl SetDisks {
|
||||
|
||||
pub(super) fn common_parity(parities: &[i32], default_parity_count: i32) -> i32 {
|
||||
let n = parities.len() as i32;
|
||||
let mut valid_parities = parities.iter().copied().filter(|&p| p != -1);
|
||||
let Some(first_parity) = valid_parities.next() else {
|
||||
return -1;
|
||||
};
|
||||
let mut same_count = 1i32;
|
||||
if valid_parities.all(|p| {
|
||||
same_count += 1;
|
||||
p == first_parity
|
||||
}) {
|
||||
let read_quorum = if default_parity_count > 0 && first_parity == 0 {
|
||||
n / 2 + 1
|
||||
} else {
|
||||
n - first_parity
|
||||
};
|
||||
return if same_count >= read_quorum { first_parity } else { -1 };
|
||||
}
|
||||
|
||||
let mut occ_map: HashMap<i32, i32> = HashMap::new();
|
||||
for &p in parities {
|
||||
@@ -349,19 +333,6 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
pub(super) fn common_time_and_occurrence(times: &[Option<OffsetDateTime>]) -> (Option<OffsetDateTime>, usize) {
|
||||
let mut valid_times = times.iter().flatten().copied();
|
||||
let Some(first_time) = valid_times.next() else {
|
||||
return (None, 0);
|
||||
};
|
||||
let mut count = 1usize;
|
||||
if valid_times.all(|t| {
|
||||
count += 1;
|
||||
t == first_time
|
||||
}) && first_time.unix_timestamp_nanos() != 0
|
||||
{
|
||||
return (Some(first_time), count);
|
||||
}
|
||||
|
||||
let mut time_occurrence_map = HashMap::new();
|
||||
|
||||
// Ignore the uuid sentinel and count the rest.
|
||||
@@ -628,25 +599,6 @@ impl SetDisks {
|
||||
return write_quorum;
|
||||
}
|
||||
|
||||
if let Some((first, _)) = parts_metadata
|
||||
.iter()
|
||||
.zip(errs.iter())
|
||||
.find(|(meta, err)| err.is_none() && file_info_is_valid_for_metadata(meta))
|
||||
{
|
||||
let same_count = parts_metadata
|
||||
.iter()
|
||||
.zip(errs.iter())
|
||||
.filter(|(meta, err)| {
|
||||
err.is_none()
|
||||
&& file_info_is_valid_for_metadata(meta)
|
||||
&& super::core::metadata_quorum::metadata_early_stop_candidate_matches(first, meta)
|
||||
})
|
||||
.count();
|
||||
if same_count >= write_quorum {
|
||||
return write_quorum;
|
||||
}
|
||||
}
|
||||
|
||||
let mut identity_counts = HashMap::with_capacity(usable_metadata);
|
||||
for (meta, err) in parts_metadata.iter().zip(errs.iter()) {
|
||||
if err.is_some() || !file_info_is_valid_for_metadata(meta) {
|
||||
@@ -1030,33 +982,6 @@ impl SetDisks {
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
let matches_time_or_etag = |meta: &FileInfo| {
|
||||
mod_time == &meta.mod_time
|
||||
|| (mod_time.is_none() && etag.as_deref().is_some_and(|e| meta.get_etag().as_deref() == Some(e)))
|
||||
};
|
||||
|
||||
if let Some(first) = metas
|
||||
.iter()
|
||||
.find(|meta| file_info_is_valid_for_metadata(meta) && matches_time_or_etag(meta))
|
||||
{
|
||||
let mut total_valid = 0usize;
|
||||
let mut same_count = 0usize;
|
||||
for meta in metas {
|
||||
if !file_info_is_valid_for_metadata(meta) || !matches_time_or_etag(meta) {
|
||||
continue;
|
||||
}
|
||||
total_valid += 1;
|
||||
if super::core::metadata_quorum::metadata_early_stop_candidate_matches(first, meta) {
|
||||
same_count += 1;
|
||||
}
|
||||
}
|
||||
if same_count >= quorum && same_count > total_valid - same_count {
|
||||
let mut fi = first.clone();
|
||||
fi.is_latest = fi.successor_mod_time.is_none();
|
||||
return Ok(fi);
|
||||
}
|
||||
}
|
||||
|
||||
let mut meta_hashes = vec![None; metas.len()];
|
||||
|
||||
for (i, meta) in metas.iter().enumerate() {
|
||||
|
||||
+150
-107
@@ -129,7 +129,7 @@ use rustfs_filemeta::{
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::{
|
||||
DriveState, HealAdmissionResult, HealChannelPriority, HealItemType, HealOpts, HealRequestSource, HealScanMode,
|
||||
send_heal_request_with_admission,
|
||||
send_heal_replacement_disk, send_heal_request_with_admission,
|
||||
};
|
||||
use rustfs_io_metrics::{
|
||||
record_object_lock_diag_acquire_duration, record_object_lock_diag_enabled, record_object_lock_diag_hold_duration,
|
||||
@@ -160,7 +160,7 @@ use rustfs_utils::http::{
|
||||
use rustfs_utils::{
|
||||
HashAlgorithm,
|
||||
crypto::hex,
|
||||
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, object_name_disk_identity, path_join_buf},
|
||||
path::{SLASH_SEPARATOR, encode_dir_object, encode_dir_object_ref, has_suffix, object_name_disk_identity, path_join_buf},
|
||||
};
|
||||
use sha2::Sha256;
|
||||
use std::borrow::Cow;
|
||||
@@ -889,6 +889,8 @@ pub(crate) use ops::heal::AbsenceProofRequest;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::heal::DanglingDeleteFailure;
|
||||
pub(crate) use ops::heal::HealedObjectAbsence;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::heal::ReadRepairPauseScope;
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::hermetic_set_disks_isolated;
|
||||
@@ -934,20 +936,12 @@ pub(in crate::set_disk) struct GetObjectFileInfo {
|
||||
}
|
||||
|
||||
struct OwnedGetObjectFileInfo {
|
||||
payload: OwnedGetObjectFileInfoPayload,
|
||||
fi: FileInfo,
|
||||
parts_metadata: Vec<FileInfo>,
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
verified_inline_body: Option<Bytes>,
|
||||
}
|
||||
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
enum OwnedGetObjectFileInfoPayload {
|
||||
Direct {
|
||||
fi: FileInfo,
|
||||
parts_metadata: Vec<FileInfo>,
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
},
|
||||
Published(Arc<GetObjectMetadataCacheEntry>),
|
||||
}
|
||||
|
||||
enum LateMetadataFanout {
|
||||
CapturedDisks(Vec<Option<DiskStore>>),
|
||||
RefreshCurrentSet,
|
||||
@@ -959,11 +953,9 @@ impl GetObjectFileInfo {
|
||||
fn owned(fi: FileInfo, parts_metadata: Vec<FileInfo>, online_disks: Vec<Option<DiskStore>>) -> Self {
|
||||
Self {
|
||||
owned: Some(OwnedGetObjectFileInfo {
|
||||
payload: OwnedGetObjectFileInfoPayload::Direct {
|
||||
fi,
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
},
|
||||
fi,
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
verified_inline_body: None,
|
||||
}),
|
||||
shared: None,
|
||||
@@ -972,21 +964,15 @@ impl GetObjectFileInfo {
|
||||
}
|
||||
}
|
||||
|
||||
fn owned_published(entry: Arc<GetObjectMetadataCacheEntry>) -> Self {
|
||||
let condition_stopped = entry.capability == GetObjectMetadataCacheCapability::ConditionOnly;
|
||||
Self {
|
||||
owned: Some(OwnedGetObjectFileInfo {
|
||||
payload: OwnedGetObjectFileInfoPayload::Published(entry),
|
||||
verified_inline_body: None,
|
||||
}),
|
||||
shared: None,
|
||||
condition_stopped,
|
||||
late_metadata_fanout: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn set_late_metadata_fanout_disks(&mut self, late_metadata_fanout_disks: Vec<Option<DiskStore>>) {
|
||||
self.late_metadata_fanout = Some(LateMetadataFanout::CapturedDisks(late_metadata_fanout_disks));
|
||||
fn owned_with_late_metadata_fanout(
|
||||
fi: FileInfo,
|
||||
parts_metadata: Vec<FileInfo>,
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
late_metadata_fanout_disks: Vec<Option<DiskStore>>,
|
||||
) -> Self {
|
||||
let mut snapshot = Self::owned(fi, parts_metadata, online_disks);
|
||||
snapshot.late_metadata_fanout = Some(LateMetadataFanout::CapturedDisks(late_metadata_fanout_disks));
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn shared(entry: Arc<GetObjectMetadataCacheEntry>) -> Self {
|
||||
@@ -1005,31 +991,28 @@ impl GetObjectFileInfo {
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn as_parts(&self) -> (&FileInfo, &[FileInfo], &[Option<DiskStore>]) {
|
||||
fn fi(&self) -> &FileInfo {
|
||||
match (&self.owned, &self.shared) {
|
||||
(Some(snapshot), None) => match &snapshot.payload {
|
||||
OwnedGetObjectFileInfoPayload::Direct {
|
||||
fi,
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
} => (fi, parts_metadata, online_disks),
|
||||
OwnedGetObjectFileInfoPayload::Published(entry) => (&entry.fi, &entry.parts_metadata, &entry.online_disks),
|
||||
},
|
||||
(None, Some(entry)) => (&entry.fi, &entry.parts_metadata, &entry.online_disks),
|
||||
(Some(snapshot), None) => &snapshot.fi,
|
||||
(None, Some(entry)) => &entry.fi,
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
}
|
||||
}
|
||||
|
||||
fn fi(&self) -> &FileInfo {
|
||||
self.as_parts().0
|
||||
}
|
||||
|
||||
fn parts_metadata(&self) -> &[FileInfo] {
|
||||
self.as_parts().1
|
||||
match (&self.owned, &self.shared) {
|
||||
(Some(snapshot), None) => &snapshot.parts_metadata,
|
||||
(None, Some(entry)) => &entry.parts_metadata,
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
}
|
||||
}
|
||||
|
||||
fn online_disks(&self) -> &[Option<DiskStore>] {
|
||||
self.as_parts().2
|
||||
match (&self.owned, &self.shared) {
|
||||
(Some(snapshot), None) => &snapshot.online_disks,
|
||||
(None, Some(entry)) => &entry.online_disks,
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
}
|
||||
}
|
||||
|
||||
fn has_late_metadata_fanout(&self) -> bool {
|
||||
@@ -1043,28 +1026,17 @@ impl GetObjectFileInfo {
|
||||
}
|
||||
|
||||
fn take_verified_inline_body(&mut self) -> Option<Bytes> {
|
||||
match (&mut self.owned, &self.shared) {
|
||||
(Some(owned), None) => owned.verified_inline_body.take().or_else(|| match &owned.payload {
|
||||
OwnedGetObjectFileInfoPayload::Direct { .. } => None,
|
||||
OwnedGetObjectFileInfoPayload::Published(entry) => entry.verified_inline_body.clone(),
|
||||
}),
|
||||
(None, Some(entry)) => entry.verified_inline_body.clone(),
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
}
|
||||
}
|
||||
|
||||
fn into_payload(self) -> (OwnedGetObjectFileInfoPayload, Option<LateMetadataFanout>) {
|
||||
match (self.owned, self.shared) {
|
||||
(Some(snapshot), None) => (snapshot.payload, self.late_metadata_fanout),
|
||||
(None, Some(entry)) => (OwnedGetObjectFileInfoPayload::Published(entry), self.late_metadata_fanout),
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
}
|
||||
self.owned.as_mut().and_then(|owned| owned.verified_inline_body.take())
|
||||
}
|
||||
|
||||
fn into_object_info(self, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
|
||||
match self.into_payload().0 {
|
||||
OwnedGetObjectFileInfoPayload::Direct { fi, .. } => ObjectInfo::from_owned_file_info(fi, bucket, object, versioned),
|
||||
OwnedGetObjectFileInfoPayload::Published(entry) => ObjectInfo::from_file_info(&entry.fi, bucket, object, versioned),
|
||||
match (self.owned, self.shared) {
|
||||
(Some(snapshot), None) => ObjectInfo::from_owned_file_info(snapshot.fi, bucket, object, versioned),
|
||||
(None, Some(entry)) => match Arc::try_unwrap(entry) {
|
||||
Ok(entry) => ObjectInfo::from_owned_file_info(entry.fi, bucket, object, versioned),
|
||||
Err(entry) => ObjectInfo::from_file_info(&entry.fi, bucket, object, versioned),
|
||||
},
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1074,17 +1046,24 @@ impl GetObjectFileInfo {
|
||||
}
|
||||
|
||||
fn into_owned_with_late_metadata_fanout(self) -> OwnedGetObjectFileInfoParts {
|
||||
let (payload, late_metadata_fanout) = self.into_payload();
|
||||
match payload {
|
||||
OwnedGetObjectFileInfoPayload::Direct {
|
||||
fi,
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
} => (fi, parts_metadata, online_disks, late_metadata_fanout),
|
||||
OwnedGetObjectFileInfoPayload::Published(entry) => {
|
||||
let entry = Arc::unwrap_or_clone(entry);
|
||||
(entry.fi, entry.parts_metadata, entry.online_disks, late_metadata_fanout)
|
||||
}
|
||||
let Self {
|
||||
owned,
|
||||
shared,
|
||||
late_metadata_fanout,
|
||||
..
|
||||
} = self;
|
||||
match (owned, shared) {
|
||||
(Some(snapshot), None) => (snapshot.fi, snapshot.parts_metadata, snapshot.online_disks, late_metadata_fanout),
|
||||
(None, Some(entry)) => match Arc::try_unwrap(entry) {
|
||||
Ok(entry) => (entry.fi, entry.parts_metadata, entry.online_disks, late_metadata_fanout),
|
||||
Err(entry) => (
|
||||
entry.fi.clone(),
|
||||
entry.parts_metadata.clone(),
|
||||
entry.online_disks.clone(),
|
||||
late_metadata_fanout,
|
||||
),
|
||||
},
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1469,7 +1448,6 @@ mod prepared_get_object_metadata_tests {
|
||||
fi,
|
||||
online_disks: vec![None],
|
||||
read_quorum: 0,
|
||||
verified_inline_body: None,
|
||||
});
|
||||
let snapshot = GetObjectFileInfo::shared(Arc::clone(&cached));
|
||||
|
||||
@@ -2786,9 +2764,7 @@ impl SetDisks {
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PreparedGetObjectMetadata> {
|
||||
let snapshot = self
|
||||
.get_object_fileinfo_for_object_read(bucket, object, opts, true, false)
|
||||
.await?;
|
||||
let snapshot = self.get_object_fileinfo(bucket, object, opts, true, false).await?;
|
||||
let object_info = build_get_object_info(snapshot.fi(), bucket, object, opts.versioned || opts.version_suspended);
|
||||
Ok(PreparedGetObjectMetadata {
|
||||
snapshot,
|
||||
@@ -4927,7 +4903,7 @@ enum GetObjectMetadataCacheCapability {
|
||||
ConditionOnly,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
#[derive(Debug)]
|
||||
struct GetObjectMetadataCacheEntry {
|
||||
#[allow(dead_code)] // Kept for debugging; moka handles TTL internally
|
||||
created_at: Instant,
|
||||
@@ -4936,7 +4912,6 @@ struct GetObjectMetadataCacheEntry {
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
read_quorum: usize,
|
||||
capability: GetObjectMetadataCacheCapability,
|
||||
verified_inline_body: Option<Bytes>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -5247,6 +5222,40 @@ impl SetDisks {
|
||||
.with_attempt(attempt))
|
||||
}
|
||||
|
||||
pub(crate) async fn read_listing_metadata_after_namespace_barrier(
|
||||
&self,
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<(MetaCacheEntries, usize)> {
|
||||
// This is only called after the lock-free LIST fast path observes an
|
||||
// unresolved metadata generation. Waiting on the ordinary object read
|
||||
// lock establishes a publication boundary with concurrent overwrites.
|
||||
let stored_object = encode_dir_object_ref(object);
|
||||
let _guard = self
|
||||
.acquire_read_lock_diag("list_object_reconcile", bucket, stored_object.as_ref())
|
||||
.await?;
|
||||
let (raw_entries, errors) = Self::read_all_raw_file_info(disks, bucket, stored_object.as_ref(), false).await;
|
||||
let confirmed_absent = errors
|
||||
.iter()
|
||||
.flatten()
|
||||
.filter(|err| DiskError::is_err_object_not_found(err) || DiskError::is_err_version_not_found(err))
|
||||
.count();
|
||||
let entries = raw_entries
|
||||
.into_iter()
|
||||
.map(|raw| {
|
||||
raw.filter(|raw| !raw.buf.is_empty()).map(|raw| MetaCacheEntry {
|
||||
name: object.to_owned(),
|
||||
metadata: raw.buf,
|
||||
cached: None,
|
||||
reusable: false,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok((MetaCacheEntries(entries), confirmed_absent))
|
||||
}
|
||||
|
||||
async fn acquire_write_lock_diag(&self, op: &'static str, bucket: &str, object: &str) -> Result<ObjectLockDiagGuard> {
|
||||
crate::hp_guard!("SetDisks::acquire_write_lock");
|
||||
let diag_enabled = is_object_lock_diag_enabled();
|
||||
@@ -8807,31 +8816,17 @@ mod tests {
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
|
||||
runtime.block_on(async {
|
||||
let healthy: Arc<dyn LockClient> =
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let failing: Arc<dyn LockClient> = Arc::new(FailingClient);
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
|
||||
// A real acquire can time out before the failing client is scheduled.
|
||||
let err = resolve_put_object_commit_lock_acquire_result(
|
||||
&set,
|
||||
"put_object_commit",
|
||||
"bucket",
|
||||
"object",
|
||||
Err(LockError::QuorumNotReached {
|
||||
required: 2,
|
||||
achieved: 1,
|
||||
}),
|
||||
)
|
||||
.expect_err("a quorum failure must remain a lock error");
|
||||
let set = make_test_set_disks_with_ctx(vec![healthy, failing], ctx).await;
|
||||
assert!(
|
||||
matches!(
|
||||
err,
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
required: 2,
|
||||
achieved: 1,
|
||||
..
|
||||
}
|
||||
),
|
||||
"expected the typed quorum error, got: {err:?}"
|
||||
set.acquire_write_lock_diag("put_object_commit", "bucket", "object")
|
||||
.await
|
||||
.is_err(),
|
||||
"one healthy locker must not satisfy the PUT commit write quorum"
|
||||
);
|
||||
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
|
||||
});
|
||||
@@ -16443,6 +16438,54 @@ mod tests {
|
||||
assert_eq!(err, StorageError::PreconditionFailed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn listing_metadata_reconcile_encodes_directory_object_and_waits_for_writer() {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
let bucket = "bucket-list-reconcile-lock";
|
||||
let object = "directory/";
|
||||
let stored_object = encode_dir_object(object);
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(b"directory marker".to_vec());
|
||||
set_disks
|
||||
.put_object(bucket, &stored_object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("directory object should be written under its encoded key");
|
||||
let disks = set_disks.disks.read().await.clone();
|
||||
let namespace_lock = set_disks
|
||||
.new_ns_lock(bucket, &stored_object)
|
||||
.await
|
||||
.expect("namespace lock should be created");
|
||||
let writer_guard = namespace_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(30))
|
||||
.await
|
||||
.expect("writer lock should be acquired");
|
||||
let read = set_disks.read_listing_metadata_after_namespace_barrier(&disks, bucket, object);
|
||||
tokio::pin!(read);
|
||||
assert!(
|
||||
tokio::time::timeout(std::time::Duration::from_millis(50), &mut read)
|
||||
.await
|
||||
.is_err(),
|
||||
"LIST reconciliation must wait for the publishing writer"
|
||||
);
|
||||
drop(writer_guard);
|
||||
|
||||
let (entries, confirmed_absent) = tokio::time::timeout(std::time::Duration::from_secs(30), read)
|
||||
.await
|
||||
.expect("LIST reconciliation should resume after writer release")
|
||||
.expect("LIST reconciliation should read after the namespace barrier");
|
||||
assert_eq!(entries.0.len(), disks.len());
|
||||
assert!(
|
||||
entries.0.iter().all(|entry| entry
|
||||
.as_ref()
|
||||
.is_some_and(|entry| entry.name == object && !entry.metadata.is_empty())),
|
||||
"LIST reconciliation must read encoded metadata and retain the logical key"
|
||||
);
|
||||
assert_eq!(confirmed_absent, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repeated_body_write_keeps_etag_but_changes_data_dir_generation() {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -21,22 +21,17 @@
|
||||
|
||||
use super::super::{
|
||||
Arc, DiskError, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, Endpoint, Error, FormatV3, HealChannelPriority, LockResult,
|
||||
NamespaceLock, NamespaceLockWrapper, ObjectKey, Result, SetDisks, StorageError, debug, disk, load_format_erasure, warn,
|
||||
NamespaceLock, NamespaceLockWrapper, ObjectKey, Result, SetDisks, StorageError, debug, disk, info, load_format_erasure,
|
||||
send_heal_replacement_disk, warn,
|
||||
};
|
||||
use crate::disk::health_state::DriveMembershipSnapshot;
|
||||
use crate::disk::{DiskAPI, new_disk};
|
||||
use crate::layout::disks_layout::MAX_ERASURE_SET_DRIVE_COUNT;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use futures::{StreamExt, stream, stream::FuturesUnordered};
|
||||
use rand::prelude::SliceRandom;
|
||||
use rustfs_heal_contracts::heal_channel::{create_auto_replacement_disks_request, send_heal_request};
|
||||
use rustfs_utils::path::object_name_disk_identity;
|
||||
#[cfg(test)]
|
||||
use uuid::Uuid;
|
||||
|
||||
// Bound batching when another reconnect is still pending.
|
||||
const REPLACEMENT_BATCH_WAIT: std::time::Duration = std::time::Duration::from_secs(1);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
|
||||
type Error = Error;
|
||||
@@ -331,83 +326,38 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
async fn connect_disks_with_active_monitoring(&self, active_monitoring: bool) {
|
||||
let disks = self.get_disks_internal().await;
|
||||
// Reconnect drives concurrently without holding the inventory lock.
|
||||
let mut reconnects = stream::iter(disks.into_iter().enumerate())
|
||||
.map(|(i, opdisk)| async move {
|
||||
if let Some(disk) = opdisk {
|
||||
if !active_monitoring
|
||||
&& disk.get_disk_location().set_idx.is_some()
|
||||
&& let disk::Disk::Local(local) = disk.as_ref()
|
||||
&& local.can_skip_background_probe().await
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if disk.is_online().await && disk.get_disk_location().set_idx.is_some() {
|
||||
return None;
|
||||
}
|
||||
let rl = self.disks.read().await;
|
||||
|
||||
let _ = disk.close().await;
|
||||
let disks = rl.clone();
|
||||
|
||||
// Explicitly release the lock
|
||||
drop(rl);
|
||||
|
||||
for (i, opdisk) in disks.iter().enumerate() {
|
||||
if let Some(disk) = opdisk {
|
||||
if !active_monitoring
|
||||
&& disk.get_disk_location().set_idx.is_some()
|
||||
&& let disk::Disk::Local(local) = disk.as_ref()
|
||||
&& local.can_skip_background_probe().await
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if disk.is_online().await && disk.get_disk_location().set_idx.is_some() {
|
||||
info!("Disk {:?} is online", disk.to_string());
|
||||
continue;
|
||||
}
|
||||
|
||||
let endpoint = self.set_endpoints.get(i)?;
|
||||
self.reconnect_disk(endpoint).await.then(|| endpoint.to_string())
|
||||
})
|
||||
.buffer_unordered(MAX_ERASURE_SET_DRIVE_COUNT);
|
||||
let mut submissions = FuturesUnordered::new();
|
||||
let mut batch = Vec::new();
|
||||
let mut deadline = None;
|
||||
let mut reconnects_done = false;
|
||||
loop {
|
||||
if reconnects_done && submissions.is_empty() {
|
||||
break;
|
||||
let _ = disk.close().await;
|
||||
}
|
||||
tokio::select! {
|
||||
result = reconnects.next(), if !reconnects_done => match result {
|
||||
Some(Some(endpoint)) => {
|
||||
batch.push(endpoint);
|
||||
deadline.get_or_insert_with(|| tokio::time::Instant::now() + REPLACEMENT_BATCH_WAIT);
|
||||
}
|
||||
Some(None) => {}
|
||||
None => {
|
||||
reconnects_done = true;
|
||||
if !batch.is_empty() {
|
||||
submissions.push(self.submit_replacement_heal(std::mem::take(&mut batch)));
|
||||
}
|
||||
deadline = None;
|
||||
}
|
||||
},
|
||||
_ = tokio::time::sleep_until(deadline.unwrap_or_else(tokio::time::Instant::now)), if deadline.is_some() => {
|
||||
submissions.push(self.submit_replacement_heal(std::mem::take(&mut batch)));
|
||||
deadline = None;
|
||||
}
|
||||
_ = submissions.next(), if !submissions.is_empty() => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn submit_replacement_heal(&self, replacements: Vec<String>) {
|
||||
if replacements.is_empty() {
|
||||
return;
|
||||
if let Some(endpoint) = self.set_endpoints.get(i) {
|
||||
info!("will renew disk, opdisk: {:?}", opdisk);
|
||||
self.renew_disk(endpoint).await;
|
||||
}
|
||||
}
|
||||
// Queue admission deduplicates by set, so publish ready targets together.
|
||||
let request = create_auto_replacement_disks_request(
|
||||
self.pool_index,
|
||||
self.set_index,
|
||||
replacements,
|
||||
Some(HealChannelPriority::Normal),
|
||||
);
|
||||
let _ = send_heal_request(request).await;
|
||||
}
|
||||
|
||||
pub async fn renew_disk(&self, ep: &Endpoint) {
|
||||
if self.reconnect_disk(ep).await {
|
||||
self.submit_replacement_heal(vec![ep.to_string()]).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Return whether this reconnect attached a verified unformatted target.
|
||||
async fn reconnect_disk(&self, ep: &Endpoint) -> bool {
|
||||
debug!("renew_disk: start {:?}", ep);
|
||||
|
||||
let previous_disk = {
|
||||
@@ -427,17 +377,28 @@ impl SetDisks {
|
||||
Err(e) => {
|
||||
warn!("renew_disk: connect_endpoint err {:?}", &e);
|
||||
if !matches!(e, DiskError::UnformattedDisk | DiskError::Io(_)) {
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
|
||||
let attached = match self.attach_unformatted_replacement_disk(ep).await {
|
||||
Ok(attached) => attached,
|
||||
Err(err) => {
|
||||
warn!(endpoint = %ep, error = ?err, "renew_disk: unformatted replacement probe failed");
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
};
|
||||
return attached;
|
||||
if !attached {
|
||||
return;
|
||||
}
|
||||
|
||||
info!("renew_disk attached unformatted replacement and will trigger heal_disk, {:?}", ep);
|
||||
let (Ok(pool_index), Ok(set_index)) = (usize::try_from(ep.pool_idx), usize::try_from(ep.set_idx)) else {
|
||||
warn!("renew_disk: replacement target has invalid pool or set index, {:?}", ep);
|
||||
return;
|
||||
};
|
||||
let _ =
|
||||
send_heal_replacement_disk(pool_index, set_index, ep.to_string(), Some(HealChannelPriority::Normal)).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -445,7 +406,7 @@ impl SetDisks {
|
||||
Ok(res) => res,
|
||||
Err(e) => {
|
||||
warn!("renew_disk: find_disk_index err {:?}", e);
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -463,7 +424,7 @@ impl SetDisks {
|
||||
expected_set_index = self.set_index,
|
||||
"renew_disk rejected a drive whose endpoint and format do not identify the same topology slot"
|
||||
);
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
|
||||
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
|
||||
@@ -483,7 +444,7 @@ impl SetDisks {
|
||||
new_disk.enable_health_check();
|
||||
*slot = Some(new_disk);
|
||||
}
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
|
||||
new_disk.enable_health_check();
|
||||
@@ -505,7 +466,6 @@ impl SetDisks {
|
||||
|
||||
let mut disk_lock = self.disks.write().await;
|
||||
disk_lock[disk_idx] = Some(new_disk);
|
||||
false
|
||||
}
|
||||
|
||||
/// Attach a replacement target only after proving that the exact local slot
|
||||
@@ -767,6 +727,10 @@ mod tests {
|
||||
let format = FormatV3::new(1, 1);
|
||||
let (_dir, endpoint, disk) = make_formatted_local_disk(0, &format).await;
|
||||
assert!(disk.is_online().await, "initialize the expected disk identity");
|
||||
let disk::Disk::Local(wrapper) = disk.as_ref() else {
|
||||
panic!("fixture must use a local disk");
|
||||
};
|
||||
let local = wrapper.get_disk();
|
||||
let set = SetDisks::new(
|
||||
"quiet-monitor".to_owned(),
|
||||
Arc::new(RwLock::new(vec![Some(disk.clone())])),
|
||||
@@ -785,136 +749,13 @@ mod tests {
|
||||
(false, RuntimeDriveHealthState::Offline),
|
||||
(false, RuntimeDriveHealthState::Returning),
|
||||
] {
|
||||
let disk = set.get_disks_internal().await[0].clone().expect("current drive");
|
||||
let disk::Disk::Local(wrapper) = disk.as_ref() else {
|
||||
panic!("fixture must use a local disk");
|
||||
};
|
||||
let local = wrapper.get_disk();
|
||||
disk.force_runtime_state_for_test(state);
|
||||
local.format_info.write().await.last_check = None;
|
||||
set.connect_disks_with_active_monitoring(active).await;
|
||||
let current = set.get_disks_internal().await[0].clone().expect("probed or renewed drive");
|
||||
let disk::Disk::Local(wrapper) = current.as_ref() else {
|
||||
panic!("fixture must use a local disk");
|
||||
};
|
||||
assert!(
|
||||
wrapper.get_disk().format_info.read().await.last_check.is_some(),
|
||||
local.format_info.read().await.last_check.is_some(),
|
||||
"probe required for active={active}, state={state:?}"
|
||||
);
|
||||
current
|
||||
.disk_info(&DiskInfoOptions::default())
|
||||
.await
|
||||
.expect("the probed or renewed drive must admit real I/O");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_reconnect_restores_ec4_io_without_all_disks_returning() {
|
||||
use crate::object_api::{ObjectOptions, PutObjReader};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::object::ObjectIO;
|
||||
use disk::health_state::RuntimeDriveHealthState;
|
||||
use http::HeaderMap;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
for active_monitoring in [true, false] {
|
||||
let format = FormatV3::new(1, 16);
|
||||
let mut dirs = Vec::new();
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for index in 0..16 {
|
||||
let (dir, endpoint, disk) = make_formatted_local_disk(index, &format).await;
|
||||
assert!(disk.is_online().await, "initialize the original drive identity");
|
||||
dirs.push(dir);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
let set = SetDisks::new(
|
||||
"partial-reconnect".to_owned(),
|
||||
Arc::new(RwLock::new(disks.clone())),
|
||||
16,
|
||||
4,
|
||||
0,
|
||||
0,
|
||||
endpoints,
|
||||
format,
|
||||
Vec::new(),
|
||||
)
|
||||
.await;
|
||||
let bucket = "partial-reconnect";
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket before the outage");
|
||||
let payload = vec![b'r'; 1_300_000];
|
||||
set.put_object(bucket, "before", &mut PutObjReader::from_vec(payload.clone()), &opts)
|
||||
.await
|
||||
.expect("write the original object");
|
||||
|
||||
let mut saved_formats = Vec::new();
|
||||
for (dir, disk) in dirs.iter().zip(&disks).take(8) {
|
||||
let disk = disk.as_ref().expect("original disk");
|
||||
disk.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
|
||||
let path = dir.path().join(disk::RUSTFS_META_BUCKET).join(disk::FORMAT_CONFIG_FILE);
|
||||
saved_formats.push(tokio::fs::read(&path).await.expect("save original format"));
|
||||
// A corrupt format makes reconnect fail without admitting a blank replacement.
|
||||
tokio::fs::write(&path, b"invalid-format")
|
||||
.await
|
||||
.expect("make the original drive unavailable");
|
||||
let disk::Disk::Local(local) = disk.as_ref() else {
|
||||
panic!("fixture must use local drives");
|
||||
};
|
||||
local.get_disk().format_info.write().await.last_check = None;
|
||||
}
|
||||
set.connect_disks_with_active_monitoring(active_monitoring).await;
|
||||
let unavailable = set.get_disks_internal().await;
|
||||
for (original, retained) in disks.iter().zip(&unavailable).take(8) {
|
||||
assert!(
|
||||
Arc::ptr_eq(original.as_ref().expect("original"), retained.as_ref().expect("retained")),
|
||||
"failed reconnect must leave the old handle to exercise subsequent recovery"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
set.put_object(bucket, "during", &mut PutObjReader::from_vec(payload.clone()), &opts)
|
||||
.await
|
||||
.is_err(),
|
||||
"eight available drives must not satisfy a 16-drive EC:4 write"
|
||||
);
|
||||
|
||||
for (dir, saved) in dirs.iter().zip(&saved_formats).take(4) {
|
||||
let path = dir.path().join(disk::RUSTFS_META_BUCKET).join(disk::FORMAT_CONFIG_FILE);
|
||||
tokio::fs::write(path, saved)
|
||||
.await
|
||||
.expect("restore four original drive identities");
|
||||
}
|
||||
set.connect_disks_with_active_monitoring(active_monitoring).await;
|
||||
assert_eq!(
|
||||
set.drive_membership_snapshot().await.online.len(),
|
||||
12,
|
||||
"the four restored drives must be usable while four others remain unavailable"
|
||||
);
|
||||
set.put_object(bucket, "after", &mut PutObjReader::from_vec(payload.clone()), &opts)
|
||||
.await
|
||||
.expect("write at the restored EC:4 quorum");
|
||||
for object in ["before", "after"] {
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("read at the restored EC:4 quorum");
|
||||
let mut actual = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("read every reconstructed byte");
|
||||
assert_eq!(actual, payload, "the complete object must survive partial recovery");
|
||||
}
|
||||
for disk in set.get_disks_internal().await.into_iter().flatten() {
|
||||
disk.close().await.expect("stop the recovered drive monitors");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1144,162 +985,6 @@ mod tests {
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconnect_sweep_collects_all_blank_targets_and_excludes_original_disks() {
|
||||
use rustfs_heal_contracts::heal_channel::{HealAdmissionResult, HealChannelCommand, init_heal_channel};
|
||||
|
||||
const CHILD: &str = "RUSTFS_TEST_GROUPED_RECONNECT_CHILD";
|
||||
if std::env::var_os(CHILD).is_none() {
|
||||
// The heal sender is process-global; keep this receiver isolated.
|
||||
let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"set_disk::ops::locking::tests::reconnect_sweep_collects_all_blank_targets_and_excludes_original_disks",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD, "1")
|
||||
.output()
|
||||
.expect("isolated reconnect test should run");
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
assert!(output.status.success(), "isolated reconnect test failed: {stdout} {stderr}");
|
||||
assert!(stdout.contains("1 passed"), "child must execute the reconnect test: {stdout}");
|
||||
return;
|
||||
}
|
||||
let mut heal_rx = init_heal_channel().expect("isolated heal receiver");
|
||||
let disk_count = 4;
|
||||
let format = FormatV3::new(1, disk_count);
|
||||
let mut directories = Vec::new();
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for disk_idx in 0..2 {
|
||||
let (directory, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
|
||||
directories.push(directory);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
for disk_idx in 2..disk_count {
|
||||
let directory = tempfile::tempdir().expect("replacement directory");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(directory.path().to_str().expect("UTF-8 replacement path")).expect("replacement endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_idx);
|
||||
directories.push(directory);
|
||||
endpoints.push(endpoint);
|
||||
disks.push(None);
|
||||
}
|
||||
let mut ready_targets = endpoints[2..].iter().map(ToString::to_string).collect::<Vec<_>>();
|
||||
ready_targets.sort_unstable();
|
||||
// Disk 1 returns with its format and data; disks 2 and 3 are blank.
|
||||
disks[1]
|
||||
.take()
|
||||
.expect("original disk handle")
|
||||
.close()
|
||||
.await
|
||||
.expect("close original disk");
|
||||
let retained_data = directories[1].path().join("retained-data");
|
||||
tokio::fs::write(&retained_data, b"original data")
|
||||
.await
|
||||
.expect("write original data");
|
||||
let set = Arc::new(
|
||||
SetDisks::new(
|
||||
"reconnect-sweep".to_string(),
|
||||
Arc::new(RwLock::new(disks)),
|
||||
disk_count,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
endpoints.clone(),
|
||||
format,
|
||||
Vec::new(),
|
||||
)
|
||||
.await,
|
||||
);
|
||||
|
||||
// Block disk 0's format check, then make it a late blank replacement.
|
||||
let slow = set.get_disks_internal().await[0].clone().expect("original disk");
|
||||
assert!(slow.is_online().await, "initialize original disk identity");
|
||||
let disk::Disk::Local(wrapper) = slow.as_ref() else {
|
||||
panic!("fixture must use a local disk");
|
||||
};
|
||||
let slow_local = wrapper.get_disk();
|
||||
let mut slow_format_guard = slow_local.format_info.write().await;
|
||||
slow_format_guard.last_check = None;
|
||||
std::fs::remove_file(
|
||||
directories[0]
|
||||
.path()
|
||||
.join(disk::RUSTFS_META_BUCKET)
|
||||
.join(disk::FORMAT_CONFIG_FILE),
|
||||
)
|
||||
.expect("make the blocked disk a blank replacement");
|
||||
let sweep = tokio::spawn({
|
||||
let set = set.clone();
|
||||
async move { set.connect_disks_with_active_monitoring(true).await }
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
loop {
|
||||
let attached = set.get_disks_internal().await;
|
||||
if attached[1..].iter().all(Option::is_some) {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("other drives must reconnect while the first drive is blocked");
|
||||
let command = tokio::time::timeout(std::time::Duration::from_secs(10), heal_rx.recv())
|
||||
.await
|
||||
.expect("ready targets must be submitted while a drive is blocked")
|
||||
.expect("heal receiver should remain open");
|
||||
let HealChannelCommand::Start { request, response_tx } = command else {
|
||||
panic!("expected a replacement heal request");
|
||||
};
|
||||
assert_eq!(request.heal_endpoints, ready_targets, "ready blank disks must share one request");
|
||||
assert!(!sweep.is_finished(), "the blocked drive must still be pending");
|
||||
drop(slow_format_guard);
|
||||
// Do not acknowledge the first request yet: a delayed admission
|
||||
// response must not stop the late drive reconnecting and submitting.
|
||||
let late_command = tokio::time::timeout(std::time::Duration::from_secs(10), heal_rx.recv())
|
||||
.await
|
||||
.expect("late target must be submitted while the first response is pending")
|
||||
.expect("heal receiver should remain open");
|
||||
let HealChannelCommand::Start {
|
||||
request: late_request,
|
||||
response_tx: late_response,
|
||||
} = late_command
|
||||
else {
|
||||
panic!("expected a late replacement heal request");
|
||||
};
|
||||
assert_eq!(late_request.heal_endpoints, [endpoints[0].to_string()]);
|
||||
response_tx
|
||||
.send(Ok(HealAdmissionResult::Accepted))
|
||||
.expect("acknowledge ready targets");
|
||||
late_response
|
||||
.send(Ok(HealAdmissionResult::Accepted))
|
||||
.expect("acknowledge late target");
|
||||
sweep.await.expect("reconnect sweep should finish");
|
||||
assert!(heal_rx.try_recv().is_err(), "no duplicate request should be sent");
|
||||
let attached = set.get_disks_internal().await;
|
||||
for (index, disk) in attached.iter().enumerate() {
|
||||
let disk = disk.as_ref().expect("each verified endpoint should be attached");
|
||||
let format = load_format_erasure(disk, false).await;
|
||||
if index == 1 {
|
||||
assert!(format.is_ok(), "original disk must retain its format");
|
||||
} else {
|
||||
assert_eq!(format.expect_err("replacement must remain unformatted"), DiskError::UnformattedDisk);
|
||||
assert!(!disk.health_check_enabled_for_test(), "format must precede replacement health checks");
|
||||
}
|
||||
disk.close().await.expect("close test disk");
|
||||
}
|
||||
assert_eq!(tokio::fs::read(&retained_data).await.expect("read original data"), b"original data");
|
||||
let local_map = runtime_sources::local_disk_map_handle();
|
||||
let mut local_map = local_map.write().await;
|
||||
for endpoint in endpoints {
|
||||
local_map.remove(&endpoint.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn renew_disk_attaches_only_a_verified_local_unformatted_replacement() {
|
||||
let disk_count = 4;
|
||||
|
||||
@@ -5617,10 +5617,7 @@ mod tests {
|
||||
.await
|
||||
.expect("abort task should join after completion releases")
|
||||
.expect_err("the committed upload should no longer exist");
|
||||
assert!(
|
||||
matches!(abort_err, StorageError::InvalidUploadID(..)),
|
||||
"the completed upload must be absent after the tail drains, got: {abort_err:?}"
|
||||
);
|
||||
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
|
||||
|
||||
for (disk_index, disk) in disk_stores.iter().enumerate() {
|
||||
let fi = disk
|
||||
@@ -9556,12 +9553,10 @@ mod tests {
|
||||
).expect("actual completion identity"));
|
||||
}
|
||||
}
|
||||
// The fourth native syscall is held by release_rx even if
|
||||
// its deadline has already retired the async wrapper.
|
||||
if published == 3 { break; }
|
||||
if published == 3 && tasks.running() == 1 { break; }
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}).await.expect("three real native publication legs must succeed while the fourth syscall remains paused");
|
||||
}).await.expect("three real native publication legs must return before the fourth deadline");
|
||||
after_publication.as_ref().expect("publication barrier").wait_until_paused().await;
|
||||
// Quorum has been validated and the parent released its object
|
||||
// guard. The successful physical tail cannot retain a clone.
|
||||
|
||||
@@ -2999,44 +2999,6 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
let object_size = usize::try_from(fi.size)
|
||||
.map_err(|_| to_object_err(Error::other("inline fast path object size is invalid"), vec![bucket, object]))?;
|
||||
|
||||
if can_try_inline_data_shards_direct(object_size, fi.erasure.block_size)
|
||||
&& let Some(body) = verified_inline_body.filter(|body| body.len() == object_size)
|
||||
{
|
||||
if let Some(inline_prepare_stage_start) = inline_prepare_stage_start.take() {
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
GET_OBJECT_PATH_INLINE_DIRECT,
|
||||
GET_STAGE_INLINE_PREPARE,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
inline_prepare_stage_start.elapsed().as_secs_f64(),
|
||||
);
|
||||
}
|
||||
if rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
GET_OBJECT_PATH_INLINE_DIRECT,
|
||||
GET_STAGE_READER_SETUP,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
0.0,
|
||||
);
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
GET_OBJECT_PATH_INLINE_DIRECT,
|
||||
GET_STAGE_DECODE,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
0.0,
|
||||
);
|
||||
}
|
||||
record_get_object_reader_path_observation(GET_OBJECT_PATH_INLINE_DIRECT, object_class, size_bucket);
|
||||
let reader = GetObjectReader {
|
||||
stream: Box::new(Cursor::new(body.clone())),
|
||||
object_info,
|
||||
buffered_body: Some(body),
|
||||
body_source: GetObjectBodySource::Unprobed,
|
||||
};
|
||||
return Ok(reader);
|
||||
}
|
||||
|
||||
let checksum_info = fi.erasure.get_checksum_info(fi.parts[0].number);
|
||||
let checksum_algo =
|
||||
if fi.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S {
|
||||
@@ -3050,6 +3012,42 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
disks.get(index).is_some_and(Option::is_some)
|
||||
})
|
||||
{
|
||||
if let Some(body) = verified_inline_body.filter(|body| body.len() == object_size) {
|
||||
if let Some(inline_prepare_stage_start) = inline_prepare_stage_start.take() {
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
GET_OBJECT_PATH_INLINE_DIRECT,
|
||||
GET_STAGE_INLINE_PREPARE,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
inline_prepare_stage_start.elapsed().as_secs_f64(),
|
||||
);
|
||||
}
|
||||
if rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
GET_OBJECT_PATH_INLINE_DIRECT,
|
||||
GET_STAGE_READER_SETUP,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
0.0,
|
||||
);
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
GET_OBJECT_PATH_INLINE_DIRECT,
|
||||
GET_STAGE_DECODE,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
0.0,
|
||||
);
|
||||
}
|
||||
record_get_object_reader_path_observation(GET_OBJECT_PATH_INLINE_DIRECT, object_class, size_bucket);
|
||||
let reader = GetObjectReader {
|
||||
stream: Box::new(Cursor::new(body.clone())),
|
||||
object_info,
|
||||
buffered_body: Some(body),
|
||||
body_source: GetObjectBodySource::Unprobed,
|
||||
};
|
||||
return Ok(reader);
|
||||
}
|
||||
|
||||
let read_length = inline_erasure_shard_file_offset(
|
||||
0,
|
||||
object_size,
|
||||
@@ -8177,13 +8175,23 @@ fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum DeleteObjectCommitPause {
|
||||
BeforeNamespace,
|
||||
BeforeCommit,
|
||||
AfterPublish,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct DeleteObjectCommitBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
pause_after_publish: bool,
|
||||
pause: DeleteObjectCommitPause,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
namespace_pending: tokio::sync::Notify,
|
||||
namespace_acquired: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -8198,21 +8206,27 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option
|
||||
#[cfg(test)]
|
||||
impl DeleteObjectCommitBarrier {
|
||||
pub(crate) fn install(bucket: &str, object: &str) -> Self {
|
||||
Self::install_with_mode(bucket, object, false)
|
||||
Self::install_with_mode(bucket, object, DeleteObjectCommitPause::BeforeCommit)
|
||||
}
|
||||
|
||||
pub(crate) fn install_before_namespace(bucket: &str, object: &str) -> Self {
|
||||
Self::install_with_mode(bucket, object, DeleteObjectCommitPause::BeforeNamespace)
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) fn install_after_publish(bucket: &str, object: &str) -> Self {
|
||||
Self::install_with_mode(bucket, object, true)
|
||||
Self::install_with_mode(bucket, object, DeleteObjectCommitPause::AfterPublish)
|
||||
}
|
||||
|
||||
fn install_with_mode(bucket: &str, object: &str, pause_after_publish: bool) -> Self {
|
||||
fn install_with_mode(bucket: &str, object: &str, pause: DeleteObjectCommitPause) -> Self {
|
||||
let state = Arc::new(DeleteObjectCommitBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
pause_after_publish,
|
||||
pause,
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
namespace_pending: tokio::sync::Notify::new(),
|
||||
namespace_acquired: std::sync::atomic::AtomicBool::new(false),
|
||||
});
|
||||
let mut slot = DELETE_OBJECT_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
@@ -8232,6 +8246,19 @@ impl DeleteObjectCommitBarrier {
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
|
||||
pub(crate) async fn release_and_wait_until_namespace_pending(&self) {
|
||||
assert_eq!(self.state.pause, DeleteObjectCommitPause::BeforeNamespace);
|
||||
let pending = self.state.namespace_pending.notified();
|
||||
self.release();
|
||||
tokio::time::timeout(Duration::from_secs(5), pending)
|
||||
.await
|
||||
.expect("delete object must poll its actual namespace acquisition to Pending");
|
||||
}
|
||||
|
||||
pub(crate) fn namespace_acquired(&self) -> bool {
|
||||
self.state.namespace_acquired.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -8248,6 +8275,43 @@ impl Drop for DeleteObjectCommitBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_delete_object_before_namespace(bucket: &str, object: &str) {
|
||||
let barrier = DELETE_OBJECT_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("delete object commit barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| {
|
||||
barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeNamespace
|
||||
})
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn notify_delete_object_namespace(bucket: &str, object: &str, acquired: bool) {
|
||||
let barrier = DELETE_OBJECT_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("delete object commit barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| {
|
||||
barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeNamespace
|
||||
})
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
if acquired {
|
||||
barrier.namespace_acquired.store(true, std::sync::atomic::Ordering::Release);
|
||||
} else {
|
||||
barrier.namespace_pending.notify_one();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_delete_object_commit(bucket: &str, object: &str) {
|
||||
let barrier = DELETE_OBJECT_COMMIT_BARRIER
|
||||
@@ -8255,7 +8319,9 @@ async fn pause_delete_object_commit(bucket: &str, object: &str) {
|
||||
.lock()
|
||||
.expect("delete object commit barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object && !barrier.pause_after_publish)
|
||||
.filter(|barrier| {
|
||||
barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeCommit
|
||||
})
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
@@ -8270,7 +8336,9 @@ async fn pause_delete_object_commit_after_publish(bucket: &str, object: &str) {
|
||||
.lock()
|
||||
.expect("delete object commit barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause_after_publish)
|
||||
.filter(|barrier| {
|
||||
barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::AfterPublish
|
||||
})
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
@@ -10529,7 +10597,20 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
|
||||
// Guard lock for single object delete
|
||||
let _lock_guard = if (!opts.delete_prefix || opts.delete_prefix_object) && !opts.no_lock {
|
||||
Some(Arc::new(self.acquire_write_lock_diag("delete_object", bucket, object).await?))
|
||||
#[cfg(test)]
|
||||
let guard = {
|
||||
pause_delete_object_before_namespace(bucket, object).await;
|
||||
let guard = self
|
||||
.acquire_write_lock_diag_with_pending_hook("delete_object", bucket, object, || {
|
||||
notify_delete_object_namespace(bucket, object, false);
|
||||
})
|
||||
.await?;
|
||||
notify_delete_object_namespace(bucket, object, true);
|
||||
guard
|
||||
};
|
||||
#[cfg(not(test))]
|
||||
let guard = self.acquire_write_lock_diag("delete_object", bucket, object).await?;
|
||||
Some(Arc::new(guard))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -11089,7 +11170,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
// shared lock and permit metadata caching can early-stop once write
|
||||
// quorum agrees on the object identity.
|
||||
let allow_early_stop = !opts.no_lock || opts.metadata_cache_safe;
|
||||
let read = self.get_object_fileinfo_for_object_read(bucket, object, opts, true, allow_early_stop);
|
||||
let read = self.get_object_fileinfo(bucket, object, opts, true, allow_early_stop);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ObjectInfoRead,
|
||||
@@ -16906,7 +16987,6 @@ mod transition_commit_failure_tests {
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
read_quorum: 2,
|
||||
verified_inline_body: None,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
@@ -21435,6 +21515,206 @@ mod put_object_tmp_cleanup_tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deep_heal_preserves_committed_generation_during_old_data_cleanup() {
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
|
||||
|
||||
for (disk_count, parity, read_repair) in [(4, 2, false), (4, 2, true), (16, 4, false), (16, 4, true)] {
|
||||
let (dirs, disks, set) = super::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated(
|
||||
disk_count, 0, parity,
|
||||
)
|
||||
.await;
|
||||
let bucket = "native-heal-commit-cleanup";
|
||||
let object = format!("generation-{disk_count}-{parity}-{read_repair}.bin");
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create actual fixture volume");
|
||||
}
|
||||
let options = ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
|
||||
..Default::default()
|
||||
};
|
||||
set.put_object(bucket, &object, &mut PutObjReader::from_vec(vec![0x17; TEST_OBJECT_SIZE]), &options)
|
||||
.await
|
||||
.expect("acknowledge original generation on every disk");
|
||||
let old = disks[0]
|
||||
.read_version("", bucket, &object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read actual original generation");
|
||||
let old_dir = old.data_dir.expect("original external data directory");
|
||||
let old_part = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(&object)
|
||||
.join(old_dir.to_string())
|
||||
.join("part.1");
|
||||
assert!(tokio::fs::try_exists(&old_part).await.expect("original part exists"));
|
||||
|
||||
// Cleanup slots follow the writer's shuffled erasure distribution,
|
||||
// not the physical fixture order. Pause every real cleanup slot so
|
||||
// no physical old shard can disappear before the proof below.
|
||||
let barriers: Vec<_> = (0..disk_count)
|
||||
.map(|slot| rename_fanout_barrier::arm(&object, slot, rename_fanout_barrier::PHASE_CLEANUP))
|
||||
.collect();
|
||||
let writer_set = Arc::clone(&set);
|
||||
let writer_object = object.clone();
|
||||
let writer_options = options.clone();
|
||||
let payload = vec![0xa3; TEST_OBJECT_SIZE];
|
||||
let writer_payload = payload.clone();
|
||||
let writer = tokio::spawn(async move {
|
||||
writer_set
|
||||
.put_object(bucket, &writer_object, &mut PutObjReader::from_vec(writer_payload), &writer_options)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
for barrier in &barriers {
|
||||
barrier.wait_until_paused().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("overwrite commits before its real old-data cleanup pause");
|
||||
assert!(!writer.is_finished(), "tail-drained ACK must remain pending at cleanup");
|
||||
assert!(
|
||||
tokio::fs::try_exists(&old_part)
|
||||
.await
|
||||
.expect("old source still present at pause")
|
||||
);
|
||||
|
||||
let mut committed = Vec::new();
|
||||
let mut healthy_metadata = Vec::new();
|
||||
for disk in &disks {
|
||||
let info = disk
|
||||
.read_version("", bucket, &object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read independently committed physical generation without GET");
|
||||
assert_eq!((info.erasure.data_blocks, info.erasure.parity_blocks), (disk_count - parity, parity));
|
||||
assert!(info.data.is_none(), "use actual external parts");
|
||||
assert_ne!(info.data_dir, Some(old_dir));
|
||||
committed.push(info);
|
||||
healthy_metadata.push(
|
||||
disk.read_all(bucket, &format!("{object}/xl.meta"))
|
||||
.await
|
||||
.expect("committed raw metadata"),
|
||||
);
|
||||
}
|
||||
let new_dir = committed[0].data_dir.expect("new committed data directory");
|
||||
assert!(committed.iter().all(|info| info.data_dir == Some(new_dir)));
|
||||
let new_part = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(&object)
|
||||
.join(new_dir.to_string())
|
||||
.join("part.1");
|
||||
let expected_part = tokio::fs::read(&new_part).await.expect("exact new source bytes");
|
||||
let mut committed_parts = Vec::new();
|
||||
for dir in &dirs {
|
||||
let original = dir.path().join(bucket).join(&object).join(old_dir.to_string()).join("part.1");
|
||||
assert!(
|
||||
tokio::fs::try_exists(original)
|
||||
.await
|
||||
.expect("every old cleanup slot is paused")
|
||||
);
|
||||
committed_parts.push(
|
||||
tokio::fs::read(dir.path().join(bucket).join(&object).join(new_dir.to_string()).join("part.1"))
|
||||
.await
|
||||
.expect("every new encoded shard before fault"),
|
||||
);
|
||||
}
|
||||
tokio::fs::remove_file(&new_part)
|
||||
.await
|
||||
.expect("remove only the newly committed target shard");
|
||||
println!(
|
||||
"native_g01: geometry={disk_count}/{parity} read_repair={read_repair} old_cleanup_paused new_generation_committed target_part_missing"
|
||||
);
|
||||
|
||||
let (result, error) = tokio::time::timeout(
|
||||
Duration::from_secs(30),
|
||||
set.heal_object(
|
||||
bucket,
|
||||
&object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
read_repair,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("real locked Heal must progress during old cleanup")
|
||||
.expect("complete actual deep Heal");
|
||||
assert!(error.is_none(), "new generation must remain repairable: {error:?}");
|
||||
assert_eq!(result.drives_healed(), Some(1));
|
||||
// Shared data-phase repair deliberately does not certify full integrity.
|
||||
assert_eq!(result.integrity_verified, !read_repair);
|
||||
assert_eq!(tokio::fs::read(&new_part).await.expect("rebuilt part before any GET"), expected_part);
|
||||
let repaired = disks[0]
|
||||
.read_version("", bucket, &object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read actual repaired generation");
|
||||
assert_eq!(repaired.data_dir, Some(new_dir));
|
||||
assert_eq!(repaired.mod_time, committed[0].mod_time);
|
||||
for (disk, expected) in disks.iter().zip(&healthy_metadata).skip(1) {
|
||||
assert_eq!(
|
||||
disk.read_all(bucket, &format!("{object}/xl.meta"))
|
||||
.await
|
||||
.expect("healthy raw metadata after Heal"),
|
||||
*expected
|
||||
);
|
||||
}
|
||||
println!(
|
||||
"native_g01: geometry={disk_count}/{parity} read_repair={read_repair} locked_heal_completed new_generation_part_rebuilt_before_get"
|
||||
);
|
||||
|
||||
for barrier in &barriers {
|
||||
barrier.release();
|
||||
}
|
||||
let acknowledged = tokio::time::timeout(Duration::from_secs(30), writer)
|
||||
.await
|
||||
.expect("cleanup drains")
|
||||
.expect("writer joins")
|
||||
.expect("overwrite ACK remains successful");
|
||||
assert_eq!(acknowledged.size, i64::try_from(payload.len()).expect("fixture size fits i64"));
|
||||
drop(barriers);
|
||||
for (slot, dir) in dirs.iter().enumerate() {
|
||||
let root = dir.path().join(bucket).join(&object);
|
||||
assert!(
|
||||
!tokio::fs::try_exists(root.join(old_dir.to_string()))
|
||||
.await
|
||||
.expect("old cleanup completed")
|
||||
);
|
||||
assert_eq!(
|
||||
tokio::fs::read(root.join(new_dir.to_string()).join("part.1"))
|
||||
.await
|
||||
.expect("new shard survives cleanup before GET"),
|
||||
committed_parts[slot],
|
||||
"old cleanup must preserve every exact committed encoded shard"
|
||||
);
|
||||
}
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, &object, None, HeaderMap::new(), &options)
|
||||
.await
|
||||
.expect("read final acknowledged generation");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("read complete final payload");
|
||||
assert_eq!(body, payload);
|
||||
for disk in &disks {
|
||||
let final_info = disk
|
||||
.read_version("", bucket, &object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("final physical generation");
|
||||
assert_eq!(final_info.data_dir, Some(new_dir), "old cleanup must not revert the healed generation");
|
||||
}
|
||||
println!(
|
||||
"native_g01: geometry={disk_count}/{parity} read_repair={read_repair} overwrite_ack_preserved final_physical_generation_and_full_body_verified"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_put_releases_namespace_lock_before_old_data_cleanup() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
@@ -22845,116 +23125,6 @@ mod put_object_tmp_cleanup_tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Per-drive view of an object: every version (including free versions)
|
||||
/// recorded in its xl.meta, and the data directories next to it. A missing
|
||||
/// xl.meta and one without versions both hold no version.
|
||||
async fn per_drive_object_state(
|
||||
dirs: &[TempDir],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Vec<(Vec<(Option<Uuid>, Option<Uuid>, Option<OffsetDateTime>)>, Vec<String>)> {
|
||||
let mut drives = Vec::with_capacity(dirs.len());
|
||||
for dir in dirs {
|
||||
let object_dir = dir.path().join(bucket).join(object);
|
||||
let versions = match tokio::fs::read(object_dir.join(STORAGE_FORMAT_FILE)).await {
|
||||
Ok(encoded) => {
|
||||
let all = FileMeta::load(&encoded)
|
||||
.expect("xl.meta should decode")
|
||||
.get_all_file_info_versions(bucket, object, true)
|
||||
.expect("xl.meta versions should be readable");
|
||||
all.versions
|
||||
.iter()
|
||||
.chain(all.free_versions.iter())
|
||||
.map(|version| (version.version_id, version.data_dir, version.mod_time))
|
||||
.collect()
|
||||
}
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Vec::new(),
|
||||
Err(err) => panic!("xl.meta at {object_dir:?} should be readable: {err}"),
|
||||
};
|
||||
let mut data_dirs = Vec::new();
|
||||
if let Ok(mut entries) = tokio::fs::read_dir(&object_dir).await {
|
||||
while let Some(entry) = entries.next_entry().await.expect("object directory entry should be readable") {
|
||||
if entry
|
||||
.file_type()
|
||||
.await
|
||||
.expect("object entry type should be readable")
|
||||
.is_dir()
|
||||
{
|
||||
data_dirs.push(entry.file_name().to_string_lossy().into_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
data_dirs.sort();
|
||||
drives.push((versions, data_dirs));
|
||||
}
|
||||
drives
|
||||
}
|
||||
|
||||
/// Stage F lists any version present on half of the drives, so a PUT that
|
||||
/// misses write quorum must not leave its version behind once its rollback
|
||||
/// completes: no drive may keep the new xl.meta entry or its data directory.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn failed_put_rollback_leaves_no_new_version_on_any_drive() {
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
|
||||
for size in [4096, TEST_OBJECT_SIZE] {
|
||||
// (versioned, seed an earlier version first)
|
||||
for (versioned, seeded) in [(false, false), (false, true), (true, true)] {
|
||||
let case = format!("{write_completion:?} size={size} versioned={versioned} seeded={seeded}");
|
||||
let (dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-rollback-no-residue";
|
||||
let object = "rolled-back-object";
|
||||
make_completion_test_bucket(&disks, bucket).await;
|
||||
if seeded {
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![b'0'; size]),
|
||||
&ObjectOptions {
|
||||
versioned,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("{case}: seed version should commit on every drive: {err}"));
|
||||
}
|
||||
wait_for_tmp_workspace_to_drain(&dirs, "seed PUT must leave no staging").await;
|
||||
let before = per_drive_object_state(&dirs, bucket, object).await;
|
||||
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let _fault = rename_fault_injection::fail_rename_on(object, &[0, 1]);
|
||||
let err = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![b'1'; size]),
|
||||
&ObjectOptions {
|
||||
versioned,
|
||||
write_completion,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("two renamed drives cannot satisfy write quorum three");
|
||||
assert!(
|
||||
matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
|
||||
"{case}: original quorum error expected: {err}"
|
||||
);
|
||||
assert_eq!(tasks.running(), 0, "{case}: rename and undo tasks must be reaped before return");
|
||||
// An incomplete rollback retains staging for recovery, so a
|
||||
// drained workspace shows the rollback completed.
|
||||
wait_for_tmp_workspace_to_drain(&dirs, "a completed rollback must release staging").await;
|
||||
let after = per_drive_object_state(&dirs, bucket, object).await;
|
||||
assert_eq!(after, before, "{case}: no drive may keep the failed PUT's version or data");
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn put_incomplete_rollback_preserves_staging_and_old_version_backup() {
|
||||
|
||||
+135
-523
@@ -264,66 +264,6 @@ enum MetadataReadPurpose {
|
||||
GetObjectReader,
|
||||
}
|
||||
|
||||
/// How a metadata read-quorum failure is reported when the replies prove the
|
||||
/// object or version dangling. Only the object read entry points
|
||||
/// (`get_object_info`, `get_object_reader` and prepared GET metadata) report
|
||||
/// it as not found; read-before-write callers of `get_object_fileinfo*` keep
|
||||
/// the quorum error so fences and delete recovery never treat a torn write as
|
||||
/// an absent key.
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum DanglingRead {
|
||||
KeepQuorumError,
|
||||
ReportNotFound,
|
||||
}
|
||||
|
||||
/// Read-only form of MinIO's GET-path dangling check: once
|
||||
/// `totalResp == setDriveCount`, `getObjectFileInfo` calls `deleteIfDangling`
|
||||
/// and a dangling verdict replaces the read-quorum error with
|
||||
/// `errFileNotFound` or `errFileVersionNotFound`.
|
||||
///
|
||||
/// Each deliberate difference keeps the quorum error instead:
|
||||
/// - nothing is deleted; heal owns cleanup and its recent-write grace window;
|
||||
/// - MinIO counts an offline drive as a response and lets `isObjectDangling`
|
||||
/// reject it, while here every slot must hold valid metadata or a not-found
|
||||
/// reply, so an offline drive, a timeout, an unanswered slot, or a corrupt or
|
||||
/// unusable xl.meta stays uncertain;
|
||||
/// - the valid replies must be one metadata generation and not a remote
|
||||
/// (transitioned) version, the candidates heal's `dangling_delete_safety`
|
||||
/// also refuses to delete.
|
||||
fn dangling_read_not_found(
|
||||
bucket: &str,
|
||||
version_id: &str,
|
||||
parts_metadata: &[FileInfo],
|
||||
errs: &[Option<DiskError>],
|
||||
fanout_complete: bool,
|
||||
) -> Option<DiskError> {
|
||||
if !fanout_complete || crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
return None;
|
||||
}
|
||||
let mut generation = None;
|
||||
for (metadata, err) in parts_metadata.iter().zip(errs) {
|
||||
match err {
|
||||
Some(DiskError::FileNotFound | DiskError::FileVersionNotFound) => {}
|
||||
Some(_) => return None,
|
||||
None => {
|
||||
if !super::file_info_is_valid_for_metadata(metadata) || metadata.is_remote() {
|
||||
return None;
|
||||
}
|
||||
let identity = SetDisks::file_info_quorum_hash(metadata);
|
||||
if *generation.get_or_insert(identity) != identity {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let (_, dangling) = super::is_object_dangling(parts_metadata, errs, &std::collections::HashMap::new());
|
||||
dangling.then_some(if version_id.is_empty() {
|
||||
DiskError::FileNotFound
|
||||
} else {
|
||||
DiskError::FileVersionNotFound
|
||||
})
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
/// A startup quorum can leave remote slots unregistered even after their
|
||||
/// peers become ready. Refresh those slots before taking the GET metadata
|
||||
@@ -487,7 +427,6 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn cache_get_object_fileinfo(
|
||||
&self,
|
||||
identity: (&str, &str),
|
||||
@@ -498,32 +437,8 @@ impl SetDisks {
|
||||
read_quorum: usize,
|
||||
) {
|
||||
let (fi, capability) = selected;
|
||||
let _ = self
|
||||
.cache_get_object_fileinfo_owned(
|
||||
identity,
|
||||
generation,
|
||||
(fi.clone(), capability, None),
|
||||
parts_metadata.to_vec(),
|
||||
online_disks.to_vec(),
|
||||
read_quorum,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn cache_get_object_fileinfo_owned(
|
||||
&self,
|
||||
identity: (&str, &str),
|
||||
generation: Option<GetObjectMetadataCacheGeneration>,
|
||||
selected: (FileInfo, GetObjectMetadataCacheCapability, Option<Bytes>),
|
||||
parts_metadata: Vec<FileInfo>,
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
read_quorum: usize,
|
||||
) -> GetObjectFileInfo {
|
||||
let (fi, capability, verified_inline_body) = selected;
|
||||
if fi.deleted || !fi.has_valid_erasure_geometry() {
|
||||
let mut snapshot = GetObjectFileInfo::owned(fi, parts_metadata, online_disks);
|
||||
snapshot.set_verified_inline_body(verified_inline_body);
|
||||
return snapshot;
|
||||
return;
|
||||
}
|
||||
let (bucket, object) = identity;
|
||||
|
||||
@@ -534,24 +449,20 @@ impl SetDisks {
|
||||
GET_METADATA_CACHE_DECISION_REJECT,
|
||||
GET_METADATA_CACHE_REASON_STALE_PUBLICATION,
|
||||
);
|
||||
let mut snapshot = GetObjectFileInfo::owned(fi, parts_metadata, online_disks);
|
||||
snapshot.set_verified_inline_body(verified_inline_body);
|
||||
return snapshot;
|
||||
return;
|
||||
};
|
||||
|
||||
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
|
||||
let entry = Arc::new(GetObjectMetadataCacheEntry {
|
||||
created_at: Instant::now(),
|
||||
fi,
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
fi: fi.clone(),
|
||||
parts_metadata: parts_metadata.to_vec(),
|
||||
online_disks: online_disks.to_vec(),
|
||||
read_quorum,
|
||||
capability,
|
||||
verified_inline_body,
|
||||
});
|
||||
self.insert_get_object_metadata_cache_entry_after_insert(key, generation, Arc::clone(&entry), || {})
|
||||
self.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || {})
|
||||
.await;
|
||||
GetObjectFileInfo::owned_published(entry)
|
||||
}
|
||||
|
||||
async fn insert_get_object_metadata_cache_entry_after_insert(
|
||||
@@ -672,31 +583,6 @@ impl SetDisks {
|
||||
read_data,
|
||||
caller_allows_early_stop,
|
||||
MetadataReadPurpose::FullLayout,
|
||||
DanglingRead::KeepQuorumError,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// `get_object_fileinfo` for `get_object_info` and prepared GET metadata:
|
||||
/// a provably dangling object or version reads as not found.
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
pub(super) async fn get_object_fileinfo_for_object_read(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
read_data: bool,
|
||||
caller_allows_early_stop: bool,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
self.get_object_fileinfo_gated_inner(
|
||||
bucket,
|
||||
object,
|
||||
opts,
|
||||
read_data,
|
||||
caller_allows_early_stop,
|
||||
MetadataReadPurpose::FullLayout,
|
||||
DanglingRead::ReportNotFound,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -718,7 +604,6 @@ impl SetDisks {
|
||||
read_data,
|
||||
caller_allows_early_stop,
|
||||
MetadataReadPurpose::GetObjectReader,
|
||||
DanglingRead::ReportNotFound,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -736,16 +621,8 @@ impl SetDisks {
|
||||
read_data: bool,
|
||||
allow_early_stop: bool,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
self.get_object_fileinfo_gated_inner(
|
||||
bucket,
|
||||
object,
|
||||
opts,
|
||||
read_data,
|
||||
allow_early_stop,
|
||||
MetadataReadPurpose::FullLayout,
|
||||
DanglingRead::KeepQuorumError,
|
||||
)
|
||||
.await
|
||||
self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, allow_early_stop, MetadataReadPurpose::FullLayout)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -757,7 +634,6 @@ impl SetDisks {
|
||||
read_data: bool,
|
||||
allow_early_stop: bool,
|
||||
purpose: MetadataReadPurpose,
|
||||
dangling_read: DanglingRead,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
let vid = opts.version_id.clone().unwrap_or_default();
|
||||
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
@@ -917,17 +793,7 @@ impl SetDisks {
|
||||
let _min_disks = self.set_drive_count - self.default_parity_count;
|
||||
|
||||
let metadata_resolve_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
// Like MinIO, check for a dangling object after either metadata quorum
|
||||
// reduction fails; only a read-quorum failure is reclassified.
|
||||
let dangling_not_found = |err: &DiskError| {
|
||||
if dangling_read == DanglingRead::ReportNotFound && *err == DiskError::ErasureReadQuorum {
|
||||
dangling_read_not_found(bucket, &vid, &parts_metadata, &errs, metadata_fanout_complete)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)
|
||||
.map_err(|err| dangling_not_found(&err).unwrap_or(err))
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object, &vid]))
|
||||
{
|
||||
Ok(v) => v,
|
||||
@@ -947,15 +813,6 @@ impl SetDisks {
|
||||
let write_quorum = usize::try_from(write_quorum)
|
||||
.map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
if let Some(not_found) = dangling_not_found(&err) {
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "not_found");
|
||||
record_get_stage_duration_if_enabled(
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
GET_STAGE_METADATA_RESOLVE,
|
||||
metadata_resolve_stage_start,
|
||||
);
|
||||
return Err(to_object_err(not_found.into(), vec![bucket, object, &vid]));
|
||||
}
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "not_reached");
|
||||
error!("reduce_read_quorum_errs: {:?}, bucket: {}, object: {}", &err, bucket, object);
|
||||
record_get_stage_duration_if_enabled(
|
||||
@@ -988,11 +845,7 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
metadata_fanout_diagnostics.record_quorum_candidate_latency(metadata_metrics_path, fileinfo_selection_quorum);
|
||||
let needs_late_metadata_fanout = !metadata_fanout_complete
|
||||
&& allow_early_stop
|
||||
&& non_inline_data_read_early_stop_allowed(read_data, bucket, object)
|
||||
&& late_materialization_candidate_is_safe(&fi);
|
||||
let mut snapshot = if !opts.suppress_read_repair && errs.iter().any(|err| err.is_some()) {
|
||||
if !opts.suppress_read_repair && errs.iter().any(|err| err.is_some()) {
|
||||
let version_id = resolved_read_repair_version_id(&fi, opts.version_id.as_deref());
|
||||
submit_read_repair_heal(
|
||||
&fi.volume,
|
||||
@@ -1004,9 +857,6 @@ impl SetDisks {
|
||||
"metadata_read_error",
|
||||
)
|
||||
.await;
|
||||
let mut snapshot = GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks);
|
||||
snapshot.set_verified_inline_body(verified_inline_body);
|
||||
snapshot
|
||||
} else if use_metadata_cache
|
||||
&& (metadata_fanout_complete
|
||||
|| condition_stopped
|
||||
@@ -1019,11 +869,11 @@ impl SetDisks {
|
||||
{
|
||||
#[cfg(test)]
|
||||
metadata_cache_tests::wait_before_metadata_cache_publish(bucket, object).await;
|
||||
self.cache_get_object_fileinfo_owned(
|
||||
self.cache_get_object_fileinfo(
|
||||
(bucket, object),
|
||||
metadata_cache_generation,
|
||||
(
|
||||
fi,
|
||||
&fi,
|
||||
if metadata_fanout_complete {
|
||||
GetObjectMetadataCacheCapability::Full
|
||||
} else if condition_stopped {
|
||||
@@ -1031,23 +881,29 @@ impl SetDisks {
|
||||
} else {
|
||||
GetObjectMetadataCacheCapability::QuorumSelected
|
||||
},
|
||||
verified_inline_body,
|
||||
),
|
||||
parts_metadata,
|
||||
op_online_disks,
|
||||
&parts_metadata,
|
||||
&op_online_disks,
|
||||
read_quorum,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
let mut snapshot = GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks);
|
||||
snapshot.set_verified_inline_body(verified_inline_body);
|
||||
snapshot
|
||||
};
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_METADATA_RESOLVE, metadata_resolve_stage_start);
|
||||
if needs_late_metadata_fanout {
|
||||
snapshot.set_late_metadata_fanout_disks(disks);
|
||||
.await;
|
||||
}
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_METADATA_RESOLVE, metadata_resolve_stage_start);
|
||||
// debug!("get_object_fileinfo pick fi {:?}", &fi);
|
||||
|
||||
// let online_disks: Vec<Option<DiskStore>> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect();
|
||||
|
||||
let mut snapshot = if !metadata_fanout_complete
|
||||
&& allow_early_stop
|
||||
&& non_inline_data_read_early_stop_allowed(read_data, bucket, object)
|
||||
&& late_materialization_candidate_is_safe(&fi)
|
||||
{
|
||||
GetObjectFileInfo::owned_with_late_metadata_fanout(fi, parts_metadata, op_online_disks, disks)
|
||||
} else {
|
||||
GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks)
|
||||
};
|
||||
snapshot.condition_stopped = condition_stopped;
|
||||
snapshot.set_verified_inline_body(verified_inline_body);
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
@@ -1576,7 +1432,10 @@ impl SetDisks {
|
||||
"Create bitrot reader failed read quorum"
|
||||
);
|
||||
record_get_object_pipeline_failure(GET_STAGE_READER_SETUP, reason);
|
||||
return Err(to_object_err(read_err.into(), vec![bucket, object]));
|
||||
// The metadata selected this object, but the part cannot
|
||||
// be reconstructed. A dominant disk I/O/not-found error
|
||||
// must not erase the object's read-quorum failure.
|
||||
return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
|
||||
}
|
||||
let reason = GetObjectFailureReason::ReadQuorum;
|
||||
error!(
|
||||
@@ -1601,7 +1460,7 @@ impl SetDisks {
|
||||
"Create bitrot reader did not have enough disks"
|
||||
);
|
||||
record_get_object_pipeline_failure(GET_STAGE_READER_SETUP, reason);
|
||||
return Err(Error::other(format!("not enough disks to read: {:?}", reader_setup.errors)));
|
||||
return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
|
||||
}
|
||||
|
||||
// Check if we have missing shards even though we can read successfully
|
||||
@@ -2136,10 +1995,10 @@ impl SetDisks {
|
||||
if let Some(read_err) = reduce_read_quorum_errs(&reader_setup.errors, OBJECT_OP_IGNORED_ERRS, erasure.data_shards) {
|
||||
let reason = classify_disk_error(&read_err);
|
||||
record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, reason);
|
||||
return Err(to_object_err(read_err.into(), vec![bucket, object]));
|
||||
return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
|
||||
}
|
||||
record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, GetObjectFailureReason::ReadQuorum);
|
||||
return Err(Error::other(format!("not enough disks to read: {:?}", reader_setup.errors)));
|
||||
return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
|
||||
}
|
||||
|
||||
let missing_shards = reader_setup.completed_failed_shards();
|
||||
@@ -4174,7 +4033,6 @@ mod metadata_cache_tests {
|
||||
parts_metadata: vec![fi],
|
||||
online_disks: vec![None],
|
||||
read_quorum: 1,
|
||||
verified_inline_body: None,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
@@ -4836,7 +4694,6 @@ mod metadata_cache_tests {
|
||||
online_disks: Vec::new(),
|
||||
read_quorum: 0,
|
||||
capability,
|
||||
verified_inline_body: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -5329,7 +5186,6 @@ mod metadata_cache_tests {
|
||||
parts_metadata: vec![fi],
|
||||
online_disks: vec![None],
|
||||
read_quorum: 1,
|
||||
verified_inline_body: None,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
@@ -5448,7 +5304,6 @@ mod metadata_cache_tests {
|
||||
parts_metadata: vec![fi],
|
||||
online_disks: Vec::new(),
|
||||
read_quorum: 0,
|
||||
verified_inline_body: None,
|
||||
});
|
||||
|
||||
set.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || {
|
||||
@@ -5584,7 +5439,6 @@ mod metadata_cache_tests {
|
||||
fi,
|
||||
online_disks: Vec::new(),
|
||||
read_quorum: 0,
|
||||
verified_inline_body: None,
|
||||
})
|
||||
};
|
||||
|
||||
@@ -7950,6 +7804,105 @@ mod tests {
|
||||
(data_dir, files)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn external_missing_shards_keep_typed_read_quorum() {
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let payload = vec![0x37; 256 * 1024];
|
||||
for readable_shards in [1, 2, 4] {
|
||||
let erasure = coding::Erasure::new(4, 2, 1024 * 1024);
|
||||
let mut fi = codec_streaming_test_fileinfo(i64::try_from(payload.len()).expect("fixture size fits i64"), 1);
|
||||
fi.erasure.data_blocks = erasure.data_shards;
|
||||
fi.erasure.parity_blocks = erasure.parity_shards;
|
||||
fi.erasure.block_size = erasure.block_size;
|
||||
fi.erasure.distribution = (1..=erasure.total_shard_count()).collect();
|
||||
let (dirs, disks) = local_test_disks(erasure.total_shard_count(), CODEC_STREAMING_TEST_BUCKET).await;
|
||||
let (data_dir, files) = codec_streaming_external_files(&erasure, &payload, &disks).await;
|
||||
fi.data_dir = Some(data_dir);
|
||||
for directory in dirs.iter().skip(readable_shards) {
|
||||
tokio::fs::remove_file(
|
||||
directory
|
||||
.path()
|
||||
.join(CODEC_STREAMING_TEST_BUCKET)
|
||||
.join(CODEC_STREAMING_TEST_OBJECT)
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1"),
|
||||
)
|
||||
.await
|
||||
.expect("remove real part while retaining its selected metadata");
|
||||
}
|
||||
|
||||
let mut output = Vec::new();
|
||||
let legacy = SetDisks::get_object_with_fileinfo(
|
||||
CODEC_STREAMING_TEST_BUCKET,
|
||||
CODEC_STREAMING_TEST_OBJECT,
|
||||
Arc::new(ErasureCache::new()),
|
||||
0,
|
||||
fi.size,
|
||||
&mut output,
|
||||
fi.clone(),
|
||||
files.clone(),
|
||||
&disks,
|
||||
0,
|
||||
0,
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
"plain_single_part",
|
||||
"le_1mib",
|
||||
)
|
||||
.await;
|
||||
let streaming = SetDisks::get_object_mid_size_reader_with_fileinfo(
|
||||
CODEC_STREAMING_TEST_BUCKET,
|
||||
CODEC_STREAMING_TEST_OBJECT,
|
||||
Arc::new(ErasureCache::new()),
|
||||
&fi,
|
||||
&files,
|
||||
&disks,
|
||||
0,
|
||||
0,
|
||||
false,
|
||||
true,
|
||||
"plain_single_part",
|
||||
"le_1mib",
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
if readable_shards < erasure.data_shards {
|
||||
for error in [
|
||||
legacy.expect_err("legacy read must reject quorum minus one"),
|
||||
streaming.err().expect("streaming setup must reject quorum minus one"),
|
||||
] {
|
||||
assert_eq!(
|
||||
error,
|
||||
Error::InsufficientReadQuorum(
|
||||
CODEC_STREAMING_TEST_BUCKET.to_owned(),
|
||||
CODEC_STREAMING_TEST_OBJECT.to_owned(),
|
||||
)
|
||||
);
|
||||
}
|
||||
assert!(output.is_empty(), "quorum rejection must precede all response bytes");
|
||||
} else {
|
||||
legacy.expect("legacy reader retains reconstructable quorum");
|
||||
assert_eq!(output, payload, "legacy read returns every original byte at quorum");
|
||||
let GetCodecStreamingReaderBuildOutcome::Reader(mut reader) =
|
||||
streaming.expect("streaming quorum remains readable")
|
||||
else {
|
||||
panic!("mid-size quorum must yield an actual reader");
|
||||
};
|
||||
let mut decoded = Vec::new();
|
||||
reader
|
||||
.read_to_end(&mut decoded)
|
||||
.await
|
||||
.expect("streaming reader returns the complete body");
|
||||
assert_eq!(decoded, payload, "streaming read returns every original byte at quorum");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn payload_read_repair_is_suppressed_only_for_read_only_requests() {
|
||||
@@ -9740,344 +9693,3 @@ mod tests {
|
||||
assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod dangling_read_tests {
|
||||
//! GET and HEAD report a metadata read-quorum failure as not found only when
|
||||
//! every drive answered and the surviving copies prove the object or version
|
||||
//! dangling; any uncertainty keeps the read-quorum error (503 SlowDownRead).
|
||||
use super::*;
|
||||
use crate::disk::{DeleteOptions, STORAGE_FORMAT_FILE};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::object::ObjectOperations;
|
||||
use http::HeaderMap;
|
||||
use std::path::PathBuf;
|
||||
use tempfile::TempDir;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
/// EC 4+2: read quorum is four, so three surviving copies are below it
|
||||
/// while still reaching the half-set probe.
|
||||
const DRIVES: usize = 6;
|
||||
const PARITY: usize = 2;
|
||||
|
||||
async fn dangling_test_set(bucket: &str) -> (Vec<TempDir>, Arc<SetDisks>) {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
ctx.update_erasure_type(crate::layout::endpoints::SetupType::Erasure).await;
|
||||
let (dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(DRIVES, PARITY, ctx).await;
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("test bucket should be created on every drive");
|
||||
(dirs, set)
|
||||
}
|
||||
|
||||
async fn put(set: &SetDisks, bucket: &str, object: &str, body: &[u8], versioned: bool) -> ObjectInfo {
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(body.to_vec()),
|
||||
&ObjectOptions {
|
||||
versioned,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("seed object should commit on every drive")
|
||||
}
|
||||
|
||||
fn object_dir(dirs: &[TempDir], disk: usize, bucket: &str, object: &str) -> PathBuf {
|
||||
dirs[disk].path().join(bucket).join(object)
|
||||
}
|
||||
|
||||
async fn remove_object_from(dirs: &[TempDir], bucket: &str, object: &str, disks: &[usize]) {
|
||||
for &disk in disks {
|
||||
tokio::fs::remove_dir_all(object_dir(dirs, disk, bucket, object))
|
||||
.await
|
||||
.expect("object directory should be removable to model a drive that never received the write");
|
||||
}
|
||||
}
|
||||
|
||||
fn read_opts(version_id: Option<&str>) -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
version_id: version_id.map(str::to_owned),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn head(set: &SetDisks, bucket: &str, object: &str, version_id: Option<&str>) -> Result<ObjectInfo> {
|
||||
set.get_object_info(bucket, object, &read_opts(version_id)).await
|
||||
}
|
||||
|
||||
async fn get(set: &SetDisks, bucket: &str, object: &str, version_id: Option<&str>) -> Result<Vec<u8>> {
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts(version_id))
|
||||
.await?;
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("an opened object should stream its body");
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
fn is_read_quorum_error(err: &StorageError) -> bool {
|
||||
matches!(err, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))
|
||||
}
|
||||
|
||||
async fn assert_get_and_head_keep_read_quorum_error(set: &SetDisks, bucket: &str, object: &str, case: &str) {
|
||||
let Err(err) = head(set, bucket, object, None).await else {
|
||||
panic!("{case}: HEAD must not serve an object below read quorum");
|
||||
};
|
||||
assert!(is_read_quorum_error(&err), "{case}: HEAD must keep the read-quorum error: {err:?}");
|
||||
let Err(err) = get(set, bucket, object, None).await else {
|
||||
panic!("{case}: GET must not serve an object below read quorum");
|
||||
};
|
||||
assert!(is_read_quorum_error(&err), "{case}: GET must keep the read-quorum error: {err:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_object_reads_as_not_found_on_get_and_head() {
|
||||
let bucket = "dangling-read-unversioned";
|
||||
let object = "torn/object";
|
||||
let (dirs, set) = dangling_test_set(bucket).await;
|
||||
put(&set, bucket, object, b"torn write payload", false).await;
|
||||
// Three copies stay below read quorum four; the other drives answer FileNotFound.
|
||||
remove_object_from(&dirs, bucket, object, &[3, 4, 5]).await;
|
||||
|
||||
let is_not_found = |err: &StorageError| matches!(err, StorageError::ObjectNotFound(b, o) if b == bucket && o == object);
|
||||
let Err(err) = head(&set, bucket, object, None).await else {
|
||||
panic!("HEAD must not serve a dangling object");
|
||||
};
|
||||
assert!(is_not_found(&err), "HEAD of a dangling object must be NoSuchKey: {err:?}");
|
||||
let Err(err) = get(&set, bucket, object, None).await else {
|
||||
panic!("GET must not serve a dangling object");
|
||||
};
|
||||
assert!(is_not_found(&err), "GET of a dangling object must be NoSuchKey: {err:?}");
|
||||
let Err(err) = set
|
||||
.prepare_get_object_metadata(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
else {
|
||||
panic!("prepared GET metadata must not resolve a dangling object");
|
||||
};
|
||||
assert!(is_not_found(&err), "prepared GET metadata must be NoSuchKey: {err:?}");
|
||||
|
||||
// Read-before-write callers keep the quorum error so fences and delete
|
||||
// recovery never treat a torn write as an absent key.
|
||||
let Err(err) = set
|
||||
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
|
||||
.await
|
||||
else {
|
||||
panic!("internal metadata reads must not resolve a dangling object");
|
||||
};
|
||||
assert!(is_read_quorum_error(&err), "internal metadata reads keep the quorum error: {err:?}");
|
||||
|
||||
// The classification is read-only; heal owns dangling cleanup.
|
||||
for disk in 0..3 {
|
||||
assert!(
|
||||
object_dir(&dirs, disk, bucket, object).join(STORAGE_FORMAT_FILE).exists(),
|
||||
"GET/HEAD must not delete the surviving copy on drive {disk}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_classification_requires_every_drive_to_answer() {
|
||||
let bucket = "dangling-read-offline";
|
||||
let object = "torn/object";
|
||||
let (dirs, set) = dangling_test_set(bucket).await;
|
||||
let payload = b"offline drive may hold the fourth copy";
|
||||
put(&set, bucket, object, payload, false).await;
|
||||
remove_object_from(&dirs, bucket, object, &[3, 4]).await;
|
||||
let offline = set.disks.write().await[5].take().expect("drive five should start online");
|
||||
|
||||
assert_get_and_head_keep_read_quorum_error(&set, bucket, object, "one drive offline").await;
|
||||
|
||||
// The offline drive held a copy: once it answers, read quorum is met.
|
||||
set.disks.write().await[5] = Some(offline);
|
||||
let body = get(&set, bucket, object, None)
|
||||
.await
|
||||
.expect("four copies satisfy read quorum once the drive returns");
|
||||
assert_eq!(body, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_version_reads_as_no_such_version_while_other_versions_stay_readable() {
|
||||
let bucket = "dangling-read-versioned";
|
||||
let object = "torn/versioned";
|
||||
let (_dirs, set) = dangling_test_set(bucket).await;
|
||||
let v1 = put(&set, bucket, object, b"version one", true)
|
||||
.await
|
||||
.version_id
|
||||
.expect("versioned PUT should assign a version id");
|
||||
let v2 = put(&set, bucket, object, b"version two", true)
|
||||
.await
|
||||
.version_id
|
||||
.expect("versioned PUT should assign a version id");
|
||||
let disks = set.disks.read().await.clone();
|
||||
for disk in disks.iter().skip(3) {
|
||||
let disk = disk.as_ref().expect("every drive should be online");
|
||||
let mut remove = FileInfo {
|
||||
volume: bucket.to_string(),
|
||||
name: object.to_string(),
|
||||
version_id: Some(v2),
|
||||
..Default::default()
|
||||
};
|
||||
remove.set_skip_tier_free_version();
|
||||
disk.delete_version(bucket, object, remove, false, DeleteOptions::default())
|
||||
.await
|
||||
.expect("dropping version two should model a drive that never received it");
|
||||
assert!(matches!(
|
||||
disk.read_version("", bucket, object, &v2.to_string(), &ReadOptions::default())
|
||||
.await,
|
||||
Err(DiskError::FileVersionNotFound | DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
let v1 = v1.to_string();
|
||||
let v2 = v2.to_string();
|
||||
let is_no_such_version =
|
||||
|err: &StorageError| matches!(err, StorageError::VersionNotFound(b, o, v) if b == bucket && o == object && v == &v2);
|
||||
let Err(err) = head(&set, bucket, object, Some(&v2)).await else {
|
||||
panic!("HEAD must not serve a dangling version");
|
||||
};
|
||||
assert!(is_no_such_version(&err), "HEAD of a dangling version must be NoSuchVersion: {err:?}");
|
||||
let Err(err) = get(&set, bucket, object, Some(&v2)).await else {
|
||||
panic!("GET must not serve a dangling version");
|
||||
};
|
||||
assert!(is_no_such_version(&err), "GET of a dangling version must be NoSuchVersion: {err:?}");
|
||||
|
||||
assert_eq!(
|
||||
get(&set, bucket, object, Some(&v1))
|
||||
.await
|
||||
.expect("the committed version must stay readable"),
|
||||
b"version one"
|
||||
);
|
||||
head(&set, bucket, object, Some(&v1))
|
||||
.await
|
||||
.expect("HEAD of the committed version must succeed");
|
||||
// The latest read sees two generations (version two on three drives,
|
||||
// version one on the rest), which is not proof of absence.
|
||||
let Err(err) = head(&set, bucket, object, None).await else {
|
||||
panic!("a split latest version must not resolve without quorum");
|
||||
};
|
||||
assert!(is_read_quorum_error(&err), "a split latest version keeps the quorum error: {err:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_classification_keeps_quorum_error_for_corrupt_metadata() {
|
||||
let bucket = "dangling-read-corrupt";
|
||||
let object = "torn/object";
|
||||
let (dirs, set) = dangling_test_set(bucket).await;
|
||||
put(&set, bucket, object, b"corrupt copies are present, not absent", false).await;
|
||||
for disk in 3..DRIVES {
|
||||
tokio::fs::write(object_dir(&dirs, disk, bucket, object).join(STORAGE_FORMAT_FILE), b"corrupt metadata")
|
||||
.await
|
||||
.expect("xl.meta should be corruptible for the test");
|
||||
}
|
||||
|
||||
assert_get_and_head_keep_read_quorum_error(&set, bucket, object, "corrupt xl.meta").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_classification_keeps_quorum_error_for_two_generations() {
|
||||
let bucket = "dangling-read-two-generations";
|
||||
let object = "torn/object";
|
||||
let (dirs, set) = dangling_test_set(bucket).await;
|
||||
put(&set, bucket, object, b"first generation", false).await;
|
||||
let mut first_generation = Vec::new();
|
||||
for disk in 0..2 {
|
||||
let path = object_dir(&dirs, disk, bucket, object).join(STORAGE_FORMAT_FILE);
|
||||
let bytes = tokio::fs::read(&path)
|
||||
.await
|
||||
.expect("first generation xl.meta should be readable");
|
||||
first_generation.push((path, bytes));
|
||||
}
|
||||
put(&set, bucket, object, b"second generation", false).await;
|
||||
// First generation on two drives, second on one, three drives absent:
|
||||
// each generation is below read quorum and neither proves the other absent.
|
||||
for (path, bytes) in first_generation {
|
||||
tokio::fs::write(&path, bytes)
|
||||
.await
|
||||
.expect("first generation xl.meta should be restorable");
|
||||
}
|
||||
remove_object_from(&dirs, bucket, object, &[3, 4, 5]).await;
|
||||
|
||||
assert_get_and_head_keep_read_quorum_error(&set, bucket, object, "two generations").await;
|
||||
}
|
||||
|
||||
/// Writes `fi` as the only version on the first `copies` drives; the
|
||||
/// remaining drives answer FileNotFound.
|
||||
async fn write_metadata_copies(set: &SetDisks, bucket: &str, object: &str, fi: &FileInfo, copies: usize) {
|
||||
let disks = set.disks.read().await.clone();
|
||||
for (index, disk) in disks.iter().take(copies).enumerate() {
|
||||
let mut copy = fi.clone();
|
||||
copy.erasure.index = index + 1;
|
||||
disk.as_ref()
|
||||
.expect("every drive should be online")
|
||||
.write_metadata(bucket, bucket, object, copy)
|
||||
.await
|
||||
.expect("hand-written metadata should be stored");
|
||||
}
|
||||
}
|
||||
|
||||
fn handwritten_version(bucket: &str, object: &str, data_blocks: usize, parity_blocks: usize) -> FileInfo {
|
||||
let mut fi = FileInfo::new(object, data_blocks, parity_blocks);
|
||||
fi.volume = bucket.to_string();
|
||||
fi.name = object.to_string();
|
||||
fi.size = 1;
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
fi.metadata.insert("etag".to_string(), "handwritten-etag".to_string());
|
||||
fi.add_object_part(1, "part-etag".to_string(), 1, None, 1, None, None);
|
||||
fi
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_zero_parity_version_reads_as_not_found() {
|
||||
// An EC:0 version on a default EC:2 set passes the common-parity check
|
||||
// with four of six copies (n/2 + 1), so its read fails at the later
|
||||
// read-quorum reduction, which must classify it as well.
|
||||
let bucket = "dangling-read-zero-parity";
|
||||
let object = "torn/zero-parity";
|
||||
let (_dirs, set) = dangling_test_set(bucket).await;
|
||||
write_metadata_copies(&set, bucket, object, &handwritten_version(bucket, object, DRIVES, 0), 4).await;
|
||||
|
||||
let is_not_found = |err: &StorageError| matches!(err, StorageError::ObjectNotFound(b, o) if b == bucket && o == object);
|
||||
let Err(err) = head(&set, bucket, object, None).await else {
|
||||
panic!("HEAD must not serve a zero-parity object missing two shards");
|
||||
};
|
||||
assert!(is_not_found(&err), "HEAD of a dangling zero-parity object must be NoSuchKey: {err:?}");
|
||||
let Err(err) = get(&set, bucket, object, None).await else {
|
||||
panic!("GET must not serve a zero-parity object missing two shards");
|
||||
};
|
||||
assert!(is_not_found(&err), "GET of a dangling zero-parity object must be NoSuchKey: {err:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_classification_keeps_quorum_error_for_remote_versions() {
|
||||
// Local metadata is the only pointer to tiered data, and heal never
|
||||
// deletes a remote candidate as dangling, so GET must not report it absent.
|
||||
let bucket = "dangling-read-remote";
|
||||
let object = "torn/transitioned";
|
||||
let (_dirs, set) = dangling_test_set(bucket).await;
|
||||
let mut fi = handwritten_version(bucket, object, DRIVES - PARITY, PARITY);
|
||||
fi.transition_status = crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string();
|
||||
fi.transition_tier = "WARM".to_string();
|
||||
fi.transitioned_objname = "remote/torn/transitioned".to_string();
|
||||
assert!(fi.is_remote(), "the fixture must be a remote version");
|
||||
write_metadata_copies(&set, bucket, object, &fi, 3).await;
|
||||
|
||||
assert_get_and_head_keep_read_quorum_error(&set, bucket, object, "remote version").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dangling_classification_skips_the_meta_bucket() {
|
||||
// Like MinIO, internal objects are never classified: their readers
|
||||
// treat not-found as absent configuration.
|
||||
let object = "config/dangling-read.json";
|
||||
let (dirs, set) = dangling_test_set("dangling-read-meta").await;
|
||||
put(&set, RUSTFS_META_BUCKET, object, b"internal config", false).await;
|
||||
remove_object_from(&dirs, RUSTFS_META_BUCKET, object, &[3, 4, 5]).await;
|
||||
|
||||
assert_get_and_head_keep_read_quorum_error(&set, RUSTFS_META_BUCKET, object, "meta bucket").await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2167,6 +2167,210 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn bucket_incarnation_fence_survives_caller_abort_at_real_repair_commit() {
|
||||
use crate::set_disk::ReadRepairPauseScope;
|
||||
|
||||
let (_root, store, shutdown) = multi_pool_heal_store().await;
|
||||
let bucket = format!("heal-commit-owner-{}", Uuid::new_v4().simple());
|
||||
let object = "history.bin";
|
||||
let version = Uuid::new_v4().to_string();
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("original bucket");
|
||||
let old = store
|
||||
.bucket_incarnation_id_from_disk(&bucket)
|
||||
.await
|
||||
.expect("original identity");
|
||||
let set = store.pools[0].get_disks(0);
|
||||
let mut reader = PutObjReader::from_vec(vec![0x17; 1024 * 1024]);
|
||||
set.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.clone()),
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("original acknowledged version");
|
||||
let missing = set.disks.read().await[0].clone().expect("repair target");
|
||||
let healthy = set.disks.read().await[1].clone().expect("healthy target");
|
||||
let baseline = healthy
|
||||
.read_all(&bucket, &format!("{object}/xl.meta"))
|
||||
.await
|
||||
.expect("healthy metadata baseline");
|
||||
let original = healthy
|
||||
.read_version("", &bucket, object, &version, &Default::default())
|
||||
.await
|
||||
.expect("original version metadata");
|
||||
assert!(original.data.is_none(), "the commit barrier fixture must use actual external data shards");
|
||||
missing
|
||||
.delete(
|
||||
&bucket,
|
||||
object,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("remove one physical shard");
|
||||
|
||||
// The scope pauses the real repair after reconstruction, immediately
|
||||
// before its generation check and final shard rename.
|
||||
let (_pause, arrived, release) = ReadRepairPauseScope::install_incarnation_commit(&bucket, object);
|
||||
let (drained_tx, drained_rx) = tokio::sync::oneshot::channel();
|
||||
let worker_store = store.clone();
|
||||
let worker_bucket = bucket.clone();
|
||||
let worker_version = version.clone();
|
||||
let opts = HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
|
||||
..Default::default()
|
||||
};
|
||||
let mut waiter = tokio::spawn(async move {
|
||||
worker_store
|
||||
.run_bucket_heal_at_incarnation(&worker_bucket, old, &opts, move |store, bucket, opts| async move {
|
||||
let result = store.heal_object_with_proof(&bucket, object, &worker_version, &opts).await?;
|
||||
drained_tx
|
||||
.send(result)
|
||||
.expect("retain the actual storage result after caller cancellation");
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::select! {
|
||||
paused = tokio::time::timeout(std::time::Duration::from_secs(10), arrived.notified()) => {
|
||||
paused.expect("real missing-shard repair reaches the final commit pause");
|
||||
}
|
||||
completed = &mut waiter => panic!("repair ended before the actual final commit barrier: {completed:?}"),
|
||||
}
|
||||
assert!(
|
||||
missing.read_xl(&bucket, object, false).await.is_err(),
|
||||
"final shard is not published at the pause"
|
||||
);
|
||||
println!("incarnation_commit_barrier: final_rename_paused");
|
||||
waiter.abort();
|
||||
assert!(waiter.await.expect_err("caller aborted").is_cancelled());
|
||||
let mut writer = Box::pin(store.acquire_bucket_lifecycle_write_lock(&bucket));
|
||||
assert!(
|
||||
futures::poll!(&mut writer).is_pending(),
|
||||
"bucket deletion/recreation cannot acquire its generation writer while the cancelled repair still owns commit"
|
||||
);
|
||||
println!("incarnation_commit_barrier: caller_aborted_writer_pending");
|
||||
release.notify_one();
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(10), drained_rx)
|
||||
.await
|
||||
.expect("actual repair drains after release")
|
||||
.expect("retained repair result");
|
||||
assert!(
|
||||
result.error.is_none(),
|
||||
"physical repair must succeed before lifecycle handoff: {:?}",
|
||||
result.error
|
||||
);
|
||||
let guard = tokio::time::timeout(std::time::Duration::from_secs(10), writer)
|
||||
.await
|
||||
.expect("generation writer acquires only after physical repair drains")
|
||||
.expect("generation writer");
|
||||
assert!(
|
||||
missing.read_xl(&bucket, object, false).await.is_ok(),
|
||||
"the physical shard is published before writer admission"
|
||||
);
|
||||
let repaired = missing
|
||||
.read_version("", &bucket, object, &version, &Default::default())
|
||||
.await
|
||||
.expect("repaired exact version metadata");
|
||||
assert_eq!(repaired.version_id, original.version_id);
|
||||
assert_eq!(repaired.data_dir, original.data_dir);
|
||||
let checked = missing
|
||||
.check_parts(&bucket, object, &repaired)
|
||||
.await
|
||||
.expect("verify the published physical shard");
|
||||
assert_eq!(
|
||||
checked.results,
|
||||
vec![crate::disk::CHECK_PART_SUCCESS],
|
||||
"the actual published part passes bitrot verification"
|
||||
);
|
||||
assert_eq!(
|
||||
healthy
|
||||
.read_all(&bucket, &format!("{object}/xl.meta"))
|
||||
.await
|
||||
.expect("healthy after repair"),
|
||||
baseline,
|
||||
"repair does not overwrite the surviving version"
|
||||
);
|
||||
println!("incarnation_commit_barrier: repair_drained_writer_acquired");
|
||||
drop(guard);
|
||||
|
||||
set.delete_object(
|
||||
&bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("normal explicit deletion of the old version");
|
||||
store
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("normal empty bucket deletion");
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("same-name successor bucket");
|
||||
let new = store
|
||||
.bucket_incarnation_id_from_disk(&bucket)
|
||||
.await
|
||||
.expect("successor identity");
|
||||
assert_ne!(old, new);
|
||||
let successor_version = Uuid::new_v4().to_string();
|
||||
let mut successor_reader = PutObjReader::from_vec(vec![0x29; 512 * 1024]);
|
||||
set.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut successor_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(successor_version),
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("acknowledged successor version");
|
||||
let successor = healthy
|
||||
.read_all(&bucket, &format!("{object}/xl.meta"))
|
||||
.await
|
||||
.expect("successor metadata");
|
||||
assert!(
|
||||
store
|
||||
.heal_object_at_incarnation(&bucket, object, &version, old, &opts)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
healthy
|
||||
.read_all(&bucket, &format!("{object}/xl.meta"))
|
||||
.await
|
||||
.expect("successor after obsolete admission"),
|
||||
successor,
|
||||
"obsolete admission cannot change the same-name successor"
|
||||
);
|
||||
println!("incarnation_commit_barrier: successor_rejects_obsolete_admission");
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn bucket_incarnation_heal_owner_survives_cancelled_waiter_and_queued_writer() {
|
||||
@@ -3221,9 +3425,6 @@ mod tests {
|
||||
#[serial_test::serial]
|
||||
async fn full_format_heal_lost_after_last_save_does_not_publish_or_renew_and_latches() {
|
||||
let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), store.pools[0].stop_endpoint_monitor_for_test())
|
||||
.await
|
||||
.expect("endpoint reconnection must stop before the format-loss fixture takes ownership");
|
||||
let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
|
||||
store.pools[0].disk_set[0].disks.write().await[3] = None;
|
||||
let barrier = HealFormatAfterSaveBarrier::install(&store.pools[0], 3);
|
||||
|
||||
@@ -5090,8 +5090,8 @@ mod tests {
|
||||
assert!(Arc::ptr_eq(disk, expected), "endpoint monitor must not replace an offline test handle");
|
||||
assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
|
||||
assert!(
|
||||
!disk.is_online().await,
|
||||
"the explicit drive fault must survive renewal despite a valid disk identity"
|
||||
disk.is_online().await,
|
||||
"the valid disk identity must prevent endpoint renewal while the IO health gate remains offline"
|
||||
);
|
||||
}
|
||||
OfflineTestDisks { disks }
|
||||
@@ -12018,9 +12018,7 @@ mod tests {
|
||||
Ok::<(), Error>(())
|
||||
}
|
||||
})
|
||||
// Pagination, not write throughput, is the oracle. Drain each pair of
|
||||
// durable receipts before preparing the next to bound fixture IO.
|
||||
.buffer_unordered(1)
|
||||
.buffer_unordered(32)
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.expect("more than one receipt page should persist");
|
||||
|
||||
+1931
-1191
File diff suppressed because it is too large
Load Diff
@@ -14,14 +14,14 @@
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::disk::{DiskAPI, STORAGE_FORMAT_FILE, WalkDirOptions};
|
||||
use crate::disk::STORAGE_FORMAT_FILE;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::{FileInfo, MetacacheReader, ObjectPartInfo};
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo};
|
||||
use std::collections::BTreeMap;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncWrite, DuplexStream, duplex};
|
||||
use tokio::io::{AsyncWrite, DuplexStream};
|
||||
use tokio::sync::Notify;
|
||||
use tokio::time::timeout;
|
||||
|
||||
@@ -468,50 +468,6 @@ async fn list_objects_shared_corpus_offline_corrupt_and_minority_stale() {
|
||||
fixture.assert_walk("", None, None, 3).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn list_objects_shared_corpus_quorum_delete_leaves_no_resurrectable_stale_copy() {
|
||||
let mut fixture = Fixture::new("list-corpus-stale-copies", namespace(&shared_keys()), true).await;
|
||||
// A delete that reached write quorum (three of four drives) leaves one
|
||||
// stale copy, the most any quorum delete can leave on this set.
|
||||
let deleted = ["a0", "z"];
|
||||
for key in deleted {
|
||||
let physical_key = rustfs_utils::path::encode_dir_object(key);
|
||||
for dir in fixture.dirs.iter().take(3) {
|
||||
tokio::fs::remove_dir_all(dir.path().join(&fixture.bucket).join(&physical_key))
|
||||
.await
|
||||
.expect("quorum delete fixture should remove the object directory");
|
||||
}
|
||||
}
|
||||
fixture.objects.retain(|object| !deleted.contains(&object.key.as_str()));
|
||||
|
||||
temp_env::async_with_vars([(ENV_API_LIST_OBJECTS_QUORUM, Some("strict"))], async {
|
||||
for max in [1, 3, 1000] {
|
||||
fixture.assert_walk("", None, None, max).await;
|
||||
fixture.assert_walk("", Some("/"), None, max).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
// A sampling mode promises only that no committed key goes missing.
|
||||
temp_env::async_with_vars([(ENV_API_LIST_OBJECTS_QUORUM, Some("optimal"))], async {
|
||||
let mut listed = Vec::new();
|
||||
let mut marker = None;
|
||||
for _ in 0..=fixture.objects.len() + deleted.len() {
|
||||
let page = fixture.page("", marker.clone(), None, 3).await;
|
||||
listed.extend(page.objects.into_iter().map(|object| object.name));
|
||||
if !page.is_truncated {
|
||||
break;
|
||||
}
|
||||
marker = page.next_marker;
|
||||
}
|
||||
for object in &fixture.objects {
|
||||
assert!(listed.contains(&object.key), "optimal LIST lost committed key {}", object.key);
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_objects_shared_corpus_versions_delete_markers_and_free_versions() {
|
||||
let mut objects = vec![
|
||||
|
||||
@@ -446,6 +446,7 @@ fn bench_metacache_resolve(c: &mut Criterion) {
|
||||
requested_versions: 1,
|
||||
bucket: "bucket".to_string(),
|
||||
strict: false,
|
||||
write_quorum_slack: 0,
|
||||
candidates: Vec::new(),
|
||||
});
|
||||
black_box(resolved);
|
||||
@@ -462,6 +463,24 @@ fn bench_metacache_resolve(c: &mut Criterion) {
|
||||
requested_versions: 1,
|
||||
bucket: "bucket".to_string(),
|
||||
strict: false,
|
||||
write_quorum_slack: 0,
|
||||
candidates: Vec::new(),
|
||||
});
|
||||
black_box(resolved);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
group.bench_function("8_disks_write_quorum/1000_objects", |b| {
|
||||
b.iter(|| {
|
||||
for entries in &sets_8_agree {
|
||||
let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams {
|
||||
dir_quorum: 4,
|
||||
obj_quorum: 4,
|
||||
requested_versions: 1,
|
||||
bucket: "bucket".to_string(),
|
||||
strict: true,
|
||||
write_quorum_slack: 0,
|
||||
candidates: Vec::new(),
|
||||
});
|
||||
black_box(resolved);
|
||||
@@ -472,12 +491,13 @@ fn bench_metacache_resolve(c: &mut Criterion) {
|
||||
group.bench_function("8_disks_disagree_merge/1000_objects", |b| {
|
||||
b.iter(|| {
|
||||
for entries in &sets_8_disagree {
|
||||
let resolved = entries.resolve(MetadataResolutionParams {
|
||||
let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams {
|
||||
dir_quorum: 4,
|
||||
obj_quorum: 4,
|
||||
requested_versions: 1,
|
||||
bucket: "bucket".to_string(),
|
||||
strict: true,
|
||||
write_quorum_slack: 0,
|
||||
candidates: Vec::new(),
|
||||
});
|
||||
black_box(resolved);
|
||||
@@ -488,12 +508,13 @@ fn bench_metacache_resolve(c: &mut Criterion) {
|
||||
group.bench_function("8_disks_agree_13_versions/1000_objects", |b| {
|
||||
b.iter(|| {
|
||||
for entries in &sets_8_agree_13v {
|
||||
let resolved = entries.resolve(MetadataResolutionParams {
|
||||
let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams {
|
||||
dir_quorum: 4,
|
||||
obj_quorum: 4,
|
||||
requested_versions: 1,
|
||||
bucket: "bucket".to_string(),
|
||||
strict: true,
|
||||
write_quorum_slack: 0,
|
||||
candidates: Vec::new(),
|
||||
});
|
||||
black_box(resolved);
|
||||
@@ -504,12 +525,13 @@ fn bench_metacache_resolve(c: &mut Criterion) {
|
||||
group.bench_function("8_disks_disagree_13_versions/1000_objects", |b| {
|
||||
b.iter(|| {
|
||||
for entries in &sets_8_disagree_13v {
|
||||
let resolved = entries.resolve(MetadataResolutionParams {
|
||||
let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams {
|
||||
dir_quorum: 4,
|
||||
obj_quorum: 4,
|
||||
requested_versions: 1,
|
||||
bucket: "bucket".to_string(),
|
||||
strict: true,
|
||||
write_quorum_slack: 0,
|
||||
candidates: Vec::new(),
|
||||
});
|
||||
black_box(resolved);
|
||||
@@ -574,6 +596,191 @@ fn bench_metacache_resolve(c: &mut Criterion) {
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn listing_history(versions: usize, cached: bool) -> MetaCacheEntry {
|
||||
let mut metadata = FileMeta::new();
|
||||
for index in 0..versions {
|
||||
let (data, parity) = if index + 1 == versions { (12, 4) } else { (8, 8) };
|
||||
let mut info = FileInfo::new("object", data, parity);
|
||||
info.volume = "bucket".to_owned();
|
||||
info.erasure.index = 1;
|
||||
info.data_dir = Some(Uuid::from_u128(0x1000 + u128::try_from(index).expect("data directory index")));
|
||||
info.versioned = true;
|
||||
info.version_id = Some(Uuid::from_u128(u128::try_from(index + 1).expect("version index")));
|
||||
info.mod_time = Some(
|
||||
OffsetDateTime::from_unix_timestamp(1_700_000_000 + i64::try_from(index).expect("timestamp")).expect("fixture time"),
|
||||
);
|
||||
info.size = 1;
|
||||
info.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
}];
|
||||
info.metadata.insert("etag".to_owned(), format!("etag-{index}"));
|
||||
info.metadata.insert("x-amz-meta-description".to_owned(), "m".repeat(1024));
|
||||
metadata.add_version(info).expect("valid version history");
|
||||
}
|
||||
MetaCacheEntry {
|
||||
name: "object".to_owned(),
|
||||
metadata: metadata.marshal_msg().expect("serialize history"),
|
||||
cached: cached.then_some(metadata),
|
||||
reusable: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn owned_agreed_latest_reference(
|
||||
mut entry: MetaCacheEntry,
|
||||
replicas: usize,
|
||||
params: &rustfs_filemeta::MetadataResolutionParams,
|
||||
enforce_write_quorum: bool,
|
||||
) -> rustfs_filemeta::MetaCacheEntryResolution {
|
||||
use rustfs_filemeta::MetaCacheEntryResolution::{NeedsSupplement, Rejected, Resolved};
|
||||
if !enforce_write_quorum {
|
||||
return Resolved(entry);
|
||||
}
|
||||
if entry.is_dir() {
|
||||
return if replicas >= params.dir_quorum {
|
||||
Resolved(entry)
|
||||
} else {
|
||||
Rejected
|
||||
};
|
||||
}
|
||||
if replicas < params.obj_quorum {
|
||||
return Rejected;
|
||||
}
|
||||
let metadata = match entry.xl_meta_ref() {
|
||||
Ok(metadata) => metadata,
|
||||
Err(_) if params.obj_quorum <= 1 => return Resolved(entry),
|
||||
Err(_) => return Rejected,
|
||||
};
|
||||
if params.requested_versions == 1
|
||||
&& let Some(first) = metadata.versions.first()
|
||||
&& !first.header.free_version()
|
||||
&& replicas >= first.listing_quorum(params.obj_quorum, params.write_quorum_slack)
|
||||
{
|
||||
return Resolved(entry);
|
||||
}
|
||||
// These fixtures cover the fast prefix, never older-version reconciliation.
|
||||
NeedsSupplement(entry)
|
||||
}
|
||||
|
||||
fn bench_listing_borrowed_metadata(c: &mut Criterion) {
|
||||
use rustfs_filemeta::{FileMetaVersion, MetaCacheEntries, MetadataResolutionParams};
|
||||
let mut group = c.benchmark_group("listing_borrowed_metadata");
|
||||
group
|
||||
.sample_size(20)
|
||||
.warm_up_time(Duration::from_millis(150))
|
||||
.measurement_time(Duration::from_millis(700));
|
||||
for versions in [2, 16, 256] {
|
||||
for cached in [false, true] {
|
||||
let entry = listing_history(versions, cached);
|
||||
let params = MetadataResolutionParams {
|
||||
dir_quorum: 8,
|
||||
obj_quorum: 8,
|
||||
requested_versions: 1,
|
||||
write_quorum_slack: 8,
|
||||
bucket: "bucket".to_owned(),
|
||||
..Default::default()
|
||||
};
|
||||
let slots = vec![Some(&entry); 8];
|
||||
let owned = MetaCacheEntries(vec![Some(entry.clone()); 8])
|
||||
.resolve_with_write_quorum(params.clone())
|
||||
.expect("previous committed version");
|
||||
let borrowed =
|
||||
MetaCacheEntries::resolve_borrowed_with_write_quorum(&slots, params.clone()).expect("borrowed previous version");
|
||||
assert_eq!(owned.metadata, borrowed.metadata);
|
||||
let id = format!("{versions}_versions/{}", if cached { "cached" } else { "cold" });
|
||||
group.throughput(Throughput::Bytes(u64::try_from(entry.metadata.len()).expect("metadata size")));
|
||||
// ABBA order permits an explicit baseline-drift check for each fixture.
|
||||
for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] {
|
||||
group.bench_function(BenchmarkId::new("replicas", format!("{id}/{phase}")), |b| {
|
||||
b.iter(|| {
|
||||
if phase.starts_with("owned") {
|
||||
black_box(
|
||||
MetaCacheEntries(vec![Some(black_box(&entry).clone()); 8])
|
||||
.resolve_with_write_quorum(params.clone()),
|
||||
)
|
||||
} else {
|
||||
let slots = vec![Some(black_box(&entry)); 8];
|
||||
black_box(MetaCacheEntries::resolve_borrowed_with_write_quorum(&slots, params.clone()))
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
let mut agreed_metadata = entry
|
||||
.cached
|
||||
.clone()
|
||||
.unwrap_or_else(|| FileMeta::load(&entry.metadata).expect("agreed fixture"));
|
||||
let mut latest = FileMetaVersion::try_from(agreed_metadata.versions[0].meta.as_slice()).expect("latest body");
|
||||
let object = latest.object.as_mut().expect("latest object");
|
||||
object.erasure_m = 8;
|
||||
object.erasure_n = 8;
|
||||
agreed_metadata.versions[0] = latest.try_into().expect("balanced latest header");
|
||||
let agreed = MetaCacheEntry {
|
||||
name: entry.name.clone(),
|
||||
metadata: agreed_metadata.marshal_msg().expect("agreed history"),
|
||||
cached: cached.then_some(agreed_metadata),
|
||||
reusable: false,
|
||||
};
|
||||
for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] {
|
||||
group.bench_function(BenchmarkId::new("agreed", format!("{id}/{phase}")), |b| {
|
||||
b.iter_batched(
|
||||
|| agreed.clone(),
|
||||
|candidate| {
|
||||
let resolved = if phase.starts_with("owned") {
|
||||
owned_agreed_latest_reference(candidate, 8, ¶ms, true)
|
||||
} else {
|
||||
rustfs_filemeta::resolve_agreed_entry(candidate, 8, ¶ms, true)
|
||||
};
|
||||
let rustfs_filemeta::MetaCacheEntryResolution::Resolved(candidate) = resolved else {
|
||||
panic!("balanced latest must use agreed fast prefix");
|
||||
};
|
||||
black_box(candidate)
|
||||
},
|
||||
BatchSize::SmallInput,
|
||||
);
|
||||
});
|
||||
}
|
||||
let replicas: Vec<_> = (1..=8)
|
||||
.map(|disk| {
|
||||
let mut metadata = entry
|
||||
.cached
|
||||
.clone()
|
||||
.unwrap_or_else(|| FileMeta::load(&entry.metadata).expect("decode fixture"));
|
||||
for version in &mut metadata.versions {
|
||||
let mut payload = FileMetaVersion::try_from(version.meta.as_slice()).expect("version body");
|
||||
payload.object.as_mut().expect("object body").erasure_index = disk;
|
||||
version.meta = payload.marshal_msg().expect("per-disk body");
|
||||
}
|
||||
MetaCacheEntry {
|
||||
name: entry.name.clone(),
|
||||
metadata: metadata.marshal_msg().expect("per-disk metadata"),
|
||||
cached: cached.then_some(metadata),
|
||||
reusable: false,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
assert!(replicas[0].headers_agree(&replicas, false));
|
||||
for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] {
|
||||
group.bench_function(BenchmarkId::new("headers", format!("{id}/{phase}")), |b| {
|
||||
b.iter(|| {
|
||||
let agreed = if phase.starts_with("owned") {
|
||||
replicas
|
||||
.iter()
|
||||
.all(|candidate| replicas[0].matches(Some(black_box(candidate)), false).1)
|
||||
} else {
|
||||
replicas[0].headers_agree(black_box(&replicas), false)
|
||||
};
|
||||
black_box(agreed)
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
bench_create_real_xlmeta,
|
||||
@@ -590,7 +797,8 @@ criterion_group!(
|
||||
bench_list_page_to_fileinfo,
|
||||
bench_delete_inline_versions,
|
||||
bench_metacache_stream_page,
|
||||
bench_metacache_resolve
|
||||
bench_metacache_resolve,
|
||||
bench_listing_borrowed_metadata
|
||||
);
|
||||
|
||||
criterion_main!(benches);
|
||||
|
||||
@@ -764,17 +764,6 @@ pub(crate) fn is_valid_distribution(distribution: &[usize], n: usize) -> bool {
|
||||
}
|
||||
|
||||
impl FileInfo {
|
||||
pub fn distribution_start(bucket: &str, object: &str, cardinality: usize) -> usize {
|
||||
if cardinality == 0 {
|
||||
return 0;
|
||||
}
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(bucket.as_bytes());
|
||||
hasher.update(b"/");
|
||||
hasher.update(object.as_bytes());
|
||||
(hasher.finalize() as u32 as usize) % cardinality
|
||||
}
|
||||
|
||||
pub fn new(object: &str, data_blocks: usize, parity_blocks: usize) -> Self {
|
||||
let indices = {
|
||||
let cardinality = data_blocks + parity_blocks;
|
||||
|
||||
@@ -115,7 +115,6 @@ pub(crate) fn data_key_for_version(version_id: Option<Uuid>) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn legacy_data_key_for_version(version_id: Option<Uuid>) -> Option<String> {
|
||||
if version_id.is_none() || version_id == Some(Uuid::nil()) {
|
||||
Some(Uuid::nil().to_string())
|
||||
@@ -155,6 +154,7 @@ pub fn is_skip_meta_key(key: &str) -> bool {
|
||||
}
|
||||
|
||||
mod codec;
|
||||
pub(crate) use codec::{BorrowedFileMeta, reconcile_headers};
|
||||
mod inline_data;
|
||||
pub(crate) mod msgp_decode;
|
||||
mod validation;
|
||||
@@ -348,21 +348,19 @@ impl FileMeta {
|
||||
self.versions.sort_by(cmp_shallow_versions_for_order);
|
||||
}
|
||||
|
||||
fn find_inline_data_slice(inline_data: &[u8], version_id: Option<Uuid>) -> Result<Option<&[u8]>> {
|
||||
if let Some(vid) = version_id.filter(|v| !v.is_nil()) {
|
||||
let mut buf = Uuid::encode_buffer();
|
||||
let key = vid.hyphenated().encode_lower(&mut buf);
|
||||
InlineData::find_slice_in_raw(inline_data, key)
|
||||
} else {
|
||||
if let Some(data) = InlineData::find_slice_in_raw(inline_data, NULL_VERSION_ID)? {
|
||||
return Ok(Some(data));
|
||||
}
|
||||
InlineData::find_slice_in_raw(inline_data, "00000000-0000-0000-0000-000000000000")
|
||||
}
|
||||
}
|
||||
|
||||
fn find_inline_data_in_slice(inline_data: &[u8], version_id: Option<Uuid>) -> Result<Option<Vec<u8>>> {
|
||||
Ok(Self::find_inline_data_slice(inline_data, version_id)?.map(<[u8]>::to_vec))
|
||||
let key = data_key_for_version(version_id);
|
||||
if let Some(data) = InlineData::find_in_slice(inline_data, key.as_str())? {
|
||||
return Ok(Some(data));
|
||||
}
|
||||
|
||||
if let Some(legacy_key) = legacy_data_key_for_version(version_id)
|
||||
&& legacy_key != key
|
||||
{
|
||||
return InlineData::find_in_slice(inline_data, legacy_key.as_str());
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn find_inline_data_for_version(&self, version_id: Option<Uuid>) -> Result<Option<Vec<u8>>> {
|
||||
@@ -2429,6 +2427,52 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_inline_xlmeta_is_file_corrupt_only_for_supported_formats() {
|
||||
let mut meta = FileMeta::new();
|
||||
meta.data
|
||||
.replace("fixture", vec![0x5a; 4095])
|
||||
.expect("seed real inline framing");
|
||||
let inline_len = meta.data.as_slice().len();
|
||||
let original = meta.marshal_msg().expect("serialize valid inline metadata");
|
||||
let inline_start = original.len() - inline_len;
|
||||
let parsed = FileMeta::load(&original).expect("complete inline metadata must decode");
|
||||
assert_eq!(parsed.data, meta.data);
|
||||
for cut in inline_start + 1..original.len() {
|
||||
assert_eq!(
|
||||
FileMeta::load(&original[..cut]).expect_err("incomplete inline framing must fail"),
|
||||
Error::FileCorrupt,
|
||||
"inline truncation at byte {cut} must remain repairable"
|
||||
);
|
||||
}
|
||||
|
||||
for inline_version in [0, 2] {
|
||||
let mut unsupported = original[..original.len() / 2].to_vec();
|
||||
unsupported[inline_start] = inline_version;
|
||||
assert_ne!(
|
||||
FileMeta::load(&unsupported).expect_err("malformed unsupported inline format must fail"),
|
||||
Error::FileCorrupt,
|
||||
"unknown inline version must not authorize repair"
|
||||
);
|
||||
}
|
||||
for (header_ver, meta_ver) in [
|
||||
(XL_HEADER_VERSION + 1, XL_META_VERSION),
|
||||
(XL_HEADER_VERSION, XL_META_VERSION + 1),
|
||||
] {
|
||||
let mut headers = Vec::new();
|
||||
rmp::encode::write_uint(&mut headers, u64::from(header_ver)).expect("write future header");
|
||||
rmp::encode::write_uint(&mut headers, u64::from(meta_ver)).expect("write future metadata version");
|
||||
rmp::encode::write_uint(&mut headers, 0).expect("write empty version count");
|
||||
let mut unsupported = build_xl_buffer(&headers);
|
||||
unsupported.extend_from_slice(&meta.data.as_slice()[..inline_len / 2]);
|
||||
assert_ne!(
|
||||
FileMeta::load(&unsupported).expect_err("unsupported metadata with truncated inline data must fail"),
|
||||
Error::FileCorrupt,
|
||||
"future metadata schema must not authorize repair"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_xlmeta_index_and_format_reads_are_file_corrupt() {
|
||||
let buf = FileMeta::default()
|
||||
|
||||
@@ -188,7 +188,13 @@ impl FileMeta {
|
||||
if !buf.is_empty() {
|
||||
InlineData::validate_slice(buf).map_err(|e| {
|
||||
error!("data validation failed: {}", e);
|
||||
e
|
||||
// Validation only examines bytes already read. Classify damage
|
||||
// as repairable corruption only for formats we understand.
|
||||
if InlineData::version_ok_slice(buf) && Self::decode_xl_headers(meta).is_ok() {
|
||||
Error::FileCorrupt
|
||||
} else {
|
||||
e
|
||||
}
|
||||
})?;
|
||||
}
|
||||
|
||||
|
||||
@@ -622,6 +622,18 @@ pub struct FileMetaShallowVersion {
|
||||
pub meta: Vec<u8>, // FileMetaVersion.marshal_msg
|
||||
}
|
||||
|
||||
fn write_quorum_from_erasure(data_blocks: usize, parity_blocks: usize) -> Option<usize> {
|
||||
if data_blocks == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(if data_blocks == parity_blocks {
|
||||
data_blocks.saturating_add(1)
|
||||
} else {
|
||||
data_blocks
|
||||
})
|
||||
}
|
||||
|
||||
impl FileMetaShallowVersion {
|
||||
/// Parse version meta with legacy format compatibility.
|
||||
/// Use this instead of `FileMetaVersion::default()` + `unmarshal_msg()` to handle old-version xl.meta.
|
||||
@@ -629,11 +641,60 @@ impl FileMetaShallowVersion {
|
||||
FileMetaVersion::try_from(self.meta.as_slice())
|
||||
}
|
||||
|
||||
pub fn write_quorum(&self, fallback_quorum: usize) -> usize {
|
||||
write_quorum_from_header_and_meta(&self.header, self.meta.as_slice(), fallback_quorum)
|
||||
}
|
||||
|
||||
/// Quorum for listing this version without accepting an unreadable payload.
|
||||
pub fn listing_quorum(&self, fallback_quorum: usize, unreachable_disks: usize) -> usize {
|
||||
listing_quorum_from_header_and_meta(&self.header, &self.meta, fallback_quorum, unreachable_disks)
|
||||
}
|
||||
|
||||
pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfo> {
|
||||
FileMetaVersion::decode_fileinfo_from_slice(self.meta.as_slice(), volume, path, all_parts, true)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn write_quorum_from_header_and_meta(header: &FileMetaVersionHeader, meta: &[u8], fallback_quorum: usize) -> usize {
|
||||
let header_quorum = header.write_quorum(fallback_quorum);
|
||||
if header.has_ec() || !matches!(header.version_type, VersionType::Object | VersionType::Legacy) {
|
||||
return header_quorum;
|
||||
}
|
||||
|
||||
FileMetaVersion::try_from(meta)
|
||||
.ok()
|
||||
.and_then(|version| version.erasure_write_quorum())
|
||||
.unwrap_or(header_quorum)
|
||||
}
|
||||
|
||||
pub(crate) fn listing_quorum_from_header_and_meta(
|
||||
header: &FileMetaVersionHeader,
|
||||
meta: &[u8],
|
||||
fallback_quorum: usize,
|
||||
unreachable_disks: usize,
|
||||
) -> usize {
|
||||
let geometry = if header.version_type == VersionType::Object && header.has_ec() {
|
||||
Some((usize::from(header.ec_m), usize::from(header.ec_n)))
|
||||
} else if matches!(header.version_type, VersionType::Object | VersionType::Legacy) {
|
||||
FileMetaVersion::try_from(meta)
|
||||
.ok()
|
||||
.and_then(|version| version.erasure_geometry())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let Some((data_blocks, parity_blocks)) = geometry else {
|
||||
return fallback_quorum;
|
||||
};
|
||||
let Some(write_quorum) = write_quorum_from_erasure(data_blocks, parity_blocks) else {
|
||||
return fallback_quorum;
|
||||
};
|
||||
write_quorum
|
||||
.saturating_sub(unreachable_disks)
|
||||
.max(data_blocks)
|
||||
.max(fallback_quorum)
|
||||
}
|
||||
|
||||
impl TryFrom<FileMetaVersion> for FileMetaShallowVersion {
|
||||
type Error = Error;
|
||||
|
||||
@@ -676,6 +737,26 @@ pub(super) struct V2ObjectSliceSummary {
|
||||
}
|
||||
|
||||
impl FileMetaVersion {
|
||||
fn erasure_geometry(&self) -> Option<(usize, usize)> {
|
||||
Some(match self.version_type {
|
||||
VersionType::Object | VersionType::Legacy => {
|
||||
if let Some(object) = &self.object {
|
||||
(object.erasure_m, object.erasure_n)
|
||||
} else if let Some(object) = &self.legacy_object {
|
||||
(object.erasure.data_blocks, object.erasure.parity_blocks)
|
||||
} else {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
fn erasure_write_quorum(&self) -> Option<usize> {
|
||||
let (data_blocks, parity_blocks) = self.erasure_geometry()?;
|
||||
write_quorum_from_erasure(data_blocks, parity_blocks)
|
||||
}
|
||||
|
||||
/// Fast borrowed scanner for `V2Obj` slices. Returning `Ok` guarantees
|
||||
/// `FileMetaVersion::try_from` would succeed with the identical summary fields;
|
||||
/// any non-`V2Obj` or invalid input returns `Err` so callers fall back to `try_from`.
|
||||
@@ -1694,6 +1775,14 @@ impl FileMetaVersionHeader {
|
||||
self.ec_m > 0 && self.ec_n > 0
|
||||
}
|
||||
|
||||
pub fn write_quorum(&self, fallback_quorum: usize) -> usize {
|
||||
if self.version_type != VersionType::Object || !self.has_ec() {
|
||||
return fallback_quorum;
|
||||
}
|
||||
|
||||
write_quorum_from_erasure(usize::from(self.ec_m), usize::from(self.ec_n)).unwrap_or(fallback_quorum)
|
||||
}
|
||||
|
||||
pub fn matches_not_strict(&self, o: &FileMetaVersionHeader) -> bool {
|
||||
let mut ok = self.version_id == o.version_id && self.version_type == o.version_type && self.matches_ec(o);
|
||||
// Disk-loaded headers keep the null version as Some(nil), not None: all null
|
||||
@@ -4078,49 +4167,15 @@ pub fn merge_file_meta_versions(
|
||||
.iter()
|
||||
.map(|v| v.iter().map(|ver| (&ver.header, ver.meta.as_slice())).collect())
|
||||
.collect();
|
||||
merge_file_meta_borrowed_streams(quorum, strict, requested_versions, streams)
|
||||
}
|
||||
|
||||
/// Picks the most common header among the tops that share `latest`'s version
|
||||
/// id; a tie keeps the header that sorts first (the newest).
|
||||
fn most_common_header_for_version_id<'a>(
|
||||
tops: &[(&'a FileMetaVersionHeader, &'a [u8])],
|
||||
mut latest: (&'a FileMetaVersionHeader, &'a [u8]),
|
||||
strict: bool,
|
||||
) -> ((&'a FileMetaVersionHeader, &'a [u8]), usize) {
|
||||
let version_id = latest.0.version_id;
|
||||
let comparable = |header: &FileMetaVersionHeader| {
|
||||
let mut header = header.clone();
|
||||
if !strict {
|
||||
header.signature = [0; 4];
|
||||
}
|
||||
header
|
||||
};
|
||||
let mut counts: HashMap<FileMetaVersionHeader, usize> = HashMap::new();
|
||||
for &top in tops {
|
||||
if top.0.version_id == version_id {
|
||||
*counts.entry(comparable(top.0)).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
let mut latest_count = 0;
|
||||
for (header, count) in &counts {
|
||||
if *count < latest_count || (*count == latest_count && latest.0.sorts_before(header)) {
|
||||
continue;
|
||||
}
|
||||
for &top in tops {
|
||||
if comparable(top.0) == *header {
|
||||
latest = top;
|
||||
}
|
||||
}
|
||||
latest_count = *count;
|
||||
}
|
||||
(latest, latest_count)
|
||||
merge_file_meta_borrowed_streams(quorum, strict, requested_versions, false, 0, streams)
|
||||
}
|
||||
|
||||
pub(crate) fn merge_file_meta_borrowed_streams(
|
||||
mut quorum: usize,
|
||||
mut strict: bool,
|
||||
requested_versions: usize,
|
||||
enforce_write_quorum: bool,
|
||||
write_quorum_slack: usize,
|
||||
mut streams: Vec<Vec<(&FileMetaVersionHeader, &[u8])>>,
|
||||
) -> Vec<FileMetaShallowVersion> {
|
||||
let to_shallow = |(header, meta): (&FileMetaVersionHeader, &[u8])| FileMetaShallowVersion {
|
||||
@@ -4137,13 +4192,32 @@ pub(crate) fn merge_file_meta_borrowed_streams(
|
||||
}
|
||||
|
||||
if streams.len() == 1 {
|
||||
return streams[0].iter().copied().map(to_shallow).collect();
|
||||
if !enforce_write_quorum {
|
||||
return streams[0].iter().copied().map(to_shallow).collect();
|
||||
}
|
||||
|
||||
let required_quorum = streams[0]
|
||||
.first()
|
||||
.map(|&(hdr, meta)| listing_quorum_from_header_and_meta(hdr, meta, quorum, write_quorum_slack))
|
||||
.unwrap_or(quorum);
|
||||
if streams.len() >= required_quorum {
|
||||
return streams[0].iter().copied().map(to_shallow).collect();
|
||||
}
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
if quorum == 1 {
|
||||
strict = true;
|
||||
}
|
||||
|
||||
let required_quorum = |(hdr, meta): (&FileMetaVersionHeader, &[u8])| {
|
||||
if enforce_write_quorum {
|
||||
listing_quorum_from_header_and_meta(hdr, meta, quorum, write_quorum_slack)
|
||||
} else {
|
||||
quorum
|
||||
}
|
||||
};
|
||||
|
||||
let mut tops: Vec<(&FileMetaVersionHeader, &[u8])> = Vec::with_capacity(streams.len());
|
||||
|
||||
let mut n_versions = 0;
|
||||
@@ -4175,7 +4249,7 @@ pub(crate) fn merge_file_meta_borrowed_streams(
|
||||
let latest: (&FileMetaVersionHeader, &[u8]);
|
||||
if consistent {
|
||||
latest = tops[0];
|
||||
if tops.len() >= quorum {
|
||||
if tops.len() >= required_quorum(latest) {
|
||||
merged.push(to_shallow(latest));
|
||||
if !latest.0.free_version() {
|
||||
n_versions += 1;
|
||||
@@ -4184,7 +4258,6 @@ pub(crate) fn merge_file_meta_borrowed_streams(
|
||||
} else {
|
||||
let mut latest_ref = tops[0];
|
||||
let mut latest_count = 1;
|
||||
let mut settled_by_version_id = false;
|
||||
for &ver in &tops[1..] {
|
||||
if ver.0 == latest_ref.0 {
|
||||
latest_count += 1;
|
||||
@@ -4208,26 +4281,39 @@ pub(crate) fn merge_file_meta_borrowed_streams(
|
||||
}
|
||||
|
||||
if ver.0.version_id == latest_ref.0.version_id {
|
||||
(latest_ref, latest_count) = most_common_header_for_version_id(&tops, latest_ref, strict);
|
||||
settled_by_version_id = true;
|
||||
let mut x: HashMap<FileMetaVersionHeader, usize> = HashMap::new();
|
||||
for &a in &tops {
|
||||
if a.0.version_id != ver.0.version_id {
|
||||
continue;
|
||||
}
|
||||
let mut hdr = a.0.clone();
|
||||
if !strict {
|
||||
hdr.signature = [0; 4];
|
||||
}
|
||||
*x.entry(hdr).or_insert(0) += 1;
|
||||
}
|
||||
latest_count = 0;
|
||||
for (k, v) in x.iter() {
|
||||
if *v < latest_count || (*v == latest_count && latest_ref.0.sorts_before(k)) {
|
||||
continue;
|
||||
}
|
||||
for &a in &tops {
|
||||
let mut hdr = a.0.clone();
|
||||
if !strict {
|
||||
hdr.signature = [0; 4];
|
||||
}
|
||||
if hdr == *k {
|
||||
latest_ref = a;
|
||||
}
|
||||
}
|
||||
|
||||
latest_count = *v;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Readers return their heads in any order. When every older header
|
||||
// sharing the newest header's version id (an interrupted null-version
|
||||
// overwrite) came first, the scan above never compared them, and the
|
||||
// stream trim below would then drop that older committed generation.
|
||||
if !settled_by_version_id
|
||||
&& tops.iter().any(|ver| {
|
||||
ver.0 != latest_ref.0
|
||||
&& ver.0.version_id == latest_ref.0.version_id
|
||||
&& (strict || !ver.0.matches_not_strict(latest_ref.0))
|
||||
})
|
||||
{
|
||||
(latest_ref, latest_count) = most_common_header_for_version_id(&tops, latest_ref, strict);
|
||||
}
|
||||
latest = latest_ref;
|
||||
if latest_count >= quorum {
|
||||
if latest_count >= required_quorum(latest) {
|
||||
if !latest.0.free_version() {
|
||||
n_versions += 1;
|
||||
}
|
||||
@@ -4419,14 +4505,14 @@ fn read_more_sync<R: std::io::Read>(
|
||||
}
|
||||
|
||||
let extra = read_size - has;
|
||||
buf.try_reserve(extra.saturating_add(1)).map_err(Error::other)?;
|
||||
std::io::Read::read_to_end(&mut std::io::Read::take(reader, extra as u64), buf)?;
|
||||
if buf.len() < read_size {
|
||||
return Err(Error::from(std::io::Error::new(
|
||||
std::io::ErrorKind::UnexpectedEof,
|
||||
"failed to fill whole buffer",
|
||||
)));
|
||||
if buf.capacity() >= read_size {
|
||||
// Extend the buffer if we have enough space.
|
||||
buf.resize(read_size, 0);
|
||||
} else {
|
||||
buf.extend(vec![0u8; extra]);
|
||||
}
|
||||
|
||||
reader.read_exact(&mut buf[has..])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -4448,8 +4534,8 @@ pub fn read_xl_meta_no_data_sync<R: std::io::Read>(reader: &mut R, size: usize)
|
||||
has_full = false;
|
||||
}
|
||||
|
||||
let mut buf = Vec::new();
|
||||
read_more_sync(reader, &mut buf, size, initial, false)?;
|
||||
let mut buf = vec![0u8; initial];
|
||||
reader.read_exact(&mut buf)?;
|
||||
|
||||
let (tmp_buf, major, minor) = FileMeta::check_xl2_v1(&buf)?;
|
||||
|
||||
@@ -5016,6 +5102,39 @@ mod tests {
|
||||
wr
|
||||
}
|
||||
|
||||
fn encode_legacy_v2_object_body(data_blocks: usize, parity_blocks: usize) -> Vec<u8> {
|
||||
let drive_count = data_blocks + parity_blocks;
|
||||
let payload = LegacyObjectVersionFixture {
|
||||
version_type: LegacyObjectVersionTypeFixture::Object,
|
||||
object: Some(LegacyObjectFixture {
|
||||
version_id: Some(sample_version_id().as_bytes().to_vec()),
|
||||
data_dir: Some(Uuid::from_u128(42).as_bytes().to_vec()),
|
||||
erasure_algorithm: "ReedSolomon".to_string(),
|
||||
erasure_m: data_blocks,
|
||||
erasure_n: parity_blocks,
|
||||
erasure_block_size: 1_048_576,
|
||||
erasure_index: 1,
|
||||
erasure_dist: (1..=drive_count)
|
||||
.map(|idx| u8::try_from(idx).expect("test drive index should fit u8"))
|
||||
.collect(),
|
||||
bitrot_checksum_algo: "HighwayHash".to_string(),
|
||||
part_numbers: vec![1],
|
||||
part_etags: vec!["etag-1".to_string()],
|
||||
part_sizes: vec![11],
|
||||
part_actual_sizes: vec![11],
|
||||
part_indices: vec![Vec::new()],
|
||||
size: 11,
|
||||
mod_time: Some(sample_mod_time()),
|
||||
meta_sys: HashMap::new(),
|
||||
meta_user: HashMap::from([("content-type".to_string(), "text/plain".to_string())]),
|
||||
}),
|
||||
delete_marker: None,
|
||||
write_version: 3,
|
||||
};
|
||||
|
||||
rmp_serde::to_vec_named(&payload).expect("legacy object payload should marshal")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_header_unmarshal_v1_uses_legacy_layout_defaults() {
|
||||
let expected = sample_header();
|
||||
@@ -5050,6 +5169,80 @@ mod tests {
|
||||
assert_eq!(decoded.ec_m, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shallow_version_write_quorum_uses_legacy_object_payload_when_header_lacks_ec() {
|
||||
let expected = sample_header();
|
||||
let encoded = encode_v2_header(&expected);
|
||||
let mut header = FileMetaVersionHeader::default();
|
||||
header.unmarshal_v(2, &encoded).expect("legacy v2 header should decode");
|
||||
|
||||
let version = FileMetaShallowVersion {
|
||||
header,
|
||||
meta: encode_legacy_v2_object_body(7, 1),
|
||||
};
|
||||
|
||||
assert_eq!(version.header.write_quorum(5), 5);
|
||||
assert_eq!(version.write_quorum(5), 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listing_quorum_uses_each_versions_data_floor() {
|
||||
for (data, parity, offline, fallback, expected) in
|
||||
[(8, 8, 0, 9, 9), (8, 8, 8, 8, 8), (12, 4, 8, 8, 12), (12, 4, 0, 9, 12)]
|
||||
{
|
||||
let version = FileMetaShallowVersion {
|
||||
header: FileMetaVersionHeader {
|
||||
version_type: VersionType::Object,
|
||||
ec_m: data,
|
||||
ec_n: parity,
|
||||
..Default::default()
|
||||
},
|
||||
// Modern headers must not decode the payload to decide quorum.
|
||||
meta: vec![0xff],
|
||||
};
|
||||
assert_eq!(version.listing_quorum(fallback, offline), expected);
|
||||
}
|
||||
let marker = FileMetaShallowVersion {
|
||||
header: FileMetaVersionHeader {
|
||||
version_type: VersionType::Delete,
|
||||
..Default::default()
|
||||
},
|
||||
meta: Vec::new(),
|
||||
};
|
||||
assert_eq!(marker.listing_quorum(9, 0), 9, "healthy marker needs majority metadata");
|
||||
assert_eq!(marker.listing_quorum(8, 8), 8, "degraded marker retains metadata floor");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listing_quorum_preserves_legacy_and_zero_parity_data_floor() {
|
||||
for (data, parity, fallback, expected) in [(12, 4, 8, 12), (8, 8, 8, 8), (4, 0, 2, 4)] {
|
||||
let version = FileMetaShallowVersion {
|
||||
header: FileMetaVersionHeader {
|
||||
version_type: VersionType::Object,
|
||||
..Default::default()
|
||||
},
|
||||
meta: encode_legacy_v2_object_body(data, parity),
|
||||
};
|
||||
assert_eq!(version.listing_quorum(fallback, 8), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shallow_version_write_quorum_uses_zero_parity_object_payload() {
|
||||
let expected = sample_header();
|
||||
let encoded = encode_v2_header(&expected);
|
||||
let mut header = FileMetaVersionHeader::default();
|
||||
header.unmarshal_v(2, &encoded).expect("legacy v2 header should decode");
|
||||
|
||||
let version = FileMetaShallowVersion {
|
||||
header,
|
||||
meta: encode_legacy_v2_object_body(4, 0),
|
||||
};
|
||||
|
||||
assert_eq!(version.header.write_quorum(2), 2);
|
||||
assert_eq!(version.write_quorum(2), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_header_unmarshal_v3_round_trips_current_layout() {
|
||||
let expected = sample_header();
|
||||
|
||||
@@ -168,11 +168,11 @@ impl InlineData {
|
||||
self.0.as_slice()
|
||||
}
|
||||
pub fn version_ok(&self) -> bool {
|
||||
if self.0.is_empty() {
|
||||
return true;
|
||||
}
|
||||
Self::version_ok_slice(&self.0)
|
||||
}
|
||||
|
||||
self.0[0] > 0 && self.0[0] <= INLINE_DATA_VER
|
||||
pub(crate) fn version_ok_slice(raw: &[u8]) -> bool {
|
||||
raw.first().is_none_or(|version| *version > 0 && *version <= INLINE_DATA_VER)
|
||||
}
|
||||
|
||||
pub fn after_version(&self) -> &[u8] {
|
||||
@@ -230,7 +230,7 @@ impl InlineData {
|
||||
Self::find_in_slice(&self.0, key)
|
||||
}
|
||||
|
||||
pub(crate) fn find_slice_in_raw<'a>(raw: &'a [u8], key: &str) -> Result<Option<&'a [u8]>> {
|
||||
pub(crate) fn find_in_slice(raw: &[u8], key: &str) -> Result<Option<Vec<u8>>> {
|
||||
if raw.is_empty() || raw[0] == 0 || raw[0] > INLINE_DATA_VER {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -243,17 +243,13 @@ impl InlineData {
|
||||
fields_len -= 1;
|
||||
let (field, value) = Self::read_entry_slice(buf, &mut cur, false)?;
|
||||
if field == key {
|
||||
return Ok(Some(value));
|
||||
return Ok(Some(value.to_vec()));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub(crate) fn find_in_slice(raw: &[u8], key: &str) -> Result<Option<Vec<u8>>> {
|
||||
Ok(Self::find_slice_in_raw(raw, key)?.map(<[u8]>::to_vec))
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
Self::validate_slice(&self.0)
|
||||
}
|
||||
|
||||
+741
-114
File diff suppressed because it is too large
Load Diff
@@ -683,17 +683,14 @@ fn create_auto_heal_disk_request(set_disk_id: String, priority: Option<HealChann
|
||||
}
|
||||
}
|
||||
|
||||
/// Build one automatic replacement request for the verified targets of a set.
|
||||
pub fn create_auto_replacement_disks_request(
|
||||
fn create_auto_replacement_disk_request(
|
||||
pool_index: usize,
|
||||
set_index: usize,
|
||||
mut replacement_endpoints: Vec<String>,
|
||||
replacement_endpoint: String,
|
||||
priority: Option<HealChannelPriority>,
|
||||
) -> HealChannelRequest {
|
||||
replacement_endpoints.sort_unstable();
|
||||
replacement_endpoints.dedup();
|
||||
let mut request = create_auto_heal_disk_request(format!("pool_{pool_index}_set_{set_index}"), priority);
|
||||
request.heal_endpoints = replacement_endpoints;
|
||||
request.heal_endpoints = vec![replacement_endpoint];
|
||||
request.pool_index = Some(pool_index);
|
||||
request.set_index = Some(set_index);
|
||||
request
|
||||
@@ -714,10 +711,10 @@ pub async fn send_heal_replacement_disk(
|
||||
replacement_endpoint: String,
|
||||
priority: Option<HealChannelPriority>,
|
||||
) -> Result<(), String> {
|
||||
send_heal_request(create_auto_replacement_disks_request(
|
||||
send_heal_request(create_auto_replacement_disk_request(
|
||||
pool_index,
|
||||
set_index,
|
||||
vec![replacement_endpoint],
|
||||
replacement_endpoint,
|
||||
priority,
|
||||
))
|
||||
.await
|
||||
@@ -729,12 +726,8 @@ mod auto_heal_disk_request_tests {
|
||||
|
||||
#[test]
|
||||
fn replacement_disk_request_carries_its_exact_endpoint() {
|
||||
let request = create_auto_replacement_disks_request(
|
||||
2,
|
||||
3,
|
||||
vec!["http://node2:9000/drive3".to_string()],
|
||||
Some(HealChannelPriority::Normal),
|
||||
);
|
||||
let request =
|
||||
create_auto_replacement_disk_request(2, 3, "http://node2:9000/drive3".to_string(), Some(HealChannelPriority::Normal));
|
||||
|
||||
assert_eq!(request.disk.as_deref(), Some("pool_2_set_3"));
|
||||
assert_eq!(request.heal_endpoints, ["http://node2:9000/drive3"]);
|
||||
@@ -743,23 +736,6 @@ mod auto_heal_disk_request_tests {
|
||||
assert_eq!(request.source, HealRequestSource::AutoHeal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replacement_disk_request_keeps_every_distinct_target() {
|
||||
let request = create_auto_replacement_disks_request(
|
||||
2,
|
||||
3,
|
||||
vec!["drive-b".to_string(), "drive-a".to_string(), "drive-b".to_string()],
|
||||
Some(HealChannelPriority::Normal),
|
||||
);
|
||||
|
||||
assert_eq!(request.disk.as_deref(), Some("pool_2_set_3"));
|
||||
assert_eq!(request.heal_endpoints, ["drive-a", "drive-b"]);
|
||||
assert_eq!(request.pool_index, Some(2));
|
||||
assert_eq!(request.set_index, Some(3));
|
||||
assert_eq!(request.priority, HealChannelPriority::Normal);
|
||||
assert_eq!(request.source, HealRequestSource::AutoHeal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_auto_heal_disk_request_has_no_replacement_endpoint() {
|
||||
let request = create_auto_heal_disk_request("pool_2_set_3".to_string(), None);
|
||||
|
||||
@@ -196,6 +196,20 @@ impl HealChannelProcessor {
|
||||
.map_err(Error::other)
|
||||
}
|
||||
|
||||
/// Query a local automatic task; other tokens require coordinator routing.
|
||||
pub async fn execute_local_automatic_query(
|
||||
&self,
|
||||
heal_path: &str,
|
||||
client_token: &str,
|
||||
since_seq: Option<u64>,
|
||||
) -> Result<Option<HealChannelResponse>> {
|
||||
self.heal_manager
|
||||
.get_local_automatic_task_report(heal_path, client_token, since_seq)
|
||||
.await?
|
||||
.map(|report| Self::query_report_response(client_token.to_owned(), Ok(report)))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// Execute cancellation directly against the manager.
|
||||
pub async fn execute_cancel_request(&self, heal_path: String, client_token: String) -> Result<HealChannelResponse> {
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
@@ -463,6 +477,13 @@ impl HealChannelProcessor {
|
||||
.await
|
||||
};
|
||||
|
||||
let response = Self::query_report_response(client_token, report)?;
|
||||
let _ = response_tx.send(Ok(response.clone()));
|
||||
self.publish_response(response);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn query_report_response(client_token: String, report: Result<HealTaskReport>) -> Result<HealChannelResponse> {
|
||||
let outcome = report.as_ref().ok().and_then(|report| report.outcome.clone());
|
||||
let settings = report
|
||||
.as_ref()
|
||||
@@ -581,27 +602,21 @@ impl HealChannelProcessor {
|
||||
0,
|
||||
),
|
||||
Err(crate::Error::InvalidClientToken) => {
|
||||
let response = HealChannelResponse {
|
||||
return Ok(HealChannelResponse {
|
||||
request_id: client_token,
|
||||
success: false,
|
||||
data: None,
|
||||
error: Some("invalid heal client token".to_string()),
|
||||
};
|
||||
let _ = response_tx.send(Ok(response.clone()));
|
||||
self.publish_response(response);
|
||||
return Ok(());
|
||||
});
|
||||
}
|
||||
Err(err) => {
|
||||
let error_text = err.to_string();
|
||||
let response = HealChannelResponse {
|
||||
return Ok(HealChannelResponse {
|
||||
request_id: client_token,
|
||||
success: false,
|
||||
data: None,
|
||||
error: Some(error_text),
|
||||
};
|
||||
let _ = response_tx.send(Ok(response.clone()));
|
||||
self.publish_response(response);
|
||||
return Ok(());
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
@@ -618,17 +633,12 @@ impl HealChannelProcessor {
|
||||
},
|
||||
)?;
|
||||
|
||||
let response = HealChannelResponse {
|
||||
Ok(HealChannelResponse {
|
||||
request_id: client_token,
|
||||
success: true,
|
||||
data: Some(data),
|
||||
error: detail,
|
||||
};
|
||||
|
||||
let _ = response_tx.send(Ok(response.clone()));
|
||||
self.publish_response(response);
|
||||
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
/// Process cancel request
|
||||
|
||||
+80
-113
@@ -234,6 +234,8 @@ fn record_displaced_terminal(
|
||||
request: &HealRequest,
|
||||
) -> Arc<CompletedHealStatus> {
|
||||
let terminal = Arc::new(CompletedHealStatus {
|
||||
task_id: request.id.clone(),
|
||||
source: request.source,
|
||||
outcome: None,
|
||||
progress: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
@@ -669,7 +671,10 @@ fn retry_budget_for_result(task: &HealTask, result: &Result<()>, retryable_batch
|
||||
if task.retry_attempts >= MAX_RECOVERABLE_HEAL_RETRIES {
|
||||
return None;
|
||||
}
|
||||
if task.has_batch_failure() && !retryable_batch_failure {
|
||||
// Admin traversals already exhaust their per-object retries. Restarting
|
||||
// the whole traversal under the same token discards those receipts and
|
||||
// reports previously repaired objects as unchanged on the next scan.
|
||||
if task.has_batch_failure() && (task.source == HealRequestSource::Admin || !retryable_batch_failure) {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -825,21 +830,6 @@ fn admin_heal_options_compatible(left: &HealOptions, right: &HealOptions) -> boo
|
||||
&& left.no_lock == right.no_lock
|
||||
}
|
||||
|
||||
fn replacement_request_compatible(
|
||||
request: &HealRequest,
|
||||
owner_type: &HealType,
|
||||
owner_options: &HealOptions,
|
||||
owner_source: HealRequestSource,
|
||||
) -> bool {
|
||||
request.source == HealRequestSource::AutoHeal
|
||||
&& owner_source == HealRequestSource::AutoHeal
|
||||
&& matches!(request.heal_type, HealType::ErasureSet { .. })
|
||||
&& request.heal_type == *owner_type
|
||||
&& request.options.pool_index == owner_options.pool_index
|
||||
&& request.options.set_index == owner_options.set_index
|
||||
&& admin_heal_options_compatible(&request.options, owner_options)
|
||||
}
|
||||
|
||||
fn admin_overlap_rejection(
|
||||
request: &HealRequest,
|
||||
owner_type: &HealType,
|
||||
@@ -1071,9 +1061,9 @@ pub struct HealManager {
|
||||
/// cascade without re-locking.
|
||||
enum TaskStateLookup {
|
||||
Active(Arc<HealTask>),
|
||||
Retrying(HealTaskStatus, HealOptions),
|
||||
Retrying(HealTaskStatus, HealOptions, HealRequestSource),
|
||||
Completed(Arc<CompletedHealStatus>),
|
||||
Queued(HealOptions),
|
||||
Queued(HealOptions, HealRequestSource),
|
||||
/// A durable root heal record that cannot be admitted or quarantined;
|
||||
/// it answers status queries with the reason until an operator repairs it.
|
||||
Unrunnable(String),
|
||||
@@ -1291,9 +1281,7 @@ impl HealManager {
|
||||
request: HealRequest,
|
||||
config: &HealConfig,
|
||||
context: &'static str,
|
||||
replacement_targets_frozen: bool,
|
||||
) -> HealAdmissionDecision {
|
||||
let replacement_targets_frozen = replacement_targets_frozen || request.retry_attempts != 0;
|
||||
let queue_len = queue.len();
|
||||
publish_heal_queue_length(queue);
|
||||
let queue_capacity = config.queue_size;
|
||||
@@ -1301,25 +1289,6 @@ impl HealManager {
|
||||
let best_effort_reserve = best_effort_queue_reserve(queue_capacity);
|
||||
let best_effort_source_reserves = best_effort_source_reserves(queue_capacity);
|
||||
|
||||
// Scanner and retry admission bypass submit_heal_request's duplicate
|
||||
// checks. Keep their target ownership checks ahead of queue capacity.
|
||||
if !request.force_start
|
||||
&& request.source == HealRequestSource::AutoHeal
|
||||
&& !request.heal_endpoints.is_empty()
|
||||
&& let Some(owner_id) = queue
|
||||
.queued_request_id_for_dedup_key(&PriorityHealQueue::make_dedup_key(&request))
|
||||
.map(ToOwned::to_owned)
|
||||
{
|
||||
let mut admission = Self::duplicate_admission_for_request(&request, config);
|
||||
if admission == HealAdmissionResult::Merged
|
||||
&& !queue.merge_replacement_targets(&owner_id, &request, true, replacement_targets_frozen)
|
||||
{
|
||||
admission = HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning);
|
||||
}
|
||||
Self::record_admission_metric(request.source, admission, context);
|
||||
return HealAdmissionDecision::new(admission);
|
||||
}
|
||||
|
||||
if queue_len >= queue_capacity && !request.force_start {
|
||||
if Self::can_displace_queued_work(&request)
|
||||
&& queue.can_displace_lower_priority_where(request.priority, Self::queued_request_can_be_displaced)
|
||||
@@ -1479,7 +1448,7 @@ impl HealManager {
|
||||
let priority = request.priority;
|
||||
let source = request.source;
|
||||
|
||||
match queue.push_with_replacement_scope(request, replacement_targets_frozen) {
|
||||
match queue.push(request) {
|
||||
QueuePushOutcome::Accepted => {
|
||||
publish_heal_queue_length(queue);
|
||||
Self::record_admission_metric(source, HealAdmissionResult::Accepted, context);
|
||||
@@ -2296,7 +2265,7 @@ impl HealManager {
|
||||
// HS-06: under the minio_error overlap policy an exact duplicate
|
||||
// admin start reports the typed AlreadyRunning rejection instead
|
||||
// of the silent merge (MinIO's ErrHealAlreadyRunning).
|
||||
let mut admission = if (request.source == HealRequestSource::Admin
|
||||
let admission = if (request.source == HealRequestSource::Admin
|
||||
&& config.overlap_policy == HealOverlapPolicy::MinioError)
|
||||
|| (duplicate_state != "queued" && mrf_notice_target.as_ref().is_some_and(|target| target.kind.is_durable()))
|
||||
{
|
||||
@@ -2306,44 +2275,6 @@ impl HealManager {
|
||||
} else {
|
||||
Self::duplicate_admission_for_request(&request, &config)
|
||||
};
|
||||
if matches!(admission, HealAdmissionResult::Merged)
|
||||
&& request.source == HealRequestSource::AutoHeal
|
||||
&& !request.heal_endpoints.is_empty()
|
||||
{
|
||||
let (owner_recovered, incoming_recovered) = {
|
||||
// Match queue -> recovery-anchor ordering in the scanner.
|
||||
let anchors = self
|
||||
.replacement_recovery_anchors
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
(anchors.contains_key(&merged_task_id), anchors.contains_key(&request.id))
|
||||
};
|
||||
let incoming_frozen = incoming_recovered || request.retry_attempts != 0;
|
||||
let covered = if incoming_frozen && request.id != merged_task_id {
|
||||
false
|
||||
} else if duplicate_state == "queued" {
|
||||
queue.merge_replacement_targets(&merged_task_id, &request, !owner_recovered, incoming_frozen)
|
||||
} else {
|
||||
let owner = active_heals
|
||||
.get(&merged_task_id)
|
||||
.map(|task| (&task.heal_type, &task.options, task.source, &task.heal_endpoints))
|
||||
.or_else(|| {
|
||||
retrying_heals.get(&merged_task_id).map(|retry| {
|
||||
let owner = &retry.request;
|
||||
(&owner.heal_type, &owner.options, owner.source, &owner.heal_endpoints)
|
||||
})
|
||||
});
|
||||
owner.is_some_and(|(heal_type, options, source, endpoints)| {
|
||||
replacement_request_compatible(&request, heal_type, options, source)
|
||||
&& request.heal_endpoints.iter().all(|endpoint| endpoints.contains(endpoint))
|
||||
})
|
||||
};
|
||||
if !covered {
|
||||
// A frozen generation cannot certify newly arriving disks.
|
||||
// Leave them discoverable by the scanner for the next pass.
|
||||
admission = HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning);
|
||||
}
|
||||
}
|
||||
if matches!(admission, HealAdmissionResult::Merged)
|
||||
&& let Some(target) = mrf_notice_target
|
||||
{
|
||||
@@ -2500,14 +2431,7 @@ impl HealManager {
|
||||
let request_id = request.id.clone();
|
||||
let request_heal_type = request.heal_type.clone();
|
||||
let request_source = request.source;
|
||||
let recovered_replacement = request.source == HealRequestSource::AutoHeal
|
||||
&& !request.heal_endpoints.is_empty()
|
||||
&& self
|
||||
.replacement_recovery_anchors
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.contains_key(&request.id);
|
||||
let admission_decision = Self::admit_request_to_queue(&mut queue, request, &config, "submit", recovered_replacement);
|
||||
let admission_decision = Self::admit_request_to_queue(&mut queue, request, &config, "submit");
|
||||
let admission = admission_decision.result;
|
||||
if durable_handoff_required && !admission.is_admitted() {
|
||||
self.root_recovery
|
||||
@@ -2586,6 +2510,23 @@ impl HealManager {
|
||||
/// completed entry. `heal_path` additionally constrains the map matches
|
||||
/// the way the `*_for_path` variants always have.
|
||||
async fn lookup_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> Result<TaskStateLookup> {
|
||||
let state = self.lookup_memory_task_state(canonical_task_id, heal_path).await;
|
||||
if !matches!(state, TaskStateLookup::NotFound) {
|
||||
return Ok(state);
|
||||
}
|
||||
if let Some(completed) = self.root_recovery.completed(canonical_task_id).await?
|
||||
&& heal_path.is_none_or(|path| heal_type_matches_path(&completed.heal_type, path))
|
||||
{
|
||||
return Ok(TaskStateLookup::Completed(Arc::new(completed)));
|
||||
}
|
||||
// A durable record that cannot run still answers with its reason.
|
||||
if let Some(reason) = self.root_recovery.record_issue(canonical_task_id).await {
|
||||
return Ok(TaskStateLookup::Unrunnable(reason));
|
||||
}
|
||||
Ok(TaskStateLookup::NotFound)
|
||||
}
|
||||
|
||||
async fn lookup_memory_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> TaskStateLookup {
|
||||
let matches_path = |heal_type: &HealType| heal_path.is_none_or(|path| heal_type_matches_path(heal_type, path));
|
||||
|
||||
{
|
||||
@@ -2594,7 +2535,7 @@ impl HealManager {
|
||||
.get(canonical_task_id)
|
||||
.filter(|task| matches_path(&task.heal_type))
|
||||
{
|
||||
return Ok(TaskStateLookup::Active(Arc::clone(task)));
|
||||
return TaskStateLookup::Active(Arc::clone(task));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2604,7 +2545,7 @@ impl HealManager {
|
||||
.get(canonical_task_id)
|
||||
.filter(|retrying| matches_path(&retrying.request.heal_type))
|
||||
{
|
||||
return Ok(TaskStateLookup::Retrying(retrying.status(), retrying.request.options.clone()));
|
||||
return TaskStateLookup::Retrying(retrying.status(), retrying.request.options.clone(), retrying.request.source);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2617,7 +2558,7 @@ impl HealManager {
|
||||
prune_completed_heal_statuses(&mut completed_heals);
|
||||
if let Some(completed) = completed_heals.get(canonical_task_id).filter(|c| matches_path(&c.heal_type)) {
|
||||
if completed_status_is_retrying(&completed.status) {
|
||||
return Ok(TaskStateLookup::Completed(Arc::clone(completed)));
|
||||
return TaskStateLookup::Completed(Arc::clone(completed));
|
||||
}
|
||||
terminal_completed = Some(Arc::clone(completed));
|
||||
}
|
||||
@@ -2626,7 +2567,7 @@ impl HealManager {
|
||||
{
|
||||
let queue = self.heal_queue.lock().await;
|
||||
if let Some(request) = queue.request_matching_id_and_path(canonical_task_id, heal_path) {
|
||||
return Ok(TaskStateLookup::Queued(request.options.clone()));
|
||||
return TaskStateLookup::Queued(request.options.clone(), request.source);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2639,22 +2580,10 @@ impl HealManager {
|
||||
.cloned();
|
||||
}
|
||||
|
||||
if terminal_completed.is_none()
|
||||
&& let Some(completed) = self.root_recovery.completed(canonical_task_id).await?
|
||||
&& matches_path(&completed.heal_type)
|
||||
{
|
||||
terminal_completed = Some(Arc::new(completed));
|
||||
}
|
||||
|
||||
if let Some(completed) = terminal_completed {
|
||||
return Ok(TaskStateLookup::Completed(completed));
|
||||
return TaskStateLookup::Completed(completed);
|
||||
}
|
||||
// A durable root record that can neither run nor be quarantined still
|
||||
// answers status queries with its reason.
|
||||
if let Some(reason) = self.root_recovery.record_issue(canonical_task_id).await {
|
||||
return Ok(TaskStateLookup::Unrunnable(reason));
|
||||
}
|
||||
Ok(TaskStateLookup::NotFound)
|
||||
TaskStateLookup::NotFound
|
||||
}
|
||||
|
||||
async fn publish_admin_terminal(
|
||||
@@ -2678,6 +2607,8 @@ impl HealManager {
|
||||
) -> Result<bool> {
|
||||
let previous = self.completed_heals.lock().await.get(task_id).cloned();
|
||||
let mut completed = CompletedHealStatus {
|
||||
task_id: task_id.to_owned(),
|
||||
source,
|
||||
outcome: None,
|
||||
progress: None,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
@@ -2713,9 +2644,9 @@ impl HealManager {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
match self.lookup_task_state(&canonical_task_id, None).await? {
|
||||
TaskStateLookup::Active(task) => Ok(task.get_status().await),
|
||||
TaskStateLookup::Retrying(status, _) => Ok(status),
|
||||
TaskStateLookup::Retrying(status, _, _) => Ok(status),
|
||||
TaskStateLookup::Completed(completed) => Ok(completed.status.clone()),
|
||||
TaskStateLookup::Queued(_) => Ok(HealTaskStatus::Pending),
|
||||
TaskStateLookup::Queued(_, _) => Ok(HealTaskStatus::Pending),
|
||||
TaskStateLookup::Unrunnable(error) => Ok(HealTaskStatus::Failed { error }),
|
||||
TaskStateLookup::NotFound => Err(Error::TaskNotFound {
|
||||
task_id: task_id.to_string(),
|
||||
@@ -2734,9 +2665,9 @@ impl HealManager {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
match self.lookup_task_state(&canonical_task_id, None).await? {
|
||||
TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await),
|
||||
TaskStateLookup::Retrying(status, options) => Ok(empty_task_report_with_options(status, options)),
|
||||
TaskStateLookup::Retrying(status, options, _) => Ok(empty_task_report_with_options(status, options)),
|
||||
TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)),
|
||||
TaskStateLookup::Queued(options) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
|
||||
TaskStateLookup::Queued(options, _) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
|
||||
TaskStateLookup::Unrunnable(error) => {
|
||||
Ok(empty_task_report_with_options(HealTaskStatus::Failed { error }, HealOptions::default()))
|
||||
}
|
||||
@@ -2750,6 +2681,42 @@ impl HealManager {
|
||||
self.get_task_report_for_path_since(heal_path, task_id, None).await
|
||||
}
|
||||
|
||||
/// Resolve a node-owned automatic task without granting local control over
|
||||
/// Admin tasks. The source and report come from the same state snapshot.
|
||||
pub async fn get_local_automatic_task_report(
|
||||
&self,
|
||||
heal_path: &str,
|
||||
task_id: &str,
|
||||
since: Option<u64>,
|
||||
) -> Result<Option<HealTaskReport>> {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
if canonical_task_id != task_id {
|
||||
return Ok(None);
|
||||
}
|
||||
let state = self.lookup_memory_task_state(&canonical_task_id, Some(heal_path)).await;
|
||||
let source = match &state {
|
||||
TaskStateLookup::Active(task) => task.source,
|
||||
TaskStateLookup::Retrying(_, _, source) | TaskStateLookup::Queued(_, source) => *source,
|
||||
TaskStateLookup::Completed(completed) if completed.task_id == task_id => completed.source,
|
||||
TaskStateLookup::Completed(_) => return Ok(None),
|
||||
TaskStateLookup::Unrunnable(_) | TaskStateLookup::NotFound => return Ok(None),
|
||||
};
|
||||
if !matches!(
|
||||
source,
|
||||
HealRequestSource::Scanner | HealRequestSource::Mrf | HealRequestSource::AutoHeal | HealRequestSource::ReadRepair
|
||||
) {
|
||||
return Ok(None);
|
||||
}
|
||||
let report = match state {
|
||||
TaskStateLookup::Active(task) => active_task_report(&task, since).await,
|
||||
TaskStateLookup::Retrying(status, options, _) => empty_task_report_with_options(status, options),
|
||||
TaskStateLookup::Completed(completed) => completed_task_report(&completed, since),
|
||||
TaskStateLookup::Queued(options, _) => empty_task_report_with_options(HealTaskStatus::Pending, options),
|
||||
TaskStateLookup::Unrunnable(_) | TaskStateLookup::NotFound => return Ok(None),
|
||||
};
|
||||
Ok(Some(report))
|
||||
}
|
||||
|
||||
/// Incremental variant of [`Self::get_task_report_for_path`] (HS-06).
|
||||
pub async fn get_task_report_for_path_since(
|
||||
&self,
|
||||
@@ -2760,9 +2727,9 @@ impl HealManager {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? {
|
||||
TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await),
|
||||
TaskStateLookup::Retrying(status, options) => Ok(empty_task_report_with_options(status, options)),
|
||||
TaskStateLookup::Retrying(status, options, _) => Ok(empty_task_report_with_options(status, options)),
|
||||
TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)),
|
||||
TaskStateLookup::Queued(options) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
|
||||
TaskStateLookup::Queued(options, _) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
|
||||
TaskStateLookup::Unrunnable(error) => {
|
||||
Ok(empty_task_report_with_options(HealTaskStatus::Failed { error }, HealOptions::default()))
|
||||
}
|
||||
@@ -2786,9 +2753,9 @@ impl HealManager {
|
||||
let canonical_task_id = self.canonical_task_id(task_id).await;
|
||||
match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? {
|
||||
TaskStateLookup::Active(task) => Ok(task.get_status().await),
|
||||
TaskStateLookup::Retrying(status, _) => Ok(status),
|
||||
TaskStateLookup::Retrying(status, _, _) => Ok(status),
|
||||
TaskStateLookup::Completed(completed) => Ok(completed.status.clone()),
|
||||
TaskStateLookup::Queued(_) => Ok(HealTaskStatus::Pending),
|
||||
TaskStateLookup::Queued(_, _) => Ok(HealTaskStatus::Pending),
|
||||
TaskStateLookup::Unrunnable(error) => Ok(HealTaskStatus::Failed { error }),
|
||||
TaskStateLookup::NotFound => {
|
||||
if self.path_has_task(heal_path).await {
|
||||
|
||||
@@ -682,7 +682,6 @@ impl HealManager {
|
||||
HealPriority::Low,
|
||||
);
|
||||
let recovery_anchor = recovery.as_ref().map(|(_, _, _, anchor)| anchor.clone());
|
||||
let recovered_replacement = recovery.is_some();
|
||||
if let Some((task_id, _, _, _)) = recovery {
|
||||
req.id = task_id;
|
||||
}
|
||||
@@ -692,8 +691,7 @@ impl HealManager {
|
||||
let endpoint_count = req.heal_endpoints.len();
|
||||
let config = config.read().await;
|
||||
let mut queue = heal_queue.lock().await;
|
||||
let admission_decision =
|
||||
Self::admit_request_to_queue(&mut queue, req, &config, "auto_scan", recovered_replacement);
|
||||
let admission_decision = Self::admit_request_to_queue(&mut queue, req, &config, "auto_scan");
|
||||
let admission = admission_decision.result;
|
||||
let should_notify =
|
||||
matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable;
|
||||
|
||||
@@ -70,8 +70,6 @@ pub(super) struct PriorityQueueItem {
|
||||
pub(super) sequence: u64,
|
||||
pub(super) dedup_key: String,
|
||||
pub(super) request: HealRequest,
|
||||
/// Execution or durable recovery fixed these targets; grace waits can have zero retries.
|
||||
pub(super) replacement_targets_frozen: bool,
|
||||
}
|
||||
|
||||
impl Eq for PriorityQueueItem {}
|
||||
@@ -109,7 +107,9 @@ pub(super) enum QueuePushOutcome {
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct CompletedHealStatus {
|
||||
pub(super) task_id: String,
|
||||
pub(super) heal_type: HealType,
|
||||
pub(super) source: HealRequestSource,
|
||||
/// Options used to execute the task, retained for token-scoped status.
|
||||
pub(super) options: HealOptions,
|
||||
pub(super) status: HealTaskStatus,
|
||||
@@ -137,6 +137,7 @@ impl CompletedHealStatus {
|
||||
fn measure_retained_bytes(&self) -> usize {
|
||||
let mut bytes = size_of::<Self>();
|
||||
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
|
||||
add(self.task_id.capacity());
|
||||
add(self.outcome.as_ref().map_or(0, |outcome| outcome.retained_bytes()));
|
||||
match &self.heal_type {
|
||||
HealType::Cluster => {}
|
||||
@@ -258,7 +259,9 @@ impl CompletedHealStatus {
|
||||
outcome.finish(Some(crate::heal::outcome::HealAbortReason::Cancelled));
|
||||
}
|
||||
let mut snapshot = Self {
|
||||
task_id: task.id.clone(),
|
||||
heal_type: task.heal_type.clone(),
|
||||
source: task.source,
|
||||
options: task.options.clone(),
|
||||
status,
|
||||
progress: Some(task.get_progress().await),
|
||||
@@ -318,10 +321,6 @@ impl PriorityHealQueue {
|
||||
}
|
||||
|
||||
pub(super) fn push(&mut self, request: HealRequest) -> QueuePushOutcome {
|
||||
self.push_with_replacement_scope(request, false)
|
||||
}
|
||||
|
||||
pub(super) fn push_with_replacement_scope(&mut self, request: HealRequest, frozen: bool) -> QueuePushOutcome {
|
||||
let key = Self::make_dedup_key(&request);
|
||||
|
||||
// Check for duplicates unless the caller explicitly forces admission.
|
||||
@@ -346,7 +345,6 @@ impl PriorityHealQueue {
|
||||
sequence: self.sequence,
|
||||
dedup_key: key,
|
||||
request,
|
||||
replacement_targets_frozen: frozen,
|
||||
});
|
||||
if is_best_effort_source(source) {
|
||||
self.best_effort_queued += 1;
|
||||
@@ -687,51 +685,6 @@ impl PriorityHealQueue {
|
||||
self.heap.iter().map(|item| &item.request)
|
||||
}
|
||||
|
||||
pub(super) fn merge_replacement_targets(
|
||||
&mut self,
|
||||
owner_id: &str,
|
||||
request: &HealRequest,
|
||||
allow_expand: bool,
|
||||
incoming_frozen: bool,
|
||||
) -> bool {
|
||||
let Some(item) = self.heap.iter().find(|item| item.request.id == owner_id) else {
|
||||
return false;
|
||||
};
|
||||
let owner = &item.request;
|
||||
let same_id = request.id == owner.id;
|
||||
let eligible = replacement_request_compatible(request, &owner.heal_type, &owner.options, owner.source)
|
||||
&& (!incoming_frozen || same_id);
|
||||
let covered = eligible
|
||||
&& request
|
||||
.heal_endpoints
|
||||
.iter()
|
||||
.all(|endpoint| owner.heal_endpoints.contains(endpoint));
|
||||
let mark_frozen = incoming_frozen && same_id && !item.replacement_targets_frozen;
|
||||
let can_expand = eligible
|
||||
&& allow_expand
|
||||
&& !item.replacement_targets_frozen
|
||||
&& !incoming_frozen
|
||||
&& owner.retry_attempts == 0
|
||||
&& request.retry_attempts == 0
|
||||
&& !owner.heal_endpoints.is_empty();
|
||||
if !mark_frozen && (covered || !can_expand) {
|
||||
return covered;
|
||||
}
|
||||
|
||||
// Neither targets nor their frozen flag affect heap ordering.
|
||||
let mut items = std::mem::take(&mut self.heap).into_vec();
|
||||
if let Some(item) = items.iter_mut().find(|item| item.request.id == owner_id) {
|
||||
item.replacement_targets_frozen |= mark_frozen;
|
||||
if !covered && can_expand {
|
||||
item.request.heal_endpoints.extend(request.heal_endpoints.iter().cloned());
|
||||
item.request.heal_endpoints.sort_unstable();
|
||||
item.request.heal_endpoints.dedup();
|
||||
}
|
||||
}
|
||||
self.heap = BinaryHeap::from(items);
|
||||
covered || can_expand
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn contains_request_id(&self, request_id: &str) -> bool {
|
||||
self.heap.iter().any(|item| item.request.id == request_id)
|
||||
|
||||
@@ -460,6 +460,8 @@ impl RootHealTerminal {
|
||||
|
||||
fn into_completed(self) -> CompletedHealStatus {
|
||||
CompletedHealStatus {
|
||||
task_id: self.task_id,
|
||||
source: HealRequestSource::Admin,
|
||||
outcome: None,
|
||||
progress: self.progress,
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
|
||||
@@ -582,7 +582,7 @@ impl HealManager {
|
||||
return;
|
||||
}
|
||||
let admission_decision =
|
||||
Self::admit_request_to_queue(&mut queue, retry_request.clone(), &retry_config, "retry", true);
|
||||
Self::admit_request_to_queue(&mut queue, retry_request.clone(), &retry_config, "retry");
|
||||
let admission = admission_decision.result;
|
||||
let should_notify = matches!(admission, HealAdmissionResult::Accepted)
|
||||
&& retry_config.event_driven_scheduler_enable;
|
||||
|
||||
@@ -118,6 +118,8 @@ struct MockStorage;
|
||||
|
||||
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
|
||||
CompletedHealStatus {
|
||||
task_id: "fixture-token".to_string(),
|
||||
source: HealRequestSource::Internal,
|
||||
outcome: None,
|
||||
heal_type: HealType::Cluster,
|
||||
options: HealOptions::default(),
|
||||
@@ -2505,301 +2507,6 @@ fn test_heal_request_and_task_metric_labels_match() {
|
||||
assert_eq!(task.metric_set_label(), "pool_0_set_1");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_replacement_queued_targets_merge_without_changing_fifo_or_token() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let owner = automatic_replacement_request(&["replacement-a"]);
|
||||
let owner_id = owner.id.clone();
|
||||
let created_at = owner.created_at;
|
||||
let first = manager
|
||||
.submit_heal_request_with_receipt(owner)
|
||||
.await
|
||||
.expect("queue first target");
|
||||
assert_eq!(first.result, HealAdmissionResult::Accepted);
|
||||
|
||||
let other = HealRequest::object("bucket".to_string(), "unrelated".to_string(), None);
|
||||
let other_id = other.id.clone();
|
||||
manager.submit_heal_request(other).await.expect("queue unrelated work");
|
||||
let merged = manager
|
||||
.submit_heal_request_with_receipt(automatic_replacement_request(&["replacement-b", "replacement-a"]))
|
||||
.await
|
||||
.expect("merge all fresh targets");
|
||||
assert_eq!(merged.result, HealAdmissionResult::Merged);
|
||||
assert_eq!(merged.task_id, owner_id);
|
||||
|
||||
let mut queue = manager.heal_queue.lock().await;
|
||||
assert_eq!(queue.len(), 2);
|
||||
let owner = queue.pop_next().expect("original task remains first");
|
||||
assert_eq!(owner.id, owner_id);
|
||||
assert_eq!(owner.created_at, created_at);
|
||||
assert_eq!(owner.heal_endpoints, ["replacement-a", "replacement-b"]);
|
||||
assert_eq!(queue.pop_next().expect("unrelated task remains queued").id, other_id);
|
||||
}
|
||||
|
||||
fn automatic_replacement_request(targets: &[&str]) -> HealRequest {
|
||||
let mut request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: Vec::new(),
|
||||
set_disk_id: "pool_0_set_1".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
pool_index: Some(0),
|
||||
set_index: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
request.source = HealRequestSource::AutoHeal;
|
||||
request.heal_endpoints = targets.iter().map(|target| (*target).to_string()).collect();
|
||||
request
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_replacement_frozen_targets_reject_late_disks_but_cover_duplicates() {
|
||||
for state in ["active", "retrying", "grace", "recovered", "recovery_anchor"] {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None);
|
||||
let mut owner = automatic_replacement_request(&["replacement-a"]);
|
||||
let owner_id = owner.id.clone();
|
||||
if state == "grace" {
|
||||
let now = SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("test clock")
|
||||
.as_secs();
|
||||
let task = HealTask::from_request(owner, storage.clone());
|
||||
owner = retry_request_for_result_with_budget(
|
||||
&task,
|
||||
&Err(Error::DanglingDeleteDeferred {
|
||||
retry_not_before: now + 60,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("grace wait keeps the generation replayable")
|
||||
.0;
|
||||
assert_eq!(owner.retry_attempts, 0, "grace must not consume a failure retry");
|
||||
}
|
||||
match state {
|
||||
"active" => {
|
||||
manager
|
||||
.active_heals
|
||||
.lock()
|
||||
.await
|
||||
.insert(owner_id.clone(), Arc::new(HealTask::from_request(owner, storage.clone())));
|
||||
}
|
||||
"retrying" => {
|
||||
manager.retrying_heals.lock().await.insert(
|
||||
owner_id.clone(),
|
||||
RetryingHeal {
|
||||
request: owner,
|
||||
error: "target unavailable".to_string(),
|
||||
cancel_token: CancellationToken::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
"grace" | "recovered" => {
|
||||
let config = manager.config.read().await.clone();
|
||||
let mut queue = manager.heal_queue.lock().await;
|
||||
let context = if state == "grace" { "retry" } else { "auto_scan" };
|
||||
assert_eq!(
|
||||
HealManager::admit_request_to_queue(&mut queue, owner, &config, context, true).result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert!(
|
||||
queue
|
||||
.heap
|
||||
.iter()
|
||||
.any(|item| item.request.id == owner_id && item.replacement_targets_frozen),
|
||||
"zero retries cannot mean the scope is fresh"
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
manager.submit_heal_request(owner).await.expect("queue frozen generation");
|
||||
if state == "recovery_anchor" {
|
||||
manager
|
||||
.replacement_recovery_anchors
|
||||
.lock()
|
||||
.expect("recovery anchors")
|
||||
.insert(owner_id.clone(), "survivor".to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let late = manager
|
||||
.submit_heal_request(automatic_replacement_request(&["replacement-b"]))
|
||||
.await
|
||||
.expect("classify late target");
|
||||
assert_eq!(late, HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning), "{state}");
|
||||
let covered = manager
|
||||
.submit_heal_request_with_receipt(automatic_replacement_request(&["replacement-a"]))
|
||||
.await
|
||||
.expect("covered target remains owned");
|
||||
assert_eq!(covered.result, HealAdmissionResult::Merged, "{state}");
|
||||
assert_eq!(covered.task_id, owner_id, "{state}");
|
||||
let targets = match state {
|
||||
"active" => manager.active_heals.lock().await[&owner_id].heal_endpoints.clone(),
|
||||
"retrying" => manager.retrying_heals.lock().await[&owner_id].request.heal_endpoints.clone(),
|
||||
_ => {
|
||||
manager
|
||||
.heal_queue
|
||||
.lock()
|
||||
.await
|
||||
.pop_next()
|
||||
.expect("retained generation")
|
||||
.heal_endpoints
|
||||
}
|
||||
};
|
||||
assert_eq!(targets, ["replacement-a"], "{state}: frozen scope must not grow");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_replacement_merge_preserves_options_policy_and_recovery_identity() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
manager
|
||||
.submit_heal_request(automatic_replacement_request(&["replacement-a"]))
|
||||
.await
|
||||
.expect("queue fresh generation");
|
||||
for variant in [
|
||||
"dry_run",
|
||||
"bucket_scope",
|
||||
"pool_scope",
|
||||
"retried_incoming",
|
||||
"recovery_anchor",
|
||||
"merge_disabled",
|
||||
] {
|
||||
let mut incoming = automatic_replacement_request(&["replacement-b"]);
|
||||
match variant {
|
||||
"dry_run" => incoming.options.dry_run = true,
|
||||
"bucket_scope" => {
|
||||
if let HealType::ErasureSet { buckets, .. } = &mut incoming.heal_type {
|
||||
buckets.push("other-bucket".to_string());
|
||||
}
|
||||
}
|
||||
"pool_scope" => incoming.options.pool_index = Some(2),
|
||||
"retried_incoming" => {
|
||||
incoming.heal_endpoints = vec!["replacement-a".to_string()];
|
||||
incoming.retry_attempts = 1;
|
||||
}
|
||||
"recovery_anchor" => {
|
||||
incoming.heal_endpoints = vec!["replacement-a".to_string()];
|
||||
manager
|
||||
.replacement_recovery_anchors
|
||||
.lock()
|
||||
.expect("recovery anchors")
|
||||
.insert(incoming.id.clone(), "survivor".to_string());
|
||||
}
|
||||
"merge_disabled" => {
|
||||
manager.config.write().await.low_priority_merge_enable = false;
|
||||
incoming.priority = HealPriority::Low;
|
||||
}
|
||||
_ => unreachable!("test variant"),
|
||||
}
|
||||
let result = manager
|
||||
.submit_heal_request(incoming)
|
||||
.await
|
||||
.expect("classify incompatible generation");
|
||||
let reason = if variant == "merge_disabled" {
|
||||
HealAdmissionDropReason::PolicyDropped
|
||||
} else {
|
||||
HealAdmissionDropReason::AlreadyRunning
|
||||
};
|
||||
assert_eq!(result, HealAdmissionResult::Dropped(reason), "{variant}");
|
||||
let queue = manager.heal_queue.lock().await;
|
||||
assert_eq!(queue.len(), 1, "{variant}");
|
||||
assert_eq!(
|
||||
queue.requests().next().expect("original generation").heal_endpoints,
|
||||
["replacement-a"],
|
||||
"{variant}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn automatic_replacement_direct_admission_checks_frozen_targets_before_capacity() {
|
||||
let config = HealConfig {
|
||||
queue_size: 1,
|
||||
..HealConfig::default()
|
||||
};
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
let owner = automatic_replacement_request(&["replacement-a"]);
|
||||
let owner_id = owner.id.clone();
|
||||
assert_eq!(queue.push(owner), QueuePushOutcome::Accepted);
|
||||
let admit = |queue: &mut PriorityHealQueue, request, frozen| {
|
||||
HealManager::admit_request_to_queue(queue, request, &config, "auto_scan", frozen).result
|
||||
};
|
||||
assert_eq!(
|
||||
admit(&mut queue, automatic_replacement_request(&["replacement-b"]), false),
|
||||
HealAdmissionResult::Merged
|
||||
);
|
||||
let mut replay = automatic_replacement_request(&["replacement-a"]);
|
||||
replay.id = owner_id.clone();
|
||||
assert_eq!(admit(&mut queue, replay, true), HealAdmissionResult::Merged);
|
||||
assert!(
|
||||
queue
|
||||
.heap
|
||||
.iter()
|
||||
.any(|item| item.request.id == owner_id && item.replacement_targets_frozen)
|
||||
);
|
||||
assert_eq!(
|
||||
admit(&mut queue, automatic_replacement_request(&["replacement-c"]), false),
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
);
|
||||
assert_eq!(
|
||||
admit(&mut queue, automatic_replacement_request(&["replacement-a"]), false),
|
||||
HealAdmissionResult::Merged
|
||||
);
|
||||
assert_eq!(
|
||||
admit(&mut queue, automatic_replacement_request(&["replacement-a"]), true),
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
);
|
||||
assert_eq!(queue.len(), 1);
|
||||
let owner = queue.pop_next().expect("frozen owner retained");
|
||||
assert_eq!(owner.id, owner_id);
|
||||
assert_eq!(owner.heal_endpoints, ["replacement-a", "replacement-b"]);
|
||||
|
||||
let owner = automatic_replacement_request(&["replacement-a"]);
|
||||
let owner_id = owner.id.clone();
|
||||
assert_eq!(queue.push(owner), QueuePushOutcome::Accepted);
|
||||
let mut conflicting_replay = automatic_replacement_request(&["replacement-b"]);
|
||||
conflicting_replay.id = owner_id.clone();
|
||||
assert_eq!(
|
||||
admit(&mut queue, conflicting_replay, true),
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
);
|
||||
assert!(
|
||||
queue
|
||||
.heap
|
||||
.iter()
|
||||
.any(|item| item.request.id == owner_id && item.replacement_targets_frozen)
|
||||
);
|
||||
assert_eq!(
|
||||
admit(&mut queue, automatic_replacement_request(&["replacement-c"]), false),
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
);
|
||||
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
let owner = automatic_replacement_request(&["replacement-a"]);
|
||||
let owner_id = owner.id.clone();
|
||||
assert_eq!(queue.push(owner), QueuePushOutcome::Accepted);
|
||||
let mut incompatible_replay = automatic_replacement_request(&["replacement-a"]);
|
||||
incompatible_replay.id = owner_id.clone();
|
||||
incompatible_replay.options.dry_run = true;
|
||||
assert_eq!(
|
||||
admit(&mut queue, incompatible_replay, true),
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
);
|
||||
assert!(
|
||||
queue
|
||||
.heap
|
||||
.iter()
|
||||
.any(|item| item.request.id == owner_id && item.replacement_targets_frozen)
|
||||
);
|
||||
assert_eq!(
|
||||
admit(&mut queue, automatic_replacement_request(&["replacement-b"]), false),
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_submit_heal_request_returns_merged_for_duplicate() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
@@ -3071,6 +2778,181 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_automatic_query_preserves_source_path_token_and_state() {
|
||||
use crate::heal::channel::HealChannelProcessor;
|
||||
use crate::heal::outcome::{
|
||||
HealExecutionOutcome, HealObjectDisposition, HealObjectIdentity, HealObjectKind, HealObjectOutcome, HealTaskOutcome,
|
||||
HealTraversalCoverage,
|
||||
};
|
||||
|
||||
for source in [
|
||||
HealRequestSource::Scanner,
|
||||
HealRequestSource::Mrf,
|
||||
HealRequestSource::AutoHeal,
|
||||
HealRequestSource::ReadRepair,
|
||||
HealRequestSource::Admin,
|
||||
HealRequestSource::Internal,
|
||||
] {
|
||||
for state in ["queued", "active", "retrying", "completed", "displaced"] {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(storage.clone(), None));
|
||||
let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
|
||||
request.source = source;
|
||||
let token = request.id.clone();
|
||||
let task = Arc::new(HealTask::from_request(request.clone(), storage));
|
||||
let mut snapshot = CompletedHealStatus::snapshot(&task, HealTaskStatus::Completed).await;
|
||||
assert_eq!(snapshot.source, source, "completion must retain actual task source");
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.execution = HealExecutionOutcome::Completed;
|
||||
outcome.coverage = HealTraversalCoverage::Complete;
|
||||
outcome.record(HealObjectOutcome {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
version_id: None,
|
||||
bucket_incarnation_id: Some(uuid::Uuid::nil()),
|
||||
pool_index: Some(0),
|
||||
set_index: Some(0),
|
||||
},
|
||||
disposition: HealObjectDisposition::VerifiedHealthy,
|
||||
detail: None,
|
||||
});
|
||||
snapshot.outcome = Some(Arc::new(outcome));
|
||||
snapshot.seqed_items = vec![(
|
||||
3,
|
||||
HealResultItem {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
)];
|
||||
snapshot.min_seq = 3;
|
||||
snapshot.next_seq = 4;
|
||||
let expected_status = match state {
|
||||
"queued" => {
|
||||
manager.heal_queue.lock().await.push(request.clone());
|
||||
HealTaskStatus::Pending
|
||||
}
|
||||
"active" => {
|
||||
manager.active_heals.lock().await.insert(token.clone(), task);
|
||||
HealTaskStatus::Pending
|
||||
}
|
||||
"retrying" => {
|
||||
manager.retrying_heals.lock().await.insert(
|
||||
token.clone(),
|
||||
RetryingHeal {
|
||||
request: request.clone(),
|
||||
error: "transient".to_string(),
|
||||
cancel_token: CancellationToken::new(),
|
||||
},
|
||||
);
|
||||
HealTaskStatus::Retrying {
|
||||
error: "transient".to_string(),
|
||||
retry_attempt: request.retry_attempts,
|
||||
}
|
||||
}
|
||||
"completed" => {
|
||||
manager.completed_heals.lock().await.insert(token.clone(), Arc::new(snapshot));
|
||||
HealTaskStatus::Completed
|
||||
}
|
||||
"displaced" => {
|
||||
let terminal = record_displaced_terminal(&manager.displaced_terminals, &request);
|
||||
terminal.status.clone()
|
||||
}
|
||||
_ => unreachable!("fixed test states"),
|
||||
};
|
||||
let automatic = !matches!(source, HealRequestSource::Admin | HealRequestSource::Internal);
|
||||
let report = manager
|
||||
.get_local_automatic_task_report("bucket/object", &token, Some(2))
|
||||
.await
|
||||
.expect("local source lookup");
|
||||
assert_eq!(report.is_some(), automatic, "source={source:?} state={state}");
|
||||
if let Some(report) = report {
|
||||
assert_eq!(report.status, expected_status);
|
||||
if state == "completed" {
|
||||
let outcome = report.outcome.as_ref().expect("retained structured outcome");
|
||||
assert_eq!(outcome.objects.len(), 1);
|
||||
assert_eq!(outcome.objects[0].identity.object, "object");
|
||||
assert_eq!(outcome.objects[0].identity.pool_index, Some(0));
|
||||
assert_eq!(outcome.objects[0].identity.set_index, Some(0));
|
||||
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::VerifiedHealthy);
|
||||
assert_eq!(report.min_seq, 3);
|
||||
assert_eq!(report.next_seq, 4);
|
||||
assert_eq!(report.result_items.len(), 1);
|
||||
assert_eq!(report.result_items[0].object, "object");
|
||||
}
|
||||
let processor = HealChannelProcessor::new(manager.clone());
|
||||
let local = processor
|
||||
.execute_local_automatic_query("bucket/object", &token, Some(2))
|
||||
.await
|
||||
.expect("local response")
|
||||
.expect("automatic task response");
|
||||
let direct = processor
|
||||
.execute_query_request_since("bucket/object".to_string(), token.clone(), Some(2))
|
||||
.await
|
||||
.expect("original query response");
|
||||
assert!(local.success);
|
||||
assert_eq!(local.request_id, direct.request_id);
|
||||
assert_eq!(local.data, direct.data, "same report encoding and cursor contract");
|
||||
assert_eq!(local.error, direct.error);
|
||||
let coordinator = Arc::new(HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None));
|
||||
let missing = HealChannelProcessor::new(coordinator)
|
||||
.execute_query_request("bucket/object".to_string(), token.clone())
|
||||
.await
|
||||
.expect("coordinator query");
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_slice(missing.data.as_deref().expect("coordinator response")).expect("coordinator JSON");
|
||||
assert_eq!(body["summary"], "notFound", "a remote manager does not own the automatic task");
|
||||
if state == "completed" {
|
||||
let completed = manager
|
||||
.completed_heals
|
||||
.lock()
|
||||
.await
|
||||
.get(&token)
|
||||
.cloned()
|
||||
.expect("original completion");
|
||||
manager
|
||||
.completed_heals
|
||||
.lock()
|
||||
.await
|
||||
.insert("retired-alias".to_string(), completed);
|
||||
assert!(
|
||||
manager
|
||||
.get_local_automatic_task_report("bucket/object", "retired-alias", None)
|
||||
.await
|
||||
.expect("retired alias lookup")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
manager
|
||||
.get_local_automatic_task_report("bucket/other", &token, None)
|
||||
.await
|
||||
.expect("wrong path lookup")
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
manager
|
||||
.get_local_automatic_task_report("bucket/object", "unknown", None)
|
||||
.await
|
||||
.expect("unknown token lookup")
|
||||
.is_none()
|
||||
);
|
||||
manager.insert_task_alias("admin-alias", &token).await;
|
||||
assert!(
|
||||
manager
|
||||
.get_local_automatic_task_report("bucket/object", "admin-alias", None)
|
||||
.await
|
||||
.expect("alias lookup")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_queued_duplicate_token_can_query_and_cancel_original_request() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
@@ -3886,6 +3768,38 @@ async fn test_retry_request_rescans_batch_when_all_exhausted_objects_are_retryab
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_exhausted_batches_do_not_restart_the_traversal() {
|
||||
for heal_type in [
|
||||
HealType::Bucket {
|
||||
bucket: "bucket".to_owned(),
|
||||
},
|
||||
HealType::Prefix {
|
||||
bucket: "bucket".to_owned(),
|
||||
prefix: "prefix/".to_owned(),
|
||||
},
|
||||
HealType::Cluster,
|
||||
] {
|
||||
let mut request = HealRequest::new(heal_type, HealOptions::default(), HealPriority::Normal);
|
||||
request.source = HealRequestSource::Admin;
|
||||
let task = HealTask::from_request(request, Arc::new(MockStorage));
|
||||
let result = Err(task
|
||||
.record_batch_failure(BatchHealFailure {
|
||||
scope: "completed-admin-traversal".to_owned(),
|
||||
failed: 1,
|
||||
retryable: 1,
|
||||
permanent: 0,
|
||||
first_object: "exhausted-object".to_owned(),
|
||||
first_error: Error::Storage(EcstoreError::SlowDown).to_string(),
|
||||
})
|
||||
.await);
|
||||
assert!(
|
||||
retry_request_for_result_with_budget(&task, &result).await.is_none(),
|
||||
"the exhausted admin batch must retain its original object receipts"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_retry_request_does_not_rescan_cancelled_or_timed_out_retryable_batch() {
|
||||
for terminal_error in [Error::TaskCancelled, Error::TaskTimeout] {
|
||||
@@ -3968,6 +3882,8 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
outcome: None,
|
||||
source: request.source,
|
||||
task_id: request.id.clone(),
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: request.heal_type,
|
||||
options: request.options.clone(),
|
||||
@@ -4816,6 +4732,8 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
outcome: None,
|
||||
source: request.source,
|
||||
task_id: request.id.clone(),
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: request.heal_type.clone(),
|
||||
options: request.options.clone(),
|
||||
@@ -4855,6 +4773,8 @@ async fn test_get_task_status_reads_recent_completed_status() {
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
outcome: None,
|
||||
source: HealRequestSource::Internal,
|
||||
task_id: "completed-token".to_string(),
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: HealType::Bucket {
|
||||
bucket: "bucket".to_string(),
|
||||
@@ -4888,6 +4808,8 @@ async fn test_get_task_report_for_path_reads_completed_items() {
|
||||
Arc::new(CompletedHealStatus {
|
||||
progress: None,
|
||||
outcome: None,
|
||||
source: HealRequestSource::Internal,
|
||||
task_id: "completed-token".to_string(),
|
||||
retained_bytes: std::sync::OnceLock::new(),
|
||||
heal_type: HealType::Object {
|
||||
bucket: "bucket".to_string(),
|
||||
|
||||
@@ -258,16 +258,6 @@ impl ScannerFixture {
|
||||
assert_eq!(request.id, self.task_id, "scanner must replay the durable UUID");
|
||||
assert_eq!(request.heal_endpoints, vec![self.local_targets[0].clone()]);
|
||||
assert_eq!(request.source, HealRequestSource::AutoHeal);
|
||||
assert!(
|
||||
self.manager
|
||||
.heal_queue
|
||||
.lock()
|
||||
.await
|
||||
.heap
|
||||
.iter()
|
||||
.any(|item| item.request.id == request.id && item.replacement_targets_frozen),
|
||||
"replayed durable targets cannot expand"
|
||||
);
|
||||
assert!(matches!(
|
||||
request.heal_type,
|
||||
HealType::ErasureSet { set_disk_id, buckets }
|
||||
|
||||
@@ -117,6 +117,8 @@ fn admin_request(heal_type: HealType) -> HealRequest {
|
||||
|
||||
fn completed_admin_status(heal_type: &HealType, completed_at: SystemTime) -> CompletedHealStatus {
|
||||
CompletedHealStatus {
|
||||
task_id: "admin-fixture".to_string(),
|
||||
source: HealRequestSource::Admin,
|
||||
outcome: None,
|
||||
heal_type: heal_type.clone(),
|
||||
options: HealOptions::default(),
|
||||
@@ -1239,6 +1241,8 @@ async fn root_recovery_completed_non_root_admin_is_queryable_after_restart() {
|
||||
.expect("durable bucket responsibility");
|
||||
let completed = CompletedHealStatus {
|
||||
outcome: None,
|
||||
source: request.source,
|
||||
task_id: request.id.clone(),
|
||||
heal_type: request.heal_type.clone(),
|
||||
options: request.options.clone(),
|
||||
status: HealTaskStatus::Completed,
|
||||
|
||||
@@ -37,7 +37,7 @@ use std::{
|
||||
collections::VecDeque,
|
||||
future::Future,
|
||||
sync::{
|
||||
Arc,
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering},
|
||||
},
|
||||
time::{Duration, Instant, SystemTime},
|
||||
@@ -59,6 +59,19 @@ pub(crate) struct OutcomeFinishTestHook {
|
||||
pub(crate) static OUTCOME_FINISH_TEST_HOOK: std::sync::LazyLock<tokio::sync::Mutex<Option<Arc<OutcomeFinishTestHook>>>> =
|
||||
std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
|
||||
|
||||
#[cfg(test)]
|
||||
static STORAGE_RECEIPT_TEST_HOOK: std::sync::LazyLock<tokio::sync::Mutex<Option<Arc<OutcomeFinishTestHook>>>> =
|
||||
std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_storage_receipt(task_id: &str) {
|
||||
let hook = STORAGE_RECEIPT_TEST_HOOK.lock().await.clone();
|
||||
if let Some(hook) = hook.filter(|hook| hook.task_id == task_id) {
|
||||
hook.reached.notify_one();
|
||||
hook.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_outcome_finish(task_id: &str) {
|
||||
let hook = OUTCOME_FINISH_TEST_HOOK.lock().await.clone();
|
||||
@@ -436,6 +449,9 @@ pub struct HealTask {
|
||||
replacement_start_retry_count: Arc<AtomicU32>,
|
||||
/// Set when the locked storage read classified the single target as absent.
|
||||
object_absent: Arc<AtomicBool>,
|
||||
/// The first resolved incarnation belongs to this task even if storage
|
||||
/// fails or the bucket is recreated before its terminal outcome is recorded.
|
||||
single_object_incarnation: Arc<OnceLock<Option<Uuid>>>,
|
||||
/// Task status
|
||||
pub status: Arc<RwLock<HealTaskStatus>>,
|
||||
/// Progress tracking
|
||||
@@ -508,6 +524,7 @@ impl HealTask {
|
||||
replacement_running: Arc::new(AtomicBool::new(false)),
|
||||
replacement_start_retry_count: Arc::new(AtomicU32::new(0)),
|
||||
object_absent: Arc::new(AtomicBool::new(false)),
|
||||
single_object_incarnation: Arc::new(OnceLock::new()),
|
||||
status: Arc::new(RwLock::new(HealTaskStatus::Pending)),
|
||||
progress: Arc::new(RwLock::new(HealProgress::new())),
|
||||
result_items: Arc::new(RwLock::new(VecDeque::with_capacity(MAX_RETAINED_HEAL_RESULT_ITEMS))),
|
||||
@@ -669,7 +686,9 @@ impl HealTask {
|
||||
} => (bucket, object, Some(version_id.as_str())),
|
||||
_ => return None,
|
||||
};
|
||||
Some(self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index))
|
||||
let mut identity = self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index);
|
||||
identity.bucket_incarnation_id = self.single_object_incarnation.get().copied().flatten();
|
||||
Some(identity)
|
||||
}
|
||||
|
||||
pub(super) async fn record_verified_storage_receipt(
|
||||
|
||||
@@ -306,9 +306,9 @@ impl HealTask {
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_stage(3, 3);
|
||||
}
|
||||
Err(Error::TaskExecutionFailed {
|
||||
message: format!("Failed to heal bucket {bucket}: {e}"),
|
||||
})
|
||||
// Keep storage/quorum identity for the cluster retry classifier.
|
||||
// The structured log above already carries the bucket context.
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -796,21 +796,27 @@ impl HealTask {
|
||||
match unavailable_recreate_error(&storage_result.item, &heal_opts) {
|
||||
Some(error) => Some(error),
|
||||
None => {
|
||||
telemetry_unknown |= !increment_counter(&mut healed);
|
||||
telemetry_unknown |= !add_bytes(
|
||||
&mut bytes,
|
||||
u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
|
||||
);
|
||||
#[cfg(test)]
|
||||
super::pause_storage_receipt(&self.id).await;
|
||||
recorded_authoritative_outcome =
|
||||
self.record_verified_storage_receipt(&identity, storage_result.receipt).await;
|
||||
if !recorded_authoritative_outcome
|
||||
&& !heal_opts.dry_run
|
||||
&& !storage_result.item.detail.is_empty()
|
||||
{
|
||||
detail = Some(storage_result.item.detail.clone());
|
||||
if !recorded_authoritative_outcome && self.cancel_token.is_cancelled() {
|
||||
Some(Error::TaskCancelled)
|
||||
} else {
|
||||
telemetry_unknown |= !increment_counter(&mut healed);
|
||||
telemetry_unknown |= !add_bytes(
|
||||
&mut bytes,
|
||||
u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
|
||||
);
|
||||
if !recorded_authoritative_outcome
|
||||
&& !heal_opts.dry_run
|
||||
&& !storage_result.item.detail.is_empty()
|
||||
{
|
||||
detail = Some(storage_result.item.detail.clone());
|
||||
}
|
||||
self.record_result_item(storage_result.item).await;
|
||||
None
|
||||
}
|
||||
self.record_result_item(storage_result.item).await;
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,6 +220,11 @@ impl HealTask {
|
||||
let mut expected_identity =
|
||||
self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index);
|
||||
expected_identity.bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, self.options.dry_run).await?;
|
||||
if matches!(self.heal_type, HealType::Object { .. }) {
|
||||
expected_identity.bucket_incarnation_id = *self
|
||||
.single_object_incarnation
|
||||
.get_or_init(|| expected_identity.bucket_incarnation_id);
|
||||
}
|
||||
|
||||
let heal_fut = self.storage.heal_object_with_receipt(bucket, object, version_id, &heal_opts);
|
||||
let heal_result = if self.source == HealRequestSource::ReadRepair {
|
||||
|
||||
@@ -305,6 +305,53 @@ mod canonical_outcome {
|
||||
assert_eq!(storage.healed_objects.lock().expect("healed objects").len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_cancellation_after_storage_success_keeps_typed_cancelled_responsibility() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt("object-a", None, HealObjectDisposition::Repaired, incarnation)]),
|
||||
)])),
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = Arc::new(bucket_task(storage.clone()));
|
||||
let hook = Arc::new(OutcomeFinishTestHook {
|
||||
task_id: task.id.clone(),
|
||||
reached: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
*STORAGE_RECEIPT_TEST_HOOK.lock().await = Some(hook.clone());
|
||||
let runner = {
|
||||
let task = task.clone();
|
||||
tokio::spawn(async move { task.execute().await })
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(10), hook.reached.notified())
|
||||
.await
|
||||
.expect("storage success must reach the receipt boundary");
|
||||
assert_eq!(storage.healed_objects.lock().expect("storage mutations").as_slice(), ["object-a"]);
|
||||
task.cancel().await.expect("cancel before receipt recording");
|
||||
hook.release.notify_one();
|
||||
let result = tokio::time::timeout(Duration::from_secs(10), runner)
|
||||
.await
|
||||
.expect("cancelled worker must stop")
|
||||
.expect("worker must not panic");
|
||||
STORAGE_RECEIPT_TEST_HOOK.lock().await.take();
|
||||
assert!(matches!(result, Err(Error::TaskCancelled)));
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Cancelled));
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
assert_eq!(outcome.counters.processed, 1);
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert_eq!(outcome.counters.unchanged, 0);
|
||||
assert_eq!(outcome.counters.unknown, 0);
|
||||
assert_eq!(outcome.counters.skipped, 1);
|
||||
assert_eq!(outcome.objects.len(), 1);
|
||||
assert_eq!(outcome.objects[0].identity.object, "object-a");
|
||||
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Cancelled);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_heal_rejects_stale_receipts_without_double_recording() {
|
||||
let expected_incarnation = Uuid::new_v4();
|
||||
@@ -809,6 +856,85 @@ mod canonical_outcome {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cluster_bucket_write_quorum_retries_preserve_budget_and_truthful_coverage() {
|
||||
for (failures, succeeds) in [(1, true), (4, false)] {
|
||||
let storage = Arc::new(MockStorage {
|
||||
listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
|
||||
typed_bucket_heal_errors: Mutex::new(HashMap::from([(
|
||||
"bucket-a".to_string(),
|
||||
(0..failures)
|
||||
.map(|_| Error::Storage(EcstoreError::ErasureWriteQuorum))
|
||||
.collect(),
|
||||
)])),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
let result = task.execute().await;
|
||||
assert_eq!(
|
||||
result.is_ok(),
|
||||
succeeds,
|
||||
"original cluster task must honor its bounded bucket retry budget"
|
||||
);
|
||||
let outcome = task.get_outcome().await;
|
||||
if succeeds {
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
} else {
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
let failure = task.take_batch_failure().await.expect("exhausted bucket retains its failure");
|
||||
assert_eq!((failure.failed, failure.retryable, failure.permanent), (1, 1, 0));
|
||||
}
|
||||
let calls = storage.bucket_heal_calls.lock().expect("original bucket calls");
|
||||
assert_eq!(
|
||||
calls.iter().filter(|bucket| bucket.as_str() == "bucket-a").count(),
|
||||
if succeeds { 2 } else { 4 }
|
||||
);
|
||||
assert_eq!(calls.last().map(String::as_str), Some("bucket-b"), "later buckets must still be visited");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_prepass_preserves_typed_quorum_and_terminal_access_denial() {
|
||||
for error in [
|
||||
Error::Storage(EcstoreError::ErasureWriteQuorum),
|
||||
Error::Disk(DiskError::FileAccessDenied),
|
||||
] {
|
||||
let retryable = error.is_recoverable_heal();
|
||||
let storage = Arc::new(MockStorage {
|
||||
typed_bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from([error]))])),
|
||||
..Default::default()
|
||||
});
|
||||
let task = bucket_task(storage.clone());
|
||||
let observed = task
|
||||
.heal_bucket("bucket-a")
|
||||
.await
|
||||
.expect_err("bucket prepass must retain the storage error");
|
||||
assert!(matches!(
|
||||
observed,
|
||||
Error::Storage(EcstoreError::ErasureWriteQuorum) | Error::Disk(DiskError::FileAccessDenied)
|
||||
));
|
||||
assert_eq!(observed.is_recoverable_heal(), retryable);
|
||||
assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]);
|
||||
assert!(
|
||||
storage.heal_object_calls.lock().expect("object calls").is_empty(),
|
||||
"failed prepass must not heal objects"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cancellation_and_deadline_leave_partial_coverage() {
|
||||
for cancel in [false, true] {
|
||||
@@ -1567,6 +1693,7 @@ struct MockStorage {
|
||||
listed_buckets: Mutex<Option<Vec<String>>>,
|
||||
bucket_discovery_incomplete: bool,
|
||||
bucket_heal_errors: Mutex<HashMap<String, VecDeque<&'static str>>>,
|
||||
typed_bucket_heal_errors: Mutex<HashMap<String, VecDeque<Error>>>,
|
||||
bucket_heal_calls: Mutex<Vec<String>>,
|
||||
delete_recovery_results: Mutex<HashMap<String, Result<Option<HealResultItem>>>>,
|
||||
delete_recovery_calls: Mutex<Vec<String>>,
|
||||
@@ -1815,6 +1942,127 @@ async fn failed_object_heal_rejects_matching_positive_storage_receipt() {
|
||||
| HealObjectDisposition::AuthoritativelyAbsent
|
||||
)
|
||||
}));
|
||||
assert_eq!(outcome.objects.len(), 1);
|
||||
assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(incarnation));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_failure_retains_incarnation_for_inner_and_outer_storage_errors() {
|
||||
for failure in [
|
||||
MockHealObjectOutcome::PermissionDenied,
|
||||
MockHealObjectOutcome::RetryableReadQuorum,
|
||||
MockHealObjectOutcome::RetryableLockTimeout,
|
||||
MockHealObjectOutcome::ErrOther("backend failed"),
|
||||
] {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
heal_object_outcome: Mutex::new(Some(failure)),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), Some("version-a".to_owned()));
|
||||
request.options.pool_index = Some(2);
|
||||
request.options.set_index = Some(3);
|
||||
let task = HealTask::from_request(request, storage);
|
||||
task.execute()
|
||||
.await
|
||||
.expect_err("the failed storage operation must remain failed");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.objects.len(), 1);
|
||||
assert_eq!(
|
||||
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
|
||||
(1, 1, 1)
|
||||
);
|
||||
assert_eq!((outcome.counters.healed, outcome.counters.unchanged, outcome.counters.unknown), (0, 0, 0));
|
||||
let object = &outcome.objects[0];
|
||||
assert_eq!(object.identity.bucket_incarnation_id, Some(incarnation));
|
||||
assert_eq!(object.identity.bucket, "bucket-a");
|
||||
assert_eq!(object.identity.object, "object-a");
|
||||
assert_eq!(object.identity.version_id.as_deref(), Some("version-a"));
|
||||
assert_eq!((object.identity.pool_index, object.identity.set_index), (Some(2), Some(3)));
|
||||
assert!(matches!(object.disposition, HealObjectDisposition::Failed(_)));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn object_abort_keeps_pre_storage_incarnation_after_bucket_recreation() {
|
||||
for cancel in [false, true] {
|
||||
let original = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(original)),
|
||||
block_heal_object: Mutex::new(true),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None);
|
||||
request.options.timeout = Some(Duration::from_secs(10));
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
let execution = task.execute();
|
||||
tokio::pin!(execution);
|
||||
assert!(futures::poll!(&mut execution).is_pending());
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("storage boundary").as_slice(), ["object-a"]);
|
||||
*storage.bucket_incarnation_id.lock().expect("recreated bucket") = Some(Uuid::new_v4());
|
||||
if cancel {
|
||||
task.cancel().await.expect("cancel blocked storage");
|
||||
} else {
|
||||
tokio::time::advance(Duration::from_secs(11)).await;
|
||||
}
|
||||
let result = execution.await;
|
||||
assert!(matches!(
|
||||
(&result, cancel),
|
||||
(Err(Error::TaskCancelled), true) | (Err(Error::TaskTimeout), false)
|
||||
));
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.objects.len(), 1);
|
||||
assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(original));
|
||||
assert_eq!((outcome.counters.processed, outcome.counters.skipped), (1, 1));
|
||||
assert_eq!((outcome.counters.failed, outcome.counters.healed, outcome.counters.unknown), (0, 0, 0));
|
||||
assert_eq!(
|
||||
outcome.objects[0].disposition,
|
||||
if cancel {
|
||||
HealObjectDisposition::Cancelled
|
||||
} else {
|
||||
HealObjectDisposition::Deferred {
|
||||
reason: HealDeferredReason::Deadline,
|
||||
retry_not_before: None,
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_unverified_outcome_keeps_original_incarnation_without_claiming_repair() {
|
||||
let original = Uuid::new_v4();
|
||||
let successor = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(original)),
|
||||
bucket_incarnation_after_object_heal: Mutex::new(Some(successor)),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None), storage.clone());
|
||||
task.execute().await.expect("legacy storage keeps its unverified result");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(original));
|
||||
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Unknown);
|
||||
assert_eq!((outcome.counters.healed, outcome.counters.unknown), (0, 1));
|
||||
assert_eq!(*storage.bucket_incarnation_id.lock().expect("successor"), Some(successor));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_dry_run_does_not_adopt_bucket_incarnation() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::new_v4())),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None);
|
||||
request.options.dry_run = true;
|
||||
let task = HealTask::from_request(request, storage);
|
||||
task.execute().await.expect("dry-run observation");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.objects.len(), 1);
|
||||
assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, None);
|
||||
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::DryRunObserved);
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2387,6 +2635,15 @@ impl HealStorageAPI for MockStorage {
|
||||
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
|
||||
self.bucket_heal_calls.lock().unwrap().push(bucket.to_string());
|
||||
self.bucket_heal_opts.lock().unwrap().push(*opts);
|
||||
if let Some(error) = self
|
||||
.typed_bucket_heal_errors
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut(bucket)
|
||||
.and_then(VecDeque::pop_front)
|
||||
{
|
||||
return Err(error);
|
||||
}
|
||||
if let Some(message) = self
|
||||
.bucket_heal_errors
|
||||
.lock()
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use rustfs_heal::{HealOptions, HealPriority, HealRequest, HealType};
|
||||
use rustfs_heal_contracts::heal_channel::HealRequestSource;
|
||||
use std::time::SystemTime;
|
||||
|
||||
#[test]
|
||||
fn existing_heal_request_literal_remains_constructible() {
|
||||
let now = SystemTime::now();
|
||||
let request = HealRequest {
|
||||
id: "request".to_string(),
|
||||
heal_type: HealType::ErasureSet {
|
||||
set_disk_id: "pool_0_set_0".to_string(),
|
||||
buckets: Vec::new(),
|
||||
},
|
||||
bucket_incarnation_id: None,
|
||||
expected_mrf_bucket_incarnation_id: None,
|
||||
options: HealOptions::default(),
|
||||
priority: HealPriority::Normal,
|
||||
source: HealRequestSource::AutoHeal,
|
||||
force_start: false,
|
||||
retry_attempts: 0,
|
||||
heal_endpoints: vec!["drive-a".to_string()],
|
||||
created_at: now,
|
||||
enqueued_at: now,
|
||||
};
|
||||
assert_eq!(request.heal_endpoints, ["drive-a"]);
|
||||
}
|
||||
@@ -550,3 +550,159 @@ fn admin_normal_protected_repairs_report_repaired_for_data_and_parity() {
|
||||
.join()
|
||||
.expect("heal fixture result");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn admin_mixed_batch_retains_repairs_after_object_retry_exhaustion() {
|
||||
std::thread::Builder::new()
|
||||
.stack_size(32 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("mixed batch runtime")
|
||||
.block_on(async {
|
||||
use rustfs_heal::heal::{
|
||||
manager::{HealConfig, HealManager},
|
||||
outcome::{HealExecutionOutcome, HealFailureClass, HealTraversalCoverage},
|
||||
task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType},
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::HealRequestSource;
|
||||
use std::sync::Arc;
|
||||
|
||||
let root = tempfile::tempdir().expect("mixed batch fixture");
|
||||
let env = TestECStoreEnv::builder().base_dir(root.path()).build().await;
|
||||
let bucket = "admin-mixed-batch";
|
||||
env.make_bucket(bucket, false).await;
|
||||
let set = env.ecstore.pools[0].get_disks(0);
|
||||
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
|
||||
let mut originals = Vec::new();
|
||||
for (object, missing, byte) in [("healthy", 0, 0x11), ("repaired", 1, 0x52), ("failed", 3, 0xa3)] {
|
||||
env.ecstore
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![byte; 5 * 1024 * 1024 + 123]),
|
||||
&ObjectOptions {
|
||||
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("commit complete fixture");
|
||||
for (slot, disk) in disks.iter().enumerate() {
|
||||
let meta = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("fixture metadata");
|
||||
assert_eq!(meta.erasure.data_blocks, 2, "failed fixture must retain R-1 shards");
|
||||
let path = env.disk_paths[slot]
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(meta.data_dir.expect("external shard").to_string())
|
||||
.join("part.1");
|
||||
let bytes = tokio::fs::read(&path).await.expect("original part");
|
||||
if slot < missing {
|
||||
tokio::fs::remove_file(&path).await.expect("inject missing part");
|
||||
}
|
||||
originals.push((object, path, bytes, slot < missing));
|
||||
}
|
||||
}
|
||||
let storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
|
||||
let manager = HealManager::new(
|
||||
storage.clone(),
|
||||
Some(HealConfig {
|
||||
enable_auto_heal: false,
|
||||
..Default::default()
|
||||
}),
|
||||
);
|
||||
manager.start().await.expect("start mixed batch manager");
|
||||
let mut request = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: bucket.to_owned(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
pool_index: Some(0),
|
||||
set_index: Some(0),
|
||||
timeout: Some(Duration::from_secs(150)),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::High,
|
||||
);
|
||||
request.source = HealRequestSource::Admin;
|
||||
let token = request.id.clone();
|
||||
manager.submit_heal_request(request).await.expect("admit original mixed task");
|
||||
let report = tokio::time::timeout(Duration::from_secs(150), async {
|
||||
loop {
|
||||
let report = manager.get_task_report(&token).await.expect("same-token report");
|
||||
if matches!(
|
||||
report.status,
|
||||
HealTaskStatus::Completed | HealTaskStatus::Failed { .. } | HealTaskStatus::Timeout
|
||||
) {
|
||||
break report;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("mixed task terminal deadline");
|
||||
manager.stop().await.expect("stop mixed task manager");
|
||||
for (object, path, original, missing) in originals {
|
||||
if object == "failed" && missing {
|
||||
assert!(!path.exists(), "below-R failure must not invent a shard");
|
||||
} else {
|
||||
assert_eq!(tokio::fs::read(&path).await.expect("preserved or repaired shard"), original);
|
||||
}
|
||||
}
|
||||
assert!(matches!(report.status, HealTaskStatus::Failed { .. }), "{report:?}");
|
||||
let outcome = report.outcome.as_ref().expect("canonical mixed outcome");
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::CompletedWithErrors);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
assert_eq!(
|
||||
(
|
||||
outcome.counters.processed,
|
||||
outcome.counters.healed,
|
||||
outcome.counters.unchanged,
|
||||
outcome.counters.failed,
|
||||
outcome.counters.skipped,
|
||||
outcome.counters.unknown,
|
||||
outcome.counters.attempt_failures
|
||||
),
|
||||
(3, 1, 1, 1, 0, 0, 4),
|
||||
"{outcome:?}"
|
||||
);
|
||||
for (object, disposition) in [
|
||||
("healthy", HealObjectDisposition::VerifiedHealthy),
|
||||
("repaired", HealObjectDisposition::Repaired),
|
||||
("failed", HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)),
|
||||
] {
|
||||
let receipt = outcome
|
||||
.objects
|
||||
.iter()
|
||||
.find(|item| item.identity.object == object)
|
||||
.expect("exact object receipt");
|
||||
assert_eq!(receipt.disposition, disposition);
|
||||
}
|
||||
drop(manager);
|
||||
let restarted = HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
enable_auto_heal: false,
|
||||
..Default::default()
|
||||
}),
|
||||
);
|
||||
restarted.start().await.expect("restart with retained failed batch");
|
||||
let restored = restarted.get_task_report(&token).await.expect("original token after restart");
|
||||
assert_eq!(restored.status, report.status);
|
||||
assert_eq!(restored.outcome, report.outcome);
|
||||
assert_eq!(restarted.get_queue_length().await, 0, "terminal batch must not replay");
|
||||
restarted.stop().await.expect("stop restarted mixed batch manager");
|
||||
});
|
||||
})
|
||||
.expect("mixed batch thread")
|
||||
.join()
|
||||
.expect("mixed batch result");
|
||||
}
|
||||
|
||||
@@ -26,9 +26,6 @@ use crate::{
|
||||
|
||||
/// Default shard count for guard storage (must be power of 2)
|
||||
const DEFAULT_GUARD_SHARD_COUNT: usize = 64;
|
||||
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
|
||||
const LOG_COMPONENT_LOCK: &str = "lock";
|
||||
const LOG_SUBSYSTEM_NAMESPACE: &str = "namespace";
|
||||
|
||||
type GuardShard = Arc<RwLock<HashMap<LockId, LocalGuardEntry>>>;
|
||||
type GuardStorage = Arc<Vec<GuardShard>>;
|
||||
@@ -336,22 +333,7 @@ impl LockClient for LocalClient {
|
||||
};
|
||||
if let Some(guard) = guard {
|
||||
// Guard automatically releases the lock when dropped
|
||||
let physical_guard_id = (!guard.guard.is_disabled()).then(|| guard.guard.guard_id());
|
||||
drop(guard.guard);
|
||||
if let Some(physical_guard_id) = physical_guard_id {
|
||||
tracing::trace!(
|
||||
target: "rustfs_lock::namespace_owner",
|
||||
event = EVENT_NAMESPACE_LOCK_RELEASE,
|
||||
component = LOG_COMPONENT_LOCK,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE,
|
||||
state = "mapped",
|
||||
peer_kind = "local",
|
||||
lock_id = %lock_id,
|
||||
process_id = std::process::id(),
|
||||
physical_guard_id,
|
||||
"Namespace peer lock mapped to physical guard"
|
||||
);
|
||||
}
|
||||
Ok(true)
|
||||
} else {
|
||||
// Lock not found or already released
|
||||
|
||||
@@ -412,11 +412,6 @@ impl DistributedLockGuard {
|
||||
&self.lock_id
|
||||
}
|
||||
|
||||
/// Actual granted peer identities, which may differ from the logical guard ID.
|
||||
pub fn granted_lock_ids(&self) -> impl Iterator<Item = &LockId> {
|
||||
self.entries.iter().map(|entry| &entry.lock_id)
|
||||
}
|
||||
|
||||
/// Whether the guard's heartbeat has observed a refresh-quorum loss.
|
||||
pub fn is_lock_lost(&self) -> bool {
|
||||
self.lock_lost.is_lost()
|
||||
|
||||
@@ -24,9 +24,6 @@ use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
/// Global counter for guard IDs to prevent double-release
|
||||
static GUARD_ID_COUNTER: AtomicU64 = AtomicU64::new(1);
|
||||
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
|
||||
const LOG_COMPONENT_LOCK: &str = "lock";
|
||||
const LOG_SUBSYSTEM_NAMESPACE: &str = "namespace";
|
||||
|
||||
/// RAII guard for fast object locks
|
||||
///
|
||||
@@ -112,7 +109,6 @@ impl FastLockGuard {
|
||||
record_lock_held_release(self.mode);
|
||||
// Unregister the guard after successful release
|
||||
shard.unregister_guard(self.guard_id);
|
||||
self.record_physical_release();
|
||||
}
|
||||
success
|
||||
} else {
|
||||
@@ -122,22 +118,6 @@ impl FastLockGuard {
|
||||
}
|
||||
}
|
||||
|
||||
fn record_physical_release(&self) {
|
||||
tracing::trace!(
|
||||
target: "rustfs_lock::namespace_owner",
|
||||
event = EVENT_NAMESPACE_LOCK_RELEASE,
|
||||
component = LOG_COMPONENT_LOCK,
|
||||
subsystem = LOG_SUBSYSTEM_NAMESPACE,
|
||||
state = "released",
|
||||
peer_kind = "local",
|
||||
process_id = std::process::id(),
|
||||
physical_guard_id = self.guard_id,
|
||||
resource = %self.key,
|
||||
owner = %self.owner,
|
||||
"Namespace physical guard release completed"
|
||||
);
|
||||
}
|
||||
|
||||
/// Check if the lock has been released
|
||||
pub fn is_released(&self) -> bool {
|
||||
self.released
|
||||
@@ -188,9 +168,6 @@ impl Drop for FastLockGuard {
|
||||
}
|
||||
// Always unregister the guard to prevent leaks, regardless of release success
|
||||
shard.unregister_guard(self.guard_id);
|
||||
if success {
|
||||
self.record_physical_release();
|
||||
}
|
||||
} else {
|
||||
// If guard was already released or disabled, just unregister it
|
||||
shard.unregister_guard(self.guard_id);
|
||||
|
||||
@@ -139,9 +139,15 @@ impl KeyName {
|
||||
Self::COMMON_KEYS.iter().filter(|k| k.is_server_derived()).map(|k| k.name())
|
||||
}
|
||||
|
||||
/// Returns the policy variable form of the key, e.g. `${jwt:sub}`.
|
||||
pub fn var_name(&self) -> String {
|
||||
format!("${{{}}}", Into::<&str>::into(self))
|
||||
match self {
|
||||
KeyName::Aws(s) => format!("${{aws:{}}}", Into::<&str>::into(s)),
|
||||
KeyName::Jwt(s) => format!("${{jwt:{}}}", Into::<&str>::into(s)),
|
||||
KeyName::Ldap(s) => format!("${{ldap:{}}}", Into::<&str>::into(s)),
|
||||
KeyName::Sts(s) => format!("${{sts:{}}}", Into::<&str>::into(s)),
|
||||
KeyName::Svc(s) => format!("${{svc:{}}}", Into::<&str>::into(s)),
|
||||
KeyName::S3(s) => format!("${{s3:{}}}", Into::<&str>::into(s)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -408,14 +414,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test_case(KeyName::Aws(AwsKeyName::AWSSourceIP), "${aws:SourceIp}")]
|
||||
#[test_case(KeyName::Jwt(JwtKeyName::JWTPrefUsername), "${jwt:preferred_username}")]
|
||||
#[test_case(KeyName::Ldap(LdapKeyName::Username), "${ldap:username}")]
|
||||
#[test_case(KeyName::S3(S3KeyName::S3LocationConstraint), "${s3:LocationConstraint}")]
|
||||
fn var_name_matches_policy_variable_syntax(key: KeyName, expected: &str) {
|
||||
assert_eq!(key.var_name(), expected);
|
||||
}
|
||||
|
||||
#[test_case("s3:x-amz-copy-source", KeyName::S3(S3KeyName::S3XAmzCopySource))]
|
||||
#[test_case("s3:VersionId", KeyName::S3(S3KeyName::S3VersionId) ; "aws_version_id")]
|
||||
#[test_case("s3:versionid", KeyName::S3(S3KeyName::S3VersionId) ; "minio_version_id")]
|
||||
|
||||
@@ -73,20 +73,6 @@ impl StringFunc {
|
||||
}
|
||||
}
|
||||
|
||||
/// Replaces every `${...}` reference to a common condition key that has a
|
||||
/// non-empty value in the request, not only the first such key.
|
||||
fn substitute_common_keys(mut c: String, values: &HashMap<String, Vec<String>>) -> String {
|
||||
for key in KeyName::COMMON_KEYS {
|
||||
if let Some(v) = values.get(key.name()).and_then(|x| x.first())
|
||||
&& !v.is_empty()
|
||||
{
|
||||
c = c.replace(&key.var_name(), v);
|
||||
}
|
||||
}
|
||||
|
||||
c
|
||||
}
|
||||
|
||||
impl FuncKeyValue<StringFuncValue> {
|
||||
async fn eval(
|
||||
&self,
|
||||
@@ -124,7 +110,17 @@ impl FuncKeyValue<StringFuncValue> {
|
||||
let fvalues = resolved_values
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|resolved_c| Cow::from(substitute_common_keys(resolved_c, values)))
|
||||
.map(|resolved_c| {
|
||||
let mut c = Cow::from(resolved_c);
|
||||
for key in KeyName::COMMON_KEYS {
|
||||
match values.get(key.name()).and_then(|x| x.first()) {
|
||||
Some(v) if !v.is_empty() => return Cow::Owned(c.to_mut().replace(&key.var_name(), v)),
|
||||
_ => continue,
|
||||
};
|
||||
}
|
||||
|
||||
c
|
||||
})
|
||||
.map(|x| if ignore_case { Cow::Owned(x.to_lowercase()) } else { x })
|
||||
.collect::<Set<_>>();
|
||||
|
||||
@@ -163,7 +159,17 @@ impl FuncKeyValue<StringFuncValue> {
|
||||
let matched = resolved_values
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|resolved_c| substitute_common_keys(resolved_c, values))
|
||||
.map(|resolved_c| {
|
||||
let mut c = Cow::from(resolved_c);
|
||||
for key in KeyName::COMMON_KEYS {
|
||||
match values.get(key.name()).and_then(|x| x.first()) {
|
||||
Some(v) if !v.is_empty() => return Cow::Owned(c.to_mut().replace(&key.var_name(), v)),
|
||||
_ => continue,
|
||||
};
|
||||
}
|
||||
|
||||
c
|
||||
})
|
||||
.any(|x| wildcard::is_match(x, v));
|
||||
|
||||
// Unqualified evaluation reports the plain aggregate match and lets the
|
||||
@@ -383,44 +389,6 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
// Real requests always carry earlier common keys such as `signatureversion`
|
||||
// and `SourceIp`; substitution must still reach later keys like `jwt:*`.
|
||||
#[test]
|
||||
fn test_condition_values_substitute_variables_after_other_common_keys() {
|
||||
let request = || {
|
||||
vec![
|
||||
("signatureversion", vec!["AWS4-HMAC-SHA256"]),
|
||||
("SourceIp", vec!["192.0.2.10"]),
|
||||
("preferred_username", vec!["alice"]),
|
||||
("prefix", vec!["alice/photos/"]),
|
||||
]
|
||||
};
|
||||
|
||||
assert!(test_eval_like(
|
||||
new_fkv("s3:prefix", vec!["${jwt:preferred_username}/*"]),
|
||||
false,
|
||||
false,
|
||||
request()
|
||||
));
|
||||
assert!(test_eval(
|
||||
new_fkv("s3:prefix", vec!["${jwt:preferred_username}/photos/"]),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
request()
|
||||
));
|
||||
assert!(!test_eval_like(
|
||||
new_fkv("s3:prefix", vec!["${jwt:preferred_username}/*"]),
|
||||
false,
|
||||
false,
|
||||
vec![
|
||||
("signatureversion", vec!["AWS4-HMAC-SHA256"]),
|
||||
("preferred_username", vec!["bob"]),
|
||||
("prefix", vec!["alice/photos/"]),
|
||||
]
|
||||
));
|
||||
}
|
||||
|
||||
#[test_case(new_fkv("s3:x-amz-copy-source", vec!["mybucket/myobject"]), false, vec![("x-amz-copy-source", vec!["mybucket/myobject"])] => true ; "1")]
|
||||
#[test_case(new_fkv("s3:x-amz-copy-source", vec!["mybucket/myobject"]), false, vec![("x-amz-copy-source", vec!["yourbucket/myobject"])] => false ; "2")]
|
||||
#[test_case(new_fkv("s3:x-amz-copy-source", vec!["mybucket/myobject"]), false, vec![] => false ; "3")]
|
||||
|
||||
@@ -352,19 +352,6 @@ mod tests {
|
||||
pollster::block_on(resource.is_match(object, &HashMap::new()))
|
||||
}
|
||||
|
||||
#[test_case("arn:aws:s3:::mybucket/${jwt:preferred_username}/*", "mybucket/alice/photo.jpg" => true; "jwt variable")]
|
||||
#[test_case("arn:aws:s3:::mybucket/${jwt:preferred_username}/*", "mybucket/bob/photo.jpg" => false; "jwt variable other user")]
|
||||
#[test_case("arn:aws:s3:::mybucket/${ldap:username}/*", "mybucket/alice-ldap/photo.jpg" => true; "ldap variable")]
|
||||
fn test_resource_is_match_substitutes_condition_variables(resource: &str, object: &str) -> bool {
|
||||
let resource: Resource = resource.try_into().unwrap();
|
||||
let conditions = HashMap::from([
|
||||
("preferred_username".to_string(), vec!["alice".to_string()]),
|
||||
("username".to_string(), vec!["alice-ldap".to_string()]),
|
||||
]);
|
||||
|
||||
pollster::block_on(resource.is_match(object, &conditions))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wildcard_resource_serialization_round_trip() {
|
||||
use crate::policy::resource::ResourceSet;
|
||||
|
||||
@@ -639,7 +639,7 @@ mod hot_reload_tests {
|
||||
format!("{d}BEGIN {l}{d}\n{body}\n{d}END {l}{d}\n", d = PEM_BOUNDARY_DASHES, l = PEM_OPENSSH_LABEL,)
|
||||
}
|
||||
|
||||
pub(super) fn test_ed25519_pem() -> String {
|
||||
fn test_ed25519_pem() -> String {
|
||||
build_pem_block(
|
||||
"b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\n\
|
||||
QyNTUxOQAAACCkeMEUpnJEbOMBXiQfjZcHZMEbHW3DlNRL+Jbi1cIqMgAAAKDviRiQ74kY\n\
|
||||
@@ -1010,7 +1010,7 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
|
||||
fn auth_publickey_offered(
|
||||
&mut self,
|
||||
user: &str,
|
||||
_key: russh::server::PublicKeyOffer<'_>,
|
||||
_key: &keys::PublicKey,
|
||||
) -> impl std::future::Future<Output = Result<Auth, Self::Error>> + Send {
|
||||
async { Ok(Auth::reject()) }
|
||||
}
|
||||
@@ -1019,7 +1019,7 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
|
||||
fn auth_publickey(
|
||||
&mut self,
|
||||
user: &str,
|
||||
_key: russh::server::VerifiedPublicKey<'_>,
|
||||
_key: &keys::PublicKey,
|
||||
) -> impl std::future::Future<Output = Result<Auth, Self::Error>> + Send {
|
||||
async { Ok(Auth::reject()) }
|
||||
}
|
||||
@@ -1594,80 +1594,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn ssh_handshake_rejects_public_key_authentication() {
|
||||
use crate::common::dummy_storage::DummyBackend;
|
||||
use russh::client::{self, AuthResult};
|
||||
use russh::keys::{PrivateKeyWithHashAlg, PublicKeyOrCertificate};
|
||||
|
||||
struct PinnedHostKey(keys::PublicKey);
|
||||
|
||||
impl client::Handler for PinnedHostKey {
|
||||
type Error = russh::Error;
|
||||
|
||||
async fn check_server_key(&mut self, offered: &PublicKeyOrCertificate) -> Result<bool, Self::Error> {
|
||||
Ok(matches!(offered, PublicKeyOrCertificate::PublicKey { key, .. } if key.key_data() == self.0.key_data()))
|
||||
}
|
||||
}
|
||||
|
||||
timeout(Duration::from_secs(30), async {
|
||||
let key = PrivateKey::from_openssh(super::hot_reload_tests::test_ed25519_pem()).expect("decode test SSH key");
|
||||
let trusted_host_key = key.public_key().clone();
|
||||
let config = build_ssh_config(vec![key.clone()], 600, "SSH-2.0-RustFS");
|
||||
let peer_addr = "127.0.0.1:12345".parse().expect("test peer address");
|
||||
let handler = SshSessionHandler {
|
||||
storage: Arc::new(DummyBackend::new()),
|
||||
peer_addr,
|
||||
session_context: None,
|
||||
channels: HashMap::new(),
|
||||
read_only: false,
|
||||
part_size: super::super::test_support::TEST_PART_SIZE,
|
||||
handles_per_session: DEFAULT_HANDLES_PER_SESSION,
|
||||
backend_op_timeout_secs: DEFAULT_BACKEND_OP_TIMEOUT_SECS,
|
||||
read_cache_window: READ_CACHE_WINDOW_DEFAULT,
|
||||
read_cache_total_mem_limit: READ_CACHE_TOTAL_MEM_DEFAULT,
|
||||
read_cache_in_use: Arc::new(AtomicU64::new(0)),
|
||||
session_diag: Arc::new(SessionDiag::new(peer_addr, peer_addr)),
|
||||
};
|
||||
let (client_stream, server_stream) = tokio::io::duplex(64 * 1024);
|
||||
let server = tokio::spawn(async move {
|
||||
let session = russh::server::run_stream(config, server_stream, handler)
|
||||
.await
|
||||
.expect("start test SSH session");
|
||||
// The client deliberately disconnects after the rejected authentication.
|
||||
let _ = session.await;
|
||||
});
|
||||
let mut client =
|
||||
client::connect_stream(Arc::new(client::Config::default()), client_stream, PinnedHostKey(trusted_host_key))
|
||||
.await
|
||||
.expect("complete SSH handshake with the pinned host key");
|
||||
let auth = client
|
||||
.authenticate_publickey("public-key-auth-disabled", PrivateKeyWithHashAlg::new(Arc::new(key), None))
|
||||
.await
|
||||
.expect("receive public-key authentication response");
|
||||
match auth {
|
||||
AuthResult::Failure {
|
||||
remaining_methods,
|
||||
partial_success,
|
||||
} => {
|
||||
assert!(!partial_success, "public keys must not grant partial authentication");
|
||||
assert!(remaining_methods.contains(&MethodKind::Password));
|
||||
assert!(!remaining_methods.contains(&MethodKind::PublicKey));
|
||||
}
|
||||
AuthResult::Success => panic!("public-key authentication must remain disabled"),
|
||||
}
|
||||
client
|
||||
.disconnect(russh::Disconnect::ByApplication, "test complete", "")
|
||||
.await
|
||||
.expect("disconnect test client");
|
||||
drop(client);
|
||||
server.await.expect("join test SSH server");
|
||||
})
|
||||
.await
|
||||
.expect("SSH authentication rejection must complete within the test deadline");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ssh_config_advertises_only_password() {
|
||||
let config = build_ssh_config(Vec::new(), 600, "SSH-2.0-RustFS");
|
||||
|
||||
@@ -1410,46 +1410,27 @@ impl DataUsageCache {
|
||||
Some((flattened, visited.len()))
|
||||
}
|
||||
|
||||
fn flatten_with_guard<'a>(
|
||||
&'a self,
|
||||
root: &'a DataUsageEntry,
|
||||
visited: &mut HashSet<&'a str>,
|
||||
depth: usize,
|
||||
) -> DataUsageEntry {
|
||||
// A summary never owns child links. Cloning and clearing the set
|
||||
// retains its allocation in every completed subtree and update.
|
||||
let mut flat = DataUsageEntry {
|
||||
children: HashSet::new(),
|
||||
size: root.size,
|
||||
objects: root.objects,
|
||||
versions: root.versions,
|
||||
delete_markers: root.delete_markers,
|
||||
obj_sizes: root.obj_sizes.clone(),
|
||||
obj_versions: root.obj_versions.clone(),
|
||||
replication_stats: root.replication_stats.clone(),
|
||||
compacted: root.compacted,
|
||||
failed_objects: root.failed_objects,
|
||||
all_tier_stats: root.all_tier_stats.clone(),
|
||||
unknown_tier_stats: root.unknown_tier_stats.clone(),
|
||||
tier_accounting_proof: root.tier_accounting_proof,
|
||||
};
|
||||
fn flatten_with_guard(&self, root: &DataUsageEntry, visited: &mut HashSet<String>, depth: usize) -> DataUsageEntry {
|
||||
let mut root = root.clone();
|
||||
if depth >= MAX_DATA_USAGE_CACHE_DEPTH {
|
||||
return flat;
|
||||
root.children.clear();
|
||||
return root;
|
||||
}
|
||||
|
||||
for id in &root.children {
|
||||
if !visited.insert(id.as_str()) {
|
||||
for id in root.children.clone().iter() {
|
||||
if !visited.insert(id.clone()) {
|
||||
continue;
|
||||
}
|
||||
if let Some(e) = self.cache.get(id) {
|
||||
if e.children.is_empty() {
|
||||
flat.merge(e);
|
||||
} else {
|
||||
flat.merge(&self.flatten_with_guard(e, visited, depth + 1));
|
||||
let mut e = e.clone();
|
||||
if !e.children.is_empty() {
|
||||
e = self.flatten_with_guard(&e, visited, depth + 1);
|
||||
}
|
||||
root.merge(&e);
|
||||
}
|
||||
}
|
||||
flat
|
||||
root.children.clear();
|
||||
root
|
||||
}
|
||||
|
||||
pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
|
||||
@@ -1486,31 +1467,38 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn delete_recursive(&mut self, hash: &DataUsageHash) {
|
||||
self.delete_recursive_guard(&hash.0, 0);
|
||||
let mut visited = HashSet::new();
|
||||
self.delete_recursive_guard(hash, &mut visited, 0);
|
||||
}
|
||||
|
||||
fn delete_recursive_guard(&mut self, path: &str, depth: usize) {
|
||||
// Removal transfers child ownership and makes cyclic/shared references
|
||||
// no-ops, without cloning paths or maintaining a second visited set.
|
||||
let Some(children) = self.cache.remove(path).map(|entry| entry.children) else {
|
||||
fn delete_recursive_guard(&mut self, hash: &DataUsageHash, visited: &mut HashSet<String>, depth: usize) {
|
||||
if !visited.insert(hash.key()) {
|
||||
return;
|
||||
};
|
||||
}
|
||||
|
||||
let mut need_remove = Vec::new();
|
||||
if let Some(v) = self.cache.get(&hash.string()) {
|
||||
for child in v.children.iter() {
|
||||
need_remove.push(child.clone());
|
||||
}
|
||||
}
|
||||
self.cache.remove(&hash.string());
|
||||
if depth >= MAX_DATA_USAGE_CACHE_DEPTH {
|
||||
return;
|
||||
}
|
||||
for child in children {
|
||||
self.delete_recursive_guard(&child, depth + 1);
|
||||
for child in need_remove {
|
||||
self.delete_recursive_guard(&DataUsageHash(child), visited, depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn size_recursive(&self, path: &str) -> Option<DataUsageEntry> {
|
||||
match self.cache.get_key_value(&hash_path(path).key()) {
|
||||
Some((key, root)) => {
|
||||
match self.find(path) {
|
||||
Some(root) => {
|
||||
if root.children.is_empty() {
|
||||
return Some(root.clone());
|
||||
}
|
||||
let mut visited = HashSet::new();
|
||||
visited.insert(key.as_str());
|
||||
visited.insert(hash_path(path).key());
|
||||
let mut flat = self.flatten_with_guard(root, &mut visited, 0);
|
||||
if flat.replication_stats.as_ref().is_some_and(|stats| stats.is_empty()) {
|
||||
flat.replication_stats = None;
|
||||
@@ -1565,7 +1553,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
|
||||
let Some(e) = self.cache.get(&path.0) else {
|
||||
let Some(e) = self.cache.get(&path.key()).cloned() else {
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -1617,13 +1605,12 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn total_children_rec(&self, path: &str) -> usize {
|
||||
let root = hash_path(path);
|
||||
let mut visited = HashSet::new();
|
||||
visited.insert(root.0.as_str());
|
||||
visited.insert(hash_path(path).key());
|
||||
self.total_children_rec_guard(path, &mut visited, 0)
|
||||
}
|
||||
|
||||
fn total_children_rec_guard<'a>(&'a self, path: &str, visited: &mut HashSet<&'a str>, depth: usize) -> usize {
|
||||
fn total_children_rec_guard(&self, path: &str, visited: &mut HashSet<String>, depth: usize) -> usize {
|
||||
let Some(root) = self.find(path) else {
|
||||
return 0;
|
||||
};
|
||||
@@ -1633,7 +1620,7 @@ impl DataUsageCache {
|
||||
|
||||
let mut n = 0;
|
||||
for ch in root.children.iter() {
|
||||
if visited.insert(ch.as_str()) {
|
||||
if visited.insert(ch.clone()) {
|
||||
n += 1 + self.total_children_rec_guard(ch, visited, depth + 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2428,69 +2428,6 @@ fn test_data_usage_cache_recursive_helpers_tolerate_cycles() {
|
||||
assert!(!copied.cache.contains_key(&child_hash.key()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_recursive_preserves_depth_boundary_and_unrelated_entries() {
|
||||
let mut cache = DataUsageCache::default();
|
||||
for depth in 0..=MAX_DATA_USAGE_CACHE_DEPTH + 1 {
|
||||
let mut entry = DataUsageEntry::default();
|
||||
if depth <= MAX_DATA_USAGE_CACHE_DEPTH {
|
||||
entry.children.insert(format!("node-{}", depth + 1));
|
||||
}
|
||||
cache.cache.insert(format!("node-{depth}"), entry);
|
||||
}
|
||||
cache.cache.insert("unrelated".to_string(), DataUsageEntry::default());
|
||||
|
||||
cache.delete_recursive(&DataUsageHash("node-0".to_string()));
|
||||
|
||||
assert_eq!(cache.cache.len(), 2, "delete only reachable entries within the depth limit");
|
||||
assert!(cache.cache.contains_key(&format!("node-{}", MAX_DATA_USAGE_CACHE_DEPTH + 1)));
|
||||
assert!(cache.cache.contains_key("unrelated"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_summary_releases_wide_child_sets() {
|
||||
let root = hash_path("bucket");
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.replace_hashed(
|
||||
&root,
|
||||
&None,
|
||||
&DataUsageEntry {
|
||||
size: 17,
|
||||
objects: 1,
|
||||
versions: 2,
|
||||
delete_markers: 1,
|
||||
failed_objects: 1,
|
||||
all_tier_stats: Some(AllTierStats::new()),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
for index in 0..10_000 {
|
||||
let child = hash_path(&format!("bucket/{index:05}"));
|
||||
cache.replace_hashed(
|
||||
&child,
|
||||
&Some(root.clone()),
|
||||
&DataUsageEntry {
|
||||
size: 3,
|
||||
objects: 1,
|
||||
versions: 2,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let summary = cache.size_recursive("bucket").expect("wide bucket summary");
|
||||
assert_eq!((summary.size, summary.objects, summary.versions), (30_017, 10_001, 20_002));
|
||||
assert_eq!((summary.delete_markers, summary.failed_objects), (1, 1));
|
||||
assert_eq!(summary.children.capacity(), 0, "summaries must not retain the wide namespace allocation");
|
||||
assert!(summary.all_tier_stats.is_some(), "preserve the root's optional statistics representation");
|
||||
assert_eq!(cache.find("bucket").expect("unchanged root").children.len(), 10_000);
|
||||
|
||||
cache.cache.insert("unrelated".to_string(), DataUsageEntry::default());
|
||||
cache.delete_recursive(&root);
|
||||
assert_eq!(cache.cache.len(), 1, "wide subtree deletion must consume every child exactly once");
|
||||
assert!(cache.cache.contains_key("unrelated"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_usage_cache_flatten_does_not_count_root_twice_in_cycle() {
|
||||
let root_hash = hash_path("bucket");
|
||||
|
||||
@@ -236,7 +236,6 @@ async fn profile_case(objects: usize, scenario: &str, samples: usize) {
|
||||
|
||||
let mut clone_ns = Vec::new();
|
||||
let mut copy_ns = Vec::new();
|
||||
let mut delete_ns = Vec::new();
|
||||
let mut flatten_ns = Vec::new();
|
||||
let mut encode_ns = Vec::new();
|
||||
let mut save_ns = Vec::new();
|
||||
@@ -258,14 +257,6 @@ async fn profile_case(objects: usize, scenario: &str, samples: usize) {
|
||||
same_cache(&copied, &cache);
|
||||
drop(copied);
|
||||
|
||||
let mut deleted = cache.clone();
|
||||
let root = hash_path("bucket");
|
||||
let started = WallInstant::now();
|
||||
deleted.delete_recursive(black_box(&root));
|
||||
delete_ns.push(elapsed_ns(started));
|
||||
assert!(deleted.cache.is_empty(), "delete the root and every reachable entry");
|
||||
drop(deleted);
|
||||
|
||||
let started = WallInstant::now();
|
||||
let aggregate = black_box(cache.checked_flatten("bucket").expect("valid fixture tree"));
|
||||
flatten_ns.push(elapsed_ns(started));
|
||||
@@ -335,7 +326,6 @@ async fn profile_case(objects: usize, scenario: &str, samples: usize) {
|
||||
"changed_entry_wire_bytes": changed_entry_wire_bytes, "save_body_bytes_per_sample": expected_wire.len() * 2,
|
||||
"snapshot_complete": cache.info.snapshot_complete,
|
||||
"clone": quantiles(clone_ns), "copy_with_children": quantiles(copy_ns), "checked_flatten": quantiles(flatten_ns),
|
||||
"delete_recursive": quantiles(delete_ns),
|
||||
"encode": quantiles(encode_ns), "save_inclusive": quantiles(save_ns), "memory_backend_ingest": quantiles(ingest_ns),
|
||||
})
|
||||
);
|
||||
|
||||
@@ -28,7 +28,7 @@ use crate::storage_api::owner::{
|
||||
EcstoreSetDisks, EcstoreStore, ecstore_is_reserved_or_invalid_bucket, ecstore_resolve_object_store_handle,
|
||||
};
|
||||
use futures::future::join_all;
|
||||
use rustfs_data_usage::{PrefixUsageEntry, PrefixUsageQuery, PrefixUsageSummary};
|
||||
use rustfs_data_usage::{PrefixUsageEntry, PrefixUsageSummary};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, SystemTime};
|
||||
@@ -133,7 +133,6 @@ async fn compute_prefix_usage(
|
||||
) -> BucketPrefixUsageResponse {
|
||||
let sets: Vec<Arc<EcstoreSetDisks>> = store.all_set_disks();
|
||||
let sets_total = sets.len();
|
||||
let per_set_max_entries = if sets_total <= 1 { max_entries } else { usize::MAX };
|
||||
let cache_name = format!("{bucket}/{DATA_USAGE_CACHE_NAME}");
|
||||
|
||||
let per_set = join_all(sets.into_iter().map(|set| {
|
||||
@@ -176,25 +175,12 @@ async fn compute_prefix_usage(
|
||||
return None;
|
||||
}
|
||||
let last_update = loaded.info.last_update;
|
||||
// Fetch all sub-prefixes per set when sets_total > 1 so cross-set
|
||||
// aggregation merges every set's contribution before truncating
|
||||
// to the caller's max_entries bound.
|
||||
let query = loaded.prefix_usage(bucket, prefix, per_set_max_entries);
|
||||
let query = loaded.prefix_usage(bucket, prefix, max_entries);
|
||||
Some((query, last_update))
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
|
||||
aggregate_set_prefix_usage(bucket, prefix, max_entries, sets_total, per_set.into_iter().flatten())
|
||||
}
|
||||
|
||||
fn aggregate_set_prefix_usage(
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
max_entries: usize,
|
||||
sets_total: usize,
|
||||
per_set: impl IntoIterator<Item = (Option<PrefixUsageQuery>, Option<SystemTime>)>,
|
||||
) -> BucketPrefixUsageResponse {
|
||||
let mut usage = PrefixUsageSummary::default();
|
||||
let mut sub_prefix_map: HashMap<String, PrefixUsageSummary> = HashMap::new();
|
||||
let mut sets_reporting = 0usize;
|
||||
@@ -204,7 +190,7 @@ fn aggregate_set_prefix_usage(
|
||||
let mut truncated = false;
|
||||
let mut last_update: Option<SystemTime> = None;
|
||||
|
||||
for (query, set_last_update) in per_set {
|
||||
for (query, set_last_update) in per_set.into_iter().flatten() {
|
||||
// last_update counts every set that has scanned the bucket, even
|
||||
// when the prefix itself is absent on that set.
|
||||
if let Some(set_last_update) = set_last_update
|
||||
@@ -243,7 +229,8 @@ fn aggregate_set_prefix_usage(
|
||||
.cmp(&left.usage.size)
|
||||
.then_with(|| left.prefix.cmp(&right.prefix))
|
||||
});
|
||||
// Enforce the caller bound on the merged cross-set breakdown.
|
||||
// Merged rows can exceed max_entries only when per-set truncation
|
||||
// already flagged; enforce the caller bound on the merged view too.
|
||||
if sub_prefixes.len() > max_entries {
|
||||
truncated = true;
|
||||
sub_prefixes.truncate(max_entries);
|
||||
@@ -359,84 +346,4 @@ mod tests {
|
||||
let map = guard.as_ref().expect("seeded");
|
||||
assert!(map.len() <= CACHE_MAX_ENTRIES, "cache must stay bounded, got {}", map.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_set_aggregation_merges_untruncated_set_breakdowns_before_applying_max_entries() {
|
||||
use crate::data_usage_define::{DataUsageCache, DataUsageEntry};
|
||||
|
||||
let mut set0 = DataUsageCache::default();
|
||||
set0.info.name = "bucket".to_string();
|
||||
set0.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
size: 190,
|
||||
objects: 2,
|
||||
versions: 2,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
set0.replace(
|
||||
"bucket/a",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: 100,
|
||||
objects: 1,
|
||||
versions: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
set0.replace(
|
||||
"bucket/b",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: 90,
|
||||
objects: 1,
|
||||
versions: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let mut set1 = DataUsageCache::default();
|
||||
set1.info.name = "bucket".to_string();
|
||||
set1.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
size: 190,
|
||||
objects: 2,
|
||||
versions: 2,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
set1.replace(
|
||||
"bucket/c",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: 100,
|
||||
objects: 1,
|
||||
versions: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
set1.replace(
|
||||
"bucket/b",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: 90,
|
||||
objects: 1,
|
||||
versions: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
let q0 = set0.prefix_usage("bucket", "", usize::MAX);
|
||||
let q1 = set1.prefix_usage("bucket", "", usize::MAX);
|
||||
let merged = super::aggregate_set_prefix_usage("bucket", "", 1, 2, vec![(q0, None), (q1, None)]);
|
||||
assert!(merged.truncated);
|
||||
assert_eq!(merged.sub_prefixes.len(), 1);
|
||||
assert_eq!(merged.sub_prefixes[0].prefix, "b");
|
||||
assert_eq!(merged.sub_prefixes[0].usage.size, 180);
|
||||
assert_eq!(merged.sub_prefixes[0].usage.objects, 2);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5440,8 +5440,8 @@ async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_met
|
||||
.await
|
||||
.expect_err("truncated primary must fail in the real object reader");
|
||||
assert!(
|
||||
error.to_string().contains("InlineData value out of range"),
|
||||
"unexpected truncated-primary error: {error}"
|
||||
matches!(error, EcstoreError::FileCorrupt),
|
||||
"unexpected truncated-primary error: {error:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
read_config_with_revision(store.clone(), &backup_path)
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs::FileType;
|
||||
use std::io::ErrorKind;
|
||||
@@ -21,10 +20,10 @@ use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use crate::ReplTargetSizeSummary;
|
||||
use crate::data_usage_define::{
|
||||
DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageHash, DataUsageHashMap,
|
||||
DataUsageRawEnumerationCursor, DataUsageScanCheckpoint, DataUsageScanCheckpointReason, MAX_DATA_USAGE_CACHE_DEPTH,
|
||||
PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry, SizeSummary, hash_path,
|
||||
scan_frontier_covers_path,
|
||||
DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageCacheSource, DataUsageEntry, DataUsageHash,
|
||||
DataUsageHashMap, DataUsageRawEnumerationCursor, DataUsageScanCheckpoint, DataUsageScanCheckpointReason,
|
||||
MAX_DATA_USAGE_CACHE_DEPTH, PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry,
|
||||
SizeSummary, hash_path, scan_frontier_covers_path,
|
||||
};
|
||||
use crate::error::ScannerError;
|
||||
use crate::raw_page_index::{RawEnumerationPageIndex, RawEnumerationPageWriter};
|
||||
@@ -752,36 +751,11 @@ struct ScannerCheckpointAncestor {
|
||||
entry: DataUsageEntry,
|
||||
}
|
||||
|
||||
struct ScannerUsageUpdates {
|
||||
sender: mpsc::Sender<DataUsageEntry>,
|
||||
cache: DataUsageCache,
|
||||
last_update: SystemTime,
|
||||
}
|
||||
|
||||
impl ScannerUsageUpdates {
|
||||
fn new(sender: mpsc::Sender<DataUsageEntry>, bucket: String) -> Self {
|
||||
Self {
|
||||
sender,
|
||||
cache: DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: bucket,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
last_update: SystemTime::UNIX_EPOCH,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_due(&self) -> bool {
|
||||
self.last_update.elapsed().unwrap_or(Duration::ZERO) >= Duration::from_secs(60)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct FolderScanner {
|
||||
root: String,
|
||||
old_cache: DataUsageCache,
|
||||
new_cache: DataUsageCache,
|
||||
update_cache: DataUsageCache,
|
||||
|
||||
data_usage_scanner_debug: bool,
|
||||
heal_object_select: u32,
|
||||
@@ -799,10 +773,11 @@ pub struct FolderScanner {
|
||||
disks: Vec<Arc<Disk>>,
|
||||
disks_quorum: usize,
|
||||
|
||||
usage_updates: Option<ScannerUsageUpdates>,
|
||||
updates: Option<mpsc::Sender<DataUsageEntry>>,
|
||||
checkpoint_tx: Option<mpsc::Sender<DataUsageCache>>,
|
||||
checkpoint_ancestors: Vec<ScannerCheckpointAncestor>,
|
||||
checkpoint_depth: usize,
|
||||
last_update: SystemTime,
|
||||
checkpoint_objects: u64,
|
||||
object_metadata_visits: metrics::Counter,
|
||||
last_checkpoint_objects: u64,
|
||||
@@ -823,6 +798,10 @@ pub struct FolderScanner {
|
||||
resume_failed_paths: Vec<String>,
|
||||
coverage_gap: bool,
|
||||
raw_enumeration_progress: Vec<RawEnumerationProgress>,
|
||||
pending_heal_sync_deferred: bool,
|
||||
pending_heal_batch_dirty: bool,
|
||||
#[cfg(test)]
|
||||
pending_heal_sync_count: usize,
|
||||
pending_size_reconciliation_keys: HashSet<String>,
|
||||
pending_size_reconciliation_scopes: HashSet<String>,
|
||||
pending_size_reconciliation_truncated: bool,
|
||||
@@ -1057,9 +1036,8 @@ impl FolderScanner {
|
||||
|
||||
fn reuse_clean_root_child(&mut self, child: &CachedFolder, child_hash: &DataUsageHash, into: &mut DataUsageEntry) {
|
||||
self.new_cache.copy_with_children(&self.old_cache, child_hash, &child.parent);
|
||||
if let Some(updates) = &mut self.usage_updates {
|
||||
updates.cache.copy_with_children(&self.old_cache, child_hash, &child.parent);
|
||||
}
|
||||
self.update_cache
|
||||
.copy_with_children(&self.old_cache, child_hash, &child.parent);
|
||||
into.add_child(child_hash);
|
||||
}
|
||||
|
||||
@@ -1167,8 +1145,8 @@ impl FolderScanner {
|
||||
}
|
||||
|
||||
/// Apply the per-object size-resolution ledger updates in one place. The
|
||||
/// scanner cache is the durable boundary; progress updates contain only
|
||||
/// usage totals and never own a copy of the reconciliation ledger.
|
||||
/// scanner cache is the durable boundary; both working copies are updated
|
||||
/// so an incremental publication cannot lose a debt or its resolution.
|
||||
fn apply_size_reconciliation(&mut self, summary: &SizeSummary) {
|
||||
let now = Self::now_secs();
|
||||
self.pending_size_reconciliation_keys
|
||||
@@ -1181,25 +1159,26 @@ impl FolderScanner {
|
||||
);
|
||||
self.pending_size_reconciliation_truncated |= summary.size_reconciliation_truncated;
|
||||
|
||||
let info = &mut self.new_cache.info;
|
||||
for incoming in &summary.size_reconciliation {
|
||||
if let Some(existing) = info.size_reconciliation.get_mut(&incoming.key) {
|
||||
existing.reason = incoming.reason.clone();
|
||||
existing.physical_size = incoming.physical_size;
|
||||
existing.generation = incoming.generation.clone();
|
||||
existing.version_id = incoming.version_id.clone();
|
||||
existing.attempts = existing.attempts.saturating_add(1);
|
||||
continue;
|
||||
}
|
||||
for info in [&mut self.new_cache.info, &mut self.update_cache.info] {
|
||||
for incoming in &summary.size_reconciliation {
|
||||
if let Some(existing) = info.size_reconciliation.get_mut(&incoming.key) {
|
||||
existing.reason = incoming.reason.clone();
|
||||
existing.physical_size = incoming.physical_size;
|
||||
existing.generation = incoming.generation.clone();
|
||||
existing.version_id = incoming.version_id.clone();
|
||||
existing.attempts = existing.attempts.saturating_add(1);
|
||||
continue;
|
||||
}
|
||||
|
||||
if size_reconciliation_entry_bytes(incoming) > MAX_SIZE_RECONCILIATION_BYTES_PER_BUCKET {
|
||||
continue;
|
||||
}
|
||||
if size_reconciliation_entry_bytes(incoming) > MAX_SIZE_RECONCILIATION_BYTES_PER_BUCKET {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut entry = incoming.clone();
|
||||
entry.first_seen = now;
|
||||
entry.attempts = 1;
|
||||
info.size_reconciliation.insert(entry.key.clone(), entry);
|
||||
let mut entry = incoming.clone();
|
||||
entry.first_seen = now;
|
||||
entry.attempts = 1;
|
||||
info.size_reconciliation.insert(entry.key.clone(), entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1209,24 +1188,27 @@ impl FolderScanner {
|
||||
let scopes = std::mem::take(&mut self.pending_size_reconciliation_scopes);
|
||||
let truncated = std::mem::replace(&mut self.pending_size_reconciliation_truncated, false);
|
||||
|
||||
let info = &mut self.new_cache.info;
|
||||
if !truncated {
|
||||
info.size_reconciliation.retain(|key, entry| {
|
||||
!scopes.contains(&size_reconciliation_scope_key(&entry.bucket, &entry.object)) || current_keys.contains(key)
|
||||
});
|
||||
for info in [&mut self.new_cache.info, &mut self.update_cache.info] {
|
||||
if !truncated {
|
||||
info.size_reconciliation.retain(|key, entry| {
|
||||
!scopes.contains(&size_reconciliation_scope_key(&entry.bucket, &entry.object)) || current_keys.contains(key)
|
||||
});
|
||||
}
|
||||
prune_size_reconciliation(info, now);
|
||||
}
|
||||
prune_size_reconciliation(info, now);
|
||||
}
|
||||
|
||||
fn record_scan_resume_hint(&mut self, folder: &str) {
|
||||
self.new_cache.info.scan_resume_after = Some(folder.to_string());
|
||||
self.update_cache.info.scan_resume_after = Some(folder.to_string());
|
||||
let checkpoint = DataUsageScanCheckpoint::new(folder.to_string(), DataUsageScanCheckpointReason::Unknown);
|
||||
global_metrics().record_scanner_checkpoint_set(
|
||||
checkpoint.version,
|
||||
checkpoint.resume_after.clone(),
|
||||
checkpoint.reason.as_str(),
|
||||
);
|
||||
self.new_cache.info.scan_checkpoint = Some(checkpoint);
|
||||
self.new_cache.info.scan_checkpoint = Some(checkpoint.clone());
|
||||
self.update_cache.info.scan_checkpoint = Some(checkpoint);
|
||||
}
|
||||
|
||||
fn record_completed_child(&mut self, folder: &str) {
|
||||
@@ -1529,7 +1511,8 @@ impl FolderScanner {
|
||||
self.record_scan_resume_hint_if_not_ancestor(&child_hash.key());
|
||||
parent_entry.add_child(child_hash);
|
||||
self.new_cache.replace_hashed(child_hash, parent, &child_entry);
|
||||
self.refresh_usage_subtree(child_hash, parent);
|
||||
self.update_cache.delete_recursive(child_hash);
|
||||
self.update_cache.copy_with_children(&self.new_cache, child_hash, parent);
|
||||
self.send_update().await;
|
||||
}
|
||||
}
|
||||
@@ -1603,7 +1586,7 @@ impl FolderScanner {
|
||||
let candidate_type = pending_scanner_heal_candidate_type(kind);
|
||||
let priority = request.priority;
|
||||
let scan_mode = request.scan_mode.unwrap_or(self.scan_mode);
|
||||
let result = match send_scanner_heal_request(candidate_type, request).await {
|
||||
let result = match send_scanner_heal_request(candidate_type, request, scanner_disk_heal_scope(&self.local_disk)).await {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
self.update_pending_scanner_heal_after_admission(
|
||||
@@ -1672,75 +1655,29 @@ impl FolderScanner {
|
||||
self.data_usage_scanner_debug = debug;
|
||||
}
|
||||
|
||||
fn compact_completed_folder(&mut self, folder: &CachedFolder, hash: &DataUsageHash, entry: &DataUsageEntry) {
|
||||
// Completed healthy subtrees can keep aggregate usage. Coverage gaps
|
||||
// and failed-object debt still need child entries for precise resume
|
||||
// and to avoid rereading healthy siblings on a retry.
|
||||
let forward_sweep = self.old_cache.info.scan_progress.is_some();
|
||||
if entry.compacted || self.new_cache.info.name == folder.name || (forward_sweep && self.coverage_gap) {
|
||||
return;
|
||||
}
|
||||
let summary = if entry.children.is_empty() {
|
||||
Cow::Borrowed(entry)
|
||||
} else if let Some(flat) = self.new_cache.size_recursive(&hash.0) {
|
||||
Cow::Owned(flat)
|
||||
} else {
|
||||
return;
|
||||
};
|
||||
if forward_sweep && summary.failed_objects != 0 {
|
||||
return;
|
||||
}
|
||||
let should_compact = summary.objects < DATA_SCANNER_COMPACT_LEAST_OBJECT
|
||||
|| entry.children.iter().all(|key| {
|
||||
self.new_cache
|
||||
.cache
|
||||
.get(key)
|
||||
.is_none_or(|child| child.children.is_empty() && child.objects <= 1)
|
||||
});
|
||||
if !should_compact {
|
||||
return;
|
||||
}
|
||||
// Heal revisits also reach this path without a parent advancing
|
||||
// coverage. Keep the endpoint addressable when its completed
|
||||
// subtree is replaced by an aggregate.
|
||||
if self.coverage_frontier.as_deref().is_some_and(|frontier| {
|
||||
frontier
|
||||
.strip_prefix(&folder.name)
|
||||
.is_some_and(|suffix| suffix.starts_with('/'))
|
||||
}) {
|
||||
self.record_completed_child(&folder.name);
|
||||
}
|
||||
match summary {
|
||||
Cow::Borrowed(_) => {
|
||||
// The leaf is already cached; only its sampling mode changes.
|
||||
if let Some(cached) = self.new_cache.cache.get_mut(&hash.0) {
|
||||
cached.compacted = true;
|
||||
}
|
||||
}
|
||||
Cow::Owned(mut flat) => {
|
||||
flat.compacted = true;
|
||||
self.new_cache.delete_recursive(hash);
|
||||
self.new_cache.replace_hashed(hash, &folder.parent, &flat);
|
||||
}
|
||||
/// Send update if enough time has passed
|
||||
/// Should be called on a regular basis when the new_cache contains more recent total than previously.
|
||||
/// May or may not send an update upstream.
|
||||
fn should_send_update(&self) -> bool {
|
||||
if self.updates.is_none() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let elapsed = self.last_update.elapsed().unwrap_or(Duration::from_secs(0));
|
||||
elapsed >= Duration::from_secs(60)
|
||||
}
|
||||
|
||||
fn refresh_usage_subtree(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
|
||||
if let Some(updates) = &mut self.usage_updates {
|
||||
updates.cache.delete_recursive(hash);
|
||||
updates.cache.copy_with_children(&self.new_cache, hash, parent);
|
||||
}
|
||||
}
|
||||
|
||||
/// Send at most one aggregate per minute when a caller requested progress.
|
||||
pub async fn send_update(&mut self) {
|
||||
let Some(updates) = &mut self.usage_updates else { return };
|
||||
if !updates.is_due() {
|
||||
// Send at most an update every minute.
|
||||
if !self.should_send_update() {
|
||||
return;
|
||||
}
|
||||
|
||||
if let Some(flat) = updates.cache.size_recursive(&self.new_cache.info.name) {
|
||||
if let Err(e) = updates.sender.send(flat).await {
|
||||
if let Some(flat) = self.update_cache.size_recursive(&self.new_cache.info.name)
|
||||
&& let Some(ref updates) = self.updates
|
||||
{
|
||||
// Try to send without blocking
|
||||
if let Err(e) = updates.send(flat).await {
|
||||
error!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_FOLDER_STATE,
|
||||
@@ -1752,17 +1689,17 @@ impl FolderScanner {
|
||||
"Scanner folder update send failed"
|
||||
);
|
||||
}
|
||||
updates.last_update = SystemTime::now();
|
||||
self.last_update = SystemTime::now();
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_update_for_entry(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>, entry: &DataUsageEntry) {
|
||||
if let Some(updates) = &mut self.usage_updates
|
||||
&& updates.is_due()
|
||||
{
|
||||
updates.cache.replace_hashed(hash, parent, entry);
|
||||
self.send_update().await;
|
||||
if !self.should_send_update() {
|
||||
return;
|
||||
}
|
||||
|
||||
self.update_cache.replace_hashed(hash, parent, entry);
|
||||
self.send_update().await;
|
||||
}
|
||||
|
||||
async fn record_missing_erasure_metadata(
|
||||
@@ -1901,7 +1838,8 @@ impl FolderScanner {
|
||||
wait_for_scanner_io(&ctx, self.sleeper.sleep_folder()).await?;
|
||||
}
|
||||
|
||||
let mut queued_folders: Vec<QueuedFolder> = Vec::new();
|
||||
let mut existing_folders: Vec<CachedFolder> = Vec::new();
|
||||
let mut new_folders: Vec<CachedFolder> = Vec::new();
|
||||
let mut found_object_metadata = false;
|
||||
let mut object_count: u64 = 0;
|
||||
let yield_every_objects = scanner_yield_every_n_objects();
|
||||
@@ -2209,21 +2147,12 @@ impl FolderScanner {
|
||||
self.reuse_clean_root_child(&this, &h, into);
|
||||
} else if exists {
|
||||
abandoned_children.remove(&h.key());
|
||||
queued_folders.push(QueuedFolder {
|
||||
folder: this,
|
||||
source: FolderScanSource::Existing,
|
||||
});
|
||||
if let Some(updates) = &mut self.usage_updates {
|
||||
updates
|
||||
.cache
|
||||
.copy_with_children(&self.old_cache, &h, &Some(this_hash.clone()));
|
||||
}
|
||||
existing_folders.push(this);
|
||||
self.update_cache
|
||||
.copy_with_children(&self.old_cache, &h, &Some(this_hash.clone()));
|
||||
} else {
|
||||
abandoned_children.remove(&h.key());
|
||||
queued_folders.push(QueuedFolder {
|
||||
folder: this,
|
||||
source: FolderScanSource::New,
|
||||
});
|
||||
new_folders.push(this);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
@@ -2249,6 +2178,7 @@ impl FolderScanner {
|
||||
lifecycle: active_life_cycle.clone(),
|
||||
object_lock: active_object_lock.clone(),
|
||||
replication: active_replication.clone(),
|
||||
heal_scope: None,
|
||||
heal_enabled,
|
||||
heal_bitrot: self.scan_mode == HealScanMode::Deep,
|
||||
debug: self.data_usage_scanner_debug,
|
||||
@@ -2445,17 +2375,17 @@ impl FolderScanner {
|
||||
// frontier. Keep the emergency compaction limit for large trees.
|
||||
let should_compact = (self.new_cache.info.name != folder.name
|
||||
&& self.old_cache.info.scan_progress.is_none()
|
||||
&& queued_folders.len() >= DATA_SCANNER_COMPACT_AT_FOLDERS)
|
||||
|| queued_folders.len() >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS;
|
||||
&& existing_folders.len() + new_folders.len() >= DATA_SCANNER_COMPACT_AT_FOLDERS)
|
||||
|| existing_folders.len() + new_folders.len() >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS;
|
||||
|
||||
let total_folders = queued_folders.len();
|
||||
let total_folders = existing_folders.len() + new_folders.len();
|
||||
self.alert_excessive_folders(&folder.name, total_folders);
|
||||
|
||||
if !into.compacted && should_compact {
|
||||
into.compacted = true;
|
||||
for queued in &mut queued_folders {
|
||||
queued.source = FolderScanSource::New;
|
||||
}
|
||||
new_folders.append(&mut existing_folders);
|
||||
|
||||
existing_folders.clear();
|
||||
|
||||
if self.data_usage_scanner_debug {
|
||||
debug!(
|
||||
@@ -2464,13 +2394,20 @@ impl FolderScanner {
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_FOLDER,
|
||||
folder = %folder.name,
|
||||
entry_count = queued_folders.len(),
|
||||
entry_count = new_folders.len(),
|
||||
state = "preemptive_compaction",
|
||||
"Scanner folder switched to compacted mode"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if !into.compacted {
|
||||
for folder_item in &existing_folders {
|
||||
let h = hash_path(&folder_item.name);
|
||||
self.update_cache.copy_with_children(&self.old_cache, &h, &folder_item.parent);
|
||||
}
|
||||
}
|
||||
|
||||
let is_scan_root = folder.name == self.old_cache.info.name;
|
||||
let scan_checkpoint = self.old_cache.info.scan_checkpoint.as_ref();
|
||||
let checkpoint_resume_after = scan_checkpoint.and_then(|checkpoint| {
|
||||
@@ -2494,6 +2431,15 @@ impl FolderScanner {
|
||||
.and_then(|resume_after| folder_resume_match(&folder.name, resume_after))
|
||||
.is_some_and(|resume_match| matches!(resume_match, FolderResumeMatch::Descendant));
|
||||
let scan_resume_after = checkpoint_resume_after.or(self.old_cache.info.scan_resume_after.as_deref());
|
||||
let mut queued_folders = Vec::with_capacity(new_folders.len() + existing_folders.len());
|
||||
queued_folders.extend(new_folders.into_iter().map(|folder| QueuedFolder {
|
||||
folder,
|
||||
source: FolderScanSource::New,
|
||||
}));
|
||||
queued_folders.extend(existing_folders.into_iter().map(|folder| QueuedFolder {
|
||||
folder,
|
||||
source: FolderScanSource::Existing,
|
||||
}));
|
||||
let has_queued_folders = !queued_folders.is_empty();
|
||||
let forward_sweep = self.old_cache.info.scan_progress.is_some();
|
||||
let resume_order = if forward_sweep {
|
||||
@@ -2534,21 +2480,19 @@ impl FolderScanner {
|
||||
match queued_folder.source {
|
||||
FolderScanSource::New => {
|
||||
// Add new folders to the update tree so totals update for these.
|
||||
if !into.compacted
|
||||
&& let Some(updates) = &mut self.usage_updates
|
||||
{
|
||||
if !into.compacted {
|
||||
let mut found_any = false;
|
||||
let mut parent = this_hash.clone();
|
||||
let update_root_hash = hash_path(&updates.cache.info.name);
|
||||
let update_cache_name_hash = hash_path(&self.update_cache.info.name);
|
||||
|
||||
while parent != update_root_hash {
|
||||
while parent != update_cache_name_hash {
|
||||
let parent_key = parent.key();
|
||||
let e = updates.cache.find(&parent_key);
|
||||
let e = self.update_cache.find(&parent_key);
|
||||
if e.is_none_or(|v| v.compacted) {
|
||||
found_any = true;
|
||||
break;
|
||||
}
|
||||
if let Some(next) = updates.cache.search_parent(&parent) {
|
||||
if let Some(next) = self.update_cache.search_parent(&parent) {
|
||||
parent = next;
|
||||
} else {
|
||||
found_any = true;
|
||||
@@ -2557,8 +2501,7 @@ impl FolderScanner {
|
||||
}
|
||||
if !found_any {
|
||||
// Add non-compacted empty entry.
|
||||
updates
|
||||
.cache
|
||||
self.update_cache
|
||||
.replace_hashed(&h, &Some(this_hash.clone()), &DataUsageEntry::default());
|
||||
}
|
||||
}
|
||||
@@ -2632,17 +2575,13 @@ impl FolderScanner {
|
||||
};
|
||||
if outcome == FolderScanOutcome::Vanished {
|
||||
self.new_cache.delete_recursive(&h);
|
||||
if let Some(updates) = &mut self.usage_updates {
|
||||
updates.cache.delete_recursive(&h);
|
||||
}
|
||||
self.update_cache.delete_recursive(&h);
|
||||
into.children.remove(&h.key());
|
||||
continue;
|
||||
}
|
||||
if outcome == FolderScanOutcome::ErasureData {
|
||||
self.new_cache.delete_recursive(&h);
|
||||
if let Some(updates) = &mut self.usage_updates {
|
||||
updates.cache.delete_recursive(&h);
|
||||
}
|
||||
self.update_cache.delete_recursive(&h);
|
||||
into.children.remove(&h.key());
|
||||
// Earlier namespace siblings may have completed, but none
|
||||
// can certify coverage of this unresolved parent object.
|
||||
@@ -2662,13 +2601,23 @@ impl FolderScanner {
|
||||
self.send_update_for_entry(&this_hash, &folder.parent, into).await;
|
||||
tokio::task::yield_now().await;
|
||||
} else {
|
||||
tokio::task::yield_now().await;
|
||||
into.add_child(&h);
|
||||
self.record_completed_child(&folder_item.name);
|
||||
tokio::task::yield_now().await;
|
||||
self.maybe_send_checkpoint(&folder, &this_hash, into);
|
||||
self.refresh_usage_subtree(&h, &folder_item.parent);
|
||||
self.update_cache.delete_recursive(&h);
|
||||
self.update_cache.copy_with_children(&self.new_cache, &h, &folder_item.parent);
|
||||
self.send_update().await;
|
||||
}
|
||||
|
||||
if queued_folder.source == FolderScanSource::New
|
||||
&& !into.compacted
|
||||
&& self.update_cache.find(&this_hash.key()).is_some_and(|v| !v.compacted)
|
||||
{
|
||||
self.update_cache.delete_recursive(&h);
|
||||
self.update_cache
|
||||
.copy_with_children(&self.new_cache, &h, &Some(this_hash.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
// Scan for healing
|
||||
@@ -3065,7 +3014,8 @@ impl FolderScanner {
|
||||
|
||||
into.add_child(&h);
|
||||
// We scanned a folder, optionally send update.
|
||||
self.refresh_usage_subtree(&h, &folder_item.parent);
|
||||
self.update_cache.delete_recursive(&h);
|
||||
self.update_cache.copy_with_children(&self.new_cache, &h, &folder_item.parent);
|
||||
self.send_update().await;
|
||||
self.maybe_send_checkpoint(&folder, &this_hash, into);
|
||||
}
|
||||
@@ -3079,7 +3029,36 @@ impl FolderScanner {
|
||||
self.new_cache.replace_hashed(&this_hash, &folder.parent, into);
|
||||
}
|
||||
|
||||
self.compact_completed_folder(&folder, &this_hash, into);
|
||||
// Keep independently accounted children while the sweep cursor may
|
||||
// reference them; the hard cardinality compaction below still applies.
|
||||
if !into.compacted
|
||||
&& self.old_cache.info.scan_progress.is_none()
|
||||
&& self.new_cache.info.name != folder.name
|
||||
&& let Some(mut flat) = self.new_cache.size_recursive(&this_hash.key())
|
||||
{
|
||||
flat.compacted = true;
|
||||
let mut should_compact = false;
|
||||
|
||||
if flat.objects < DATA_SCANNER_COMPACT_LEAST_OBJECT {
|
||||
should_compact = true;
|
||||
} else {
|
||||
// Compact if we only have objects as children...
|
||||
should_compact = true;
|
||||
for k in &into.children {
|
||||
if let Some(v) = self.new_cache.cache.get(k)
|
||||
&& (!v.children.is_empty() || v.objects > 1)
|
||||
{
|
||||
should_compact = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if should_compact {
|
||||
self.new_cache.delete_recursive(&this_hash);
|
||||
self.new_cache.replace_hashed(&this_hash, &folder.parent, &flat);
|
||||
}
|
||||
}
|
||||
|
||||
// Compact if too many children...
|
||||
if !into.compacted {
|
||||
@@ -3092,13 +3071,12 @@ impl FolderScanner {
|
||||
done_compact();
|
||||
}
|
||||
|
||||
if !was_compacted
|
||||
&& let Some(updates) = &mut self.usage_updates
|
||||
&& updates.cache.cache.contains_key(&this_hash.key())
|
||||
&& let Some(flat) = self.new_cache.size_recursive(&this_hash.key())
|
||||
{
|
||||
updates.cache.delete_recursive(&this_hash);
|
||||
updates.cache.replace_hashed(&this_hash, &folder.parent, &flat);
|
||||
if self.update_cache.cache.contains_key(&this_hash.key()) && !was_compacted {
|
||||
// Replace if existed before.
|
||||
if let Some(flat) = self.new_cache.size_recursive(&this_hash.key()) {
|
||||
self.update_cache.delete_recursive(&this_hash);
|
||||
self.update_cache.replace_hashed(&this_hash, &folder.parent, &flat);
|
||||
}
|
||||
}
|
||||
|
||||
self.finish_size_reconciliation_batch();
|
||||
@@ -3196,7 +3174,10 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
info: cache.info.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let usage_updates = updates.map(|sender| ScannerUsageUpdates::new(sender, bucket_name.clone()));
|
||||
let update_cache = DataUsageCache {
|
||||
info: cache.info.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
let resume_failed_paths = if resume_frontier.is_some() {
|
||||
failed_scan_paths(&cache)
|
||||
} else {
|
||||
@@ -3207,6 +3188,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
root: base_path,
|
||||
old_cache: cache,
|
||||
new_cache,
|
||||
update_cache,
|
||||
data_usage_scanner_debug: false,
|
||||
heal_object_select,
|
||||
scan_mode,
|
||||
@@ -3219,10 +3201,11 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
sleeper,
|
||||
disks,
|
||||
disks_quorum,
|
||||
usage_updates,
|
||||
updates,
|
||||
checkpoint_tx,
|
||||
checkpoint_ancestors: Vec::new(),
|
||||
checkpoint_depth: 0,
|
||||
last_update: SystemTime::UNIX_EPOCH,
|
||||
checkpoint_objects: 0,
|
||||
object_metadata_visits,
|
||||
last_checkpoint_objects: 0,
|
||||
@@ -3238,7 +3221,11 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
resume_frontier,
|
||||
resume_failed_paths,
|
||||
coverage_gap: false,
|
||||
pending_heal_sync_deferred: false,
|
||||
pending_heal_batch_dirty: false,
|
||||
raw_enumeration_progress: Vec::new(),
|
||||
#[cfg(test)]
|
||||
pending_heal_sync_count: 0,
|
||||
pending_size_reconciliation_keys: HashSet::new(),
|
||||
pending_size_reconciliation_scopes: HashSet::new(),
|
||||
pending_size_reconciliation_truncated: false,
|
||||
@@ -3248,6 +3235,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
|
||||
let now = FolderScanner::now_secs();
|
||||
prune_size_reconciliation(&mut scanner.new_cache.info, now);
|
||||
prune_size_reconciliation(&mut scanner.update_cache.info, now);
|
||||
|
||||
// Check if context is cancelled
|
||||
if ctx.is_cancelled() {
|
||||
@@ -3418,6 +3406,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
mod item_actions;
|
||||
mod ledger;
|
||||
|
||||
pub(crate) use item_actions::scanner_disk_heal_scope;
|
||||
use item_actions::*;
|
||||
pub use item_actions::{GetSizeFn, ScannerItem};
|
||||
use ledger::*;
|
||||
|
||||
@@ -474,10 +474,23 @@ pub(super) fn record_heal_candidate_admission(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_disk_heal_scope(disk: &Disk) -> Option<DataUsageCacheSource> {
|
||||
let location = disk.get_disk_location();
|
||||
location.valid().then_some(())?;
|
||||
Some(DataUsageCacheSource::new(location.pool_idx?, location.set_idx?))
|
||||
}
|
||||
|
||||
pub(super) async fn send_scanner_heal_request(
|
||||
candidate_type: &'static str,
|
||||
request: HealChannelRequest,
|
||||
mut request: HealChannelRequest,
|
||||
scope: Option<DataUsageCacheSource>,
|
||||
) -> Result<HealAdmissionResult, ScannerError> {
|
||||
if request.object_prefix.as_deref().is_some_and(|object| !object.is_empty()) {
|
||||
let scope = scope
|
||||
.ok_or_else(|| ScannerError::Other("scanner object heal requires the scanning disk's pool and set".to_string()))?;
|
||||
request.pool_index = Some(scope.pool_index);
|
||||
request.set_index = Some(scope.set_index);
|
||||
}
|
||||
let priority = request.priority;
|
||||
let trace_context = scanner_heal_candidate_trace_context(&request);
|
||||
let candidate = crate::heal_completion::prepare(&request);
|
||||
@@ -537,6 +550,8 @@ pub struct ScannerItem {
|
||||
pub lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
|
||||
pub object_lock: Option<Arc<ObjectLockConfiguration>>,
|
||||
pub replication: Option<Arc<ReplicationConfig>>,
|
||||
/// Actual scanning-disk location; refreshed by the disk reader before actions.
|
||||
pub heal_scope: Option<DataUsageCacheSource>,
|
||||
pub heal_enabled: bool,
|
||||
pub heal_bitrot: bool,
|
||||
pub debug: bool,
|
||||
@@ -1161,6 +1176,7 @@ impl ScannerItem {
|
||||
scan_mode,
|
||||
HealChannelPriority::Low,
|
||||
),
|
||||
self.heal_scope,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -1322,6 +1338,7 @@ mod tests {
|
||||
lifecycle: None,
|
||||
object_lock: None,
|
||||
replication: None,
|
||||
heal_scope: None,
|
||||
heal_enabled: false,
|
||||
heal_bitrot: false,
|
||||
debug: false,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user