Compare commits

..

73 Commits

Author SHA1 Message Date
cxymds 3e337bad97 ci(validation): bind recovery to retained runner evidence 2026-10-26 15:31:40 +08:00
cxymds 12a23859a1 ci(validation): retain interrupted evidence without fleet recovery 2026-10-26 15:08:15 +08:00
cxymds 081d3b1469 ci(validation): isolate each client virtual environment 2026-10-26 14:39:21 +08:00
cxymds 56a80f7b62 ci(validation): prepare Python before creating client environments 2026-10-26 14:37:43 +08:00
overtrue 0a366d967b chore: merge main into validation runner branch 2026-10-26 12:43:57 +08:00
cxymds 31c9d85645 ci(validation): inspect shared runner without fleet access 2026-10-26 09:59:52 +08:00
overtrue b1f7fb2769 test(scanner): adapt scope regression to admission receipts 2026-10-24 09:35:28 +08:00
overtrue c20448dfce chore: merge main into validation runner 2026-10-24 08:00:59 +08:00
overtrue 887716ca61 chore: merge main into validation runner 2026-10-23 19:34:20 +08:00
cxymds dae95d7afe fix(heal): retain resolved object identity on terminal failures 2026-10-23 11:58:02 +08:00
cxymds b0f60ed8d9 fix(ci): retain validation evidence outside runner temp 2026-10-23 09:43:20 +08:00
cxymds 78c803e1ef fix(heal): retain cancellation after successful bucket storage I/O 2026-10-23 06:26:00 +08:00
cxymds a2d6aa5a7f test(admin): reuse error facade in heal query regression 2026-10-23 05:12:08 +08:00
cxymds 553be4b4c6 fix(heal): query automatic tasks on their owning node 2026-10-23 04:45:20 +08:00
overtrue 96858cd1f1 style: format RPC digest precedence assertion 2026-10-23 04:37:42 +08:00
overtrue 3d357cec44 chore: merge main into validation runner 2026-10-23 04:30:19 +08:00
cxymds 9bce929ae5 test(scanner): reuse storage boundary for scope regression 2026-10-23 01:58:03 +08:00
cxymds a74ddfab3c fix(scanner): bind object heals to the scanning disk scope 2026-10-23 01:22:15 +08:00
cxymds 1676878676 test(validation): wire I05 queued-successor native prerequisites 2026-10-22 21:05:52 +08:00
overtrue b3031bed59 chore: merge main into validation runner 2026-10-21 21:16:33 +08:00
cxymds c86faf193a fix(ci): require complete native admission for multipart races 2026-10-21 10:09:01 +08:00
cxymds e30012c4ef test(scanner): assert typed corruption after metadata truncation 2026-10-20 12:47:13 +08:00
cxymds 2babf7624d ci(validation): admit source-bound owner and copy race proofs 2026-10-20 12:16:00 +08:00
cxymds 757b751482 test(ecstore): cover default heal owners and copy metadata races 2026-10-20 11:32:20 +08:00
cxymds 09fb813a00 test(ecstore): fence multipart continue complete and abort during heal 2026-10-20 10:06:54 +08:00
cxymds 738e5fa14a ci(validation): require G04 native and runtime admission 2026-10-20 09:31:57 +08:00
cxymds 8380466f3a test(ecstore): fence marker and batch deletes during heal 2026-10-20 09:26:19 +08:00
cxymds 170ec8d983 ci(validation): require complete race native runtime preflight 2026-10-20 08:59:50 +08:00
cxymds bd0671ba78 test(ecstore): fence version races and bind cleanup slots 2026-10-20 06:27:54 +08:00
cxymds f954164048 ci(validation): admit source-bound G01 native evidence before fleet execution 2026-10-20 04:35:09 +08:00
cxymds 6afdec9fe1 test(ecstore): exercise heal during committed overwrite cleanup 2026-10-20 04:07:05 +08:00
cxymds 87bd36b313 merge: sync validation fixes with current main 2026-10-20 03:35:22 +08:00
cxymds a4c02d5ac3 fix(filemeta): classify supported inline framing damage as corrupt 2026-10-20 01:59:26 +08:00
cxymds 62ff96774f fix(s3): map typed RPC unavailability to service refusal 2026-10-20 00:34:17 +08:00
cxymds 85c706cac0 fix(observability): retain typed object error diagnostics 2026-10-19 22:13:24 +08:00
cxymds 7951bb6298 fix(heal): retain exhausted admin batch repair receipts 2026-10-16 15:33:15 +08:00
cxymds a3230599a8 fix(ci): allow recovery of completed cancelled validation runs 2026-10-15 19:19:31 +08:00
cxymds 1c7fc818dd fix(ecstore): retain below-quorum data during no-remove heal 2026-10-13 02:02:13 +08:00
cxymds 5475bd06bc chore(ecstore): shrink typed-error ratchet baseline 2026-10-13 01:07:12 +08:00
cxymds a3ba311ab0 fix(ecstore): retain typed quorum errors for missing parts 2026-10-13 00:59:50 +08:00
cxymds e517d35fa2 test(ci): archive scoped F07 directory fsync evidence 2026-10-12 23:47:15 +08:00
cxymds 1b7a197267 fix(ci): isolate validation package publication credentials 2026-10-12 23:30:45 +08:00
Chris 3b416b76cd Merge branch 'main' into cxymds/validation-runner 2026-10-12 06:18:15 -07:00
cxymds bebee3d5d0 ci(validation): verify C11 native storage evidence 2026-10-12 19:07:12 +08:00
cxymds 1aa9be87d1 test(ecstore): verify incarnation fence at repair commit 2026-10-12 18:38:26 +08:00
cxymds e224eb48e5 fix(heal): deeply verify read repair payload failures 2026-10-12 17:40:36 +08:00
cxymds 16cf19817e ci(validation): dispatch isolated packages through the driver 2026-10-12 12:29:11 +08:00
cxymds 32fd1e704a test(validation): verify isolated package publication failures 2026-10-12 12:22:53 +08:00
cxymds ac197ea655 test(validation): enforce private candidate manifest schema 2026-10-12 12:18:15 +08:00
cxymds e18ed73942 ci(validation): publish isolated immutable candidate packages 2026-10-12 12:06:59 +08:00
cxymds 3c131f2b2e fix(heal): preserve typed errors and SSE-C response headers 2026-10-12 11:36:29 +08:00
cxymds c5f4b5a4f7 fix(heal): preserve typed bucket errors for bounded retries 2026-10-12 10:45:27 +08:00
cxymds 03fb3d78a0 ci(validation): consume pinned manager native prerequisites 2026-10-11 11:55:36 +08:00
cxymds b8126e50f7 test(ci): expose manager regressions through tools-only dispatch 2026-10-11 11:21:35 +08:00
cxymds 59fb72aa82 test(ci): add isolated manager native validation builds 2026-10-11 11:20:46 +08:00
cxymds 9de01c9170 docs(ci): describe supported validation recovery 2026-10-11 11:07:20 +08:00
cxymds 2ec26fa4b9 feat(ci): expose read-only validation fleet inventory 2026-10-11 10:42:48 +08:00
cxymds f4507178a3 test(ci): bind C10 execution to the verified shared codec 2026-10-11 10:09:26 +08:00
cxymds 9b4cc40abd test(validation): build optional C10 fixture codec on hosted Linux 2026-10-11 09:44:09 +08:00
cxymds b14ff4eeb1 test(validation): bind C07 to pinned native authority evidence 2026-10-11 09:27:59 +08:00
cxymds ae9b5cbabf Run validation tool resolution on hosted Linux without acquiring the fleet lock 2026-10-11 07:49:26 +08:00
cxymds 6c78b0d996 test(validation): route native tools through the existing driver 2026-10-11 06:06:17 +08:00
cxymds 0066e34764 test(validation): add optional native Heal regression tooling 2026-10-11 06:03:43 +08:00
cxymds fbddcc2965 ci: audit retained validation environments before finalization 2026-10-10 23:28:39 +08:00
cxymds b1793b55a0 test(validation): verify Heal tools against released shard captures 2026-10-10 21:11:43 +08:00
cxymds 835bd3f52d test(validation): verify pinned Heal tools before fleet execution 2026-10-10 20:46:40 +08:00
cxymds cb32d55878 test(validation): expose tool artifacts through the existing manual lane 2026-10-10 20:35:14 +08:00
cxymds 864e58592c test(validation): build the candidate-bound Heal inspector artifact 2026-10-10 20:32:34 +08:00
cxymds 3016209330 test(validation): provide the Python client for Heal cases 2026-10-10 20:30:10 +08:00
cxymds 64e0eeba91 Install validation client for replication cases 2026-10-10 18:52:42 +08:00
cxymds 35fa7488a5 Prepare Python client for pool validation cases 2026-10-10 18:41:48 +08:00
cxymds 1b77846f8e Install pinned ILM client for canonical validation lane 2026-10-10 18:26:49 +08:00
cxymds b0c861841e test(validation): add pinned auto-testing manual lane 2026-10-10 17:34:37 +08:00
168 changed files with 11973 additions and 10196 deletions
@@ -6,17 +6,7 @@
- Route every new sync/flush through the durability-mode and bucket override
gates; mode `none` must not pay the new fsync.
- Keep blocking filesystem/CPU work off async runtime threads, but do not split
one small operation into many `spawn_blocking` round trips; coalesce `open`,
`read`/`write`, `fsync`, and `close` into one blocking closure per disk.
- Check for per-disk syscall amplification (`disks × parts/files`): pin a
directory once (`openat2` on Linux) and reuse the handle across sibling reads
instead of re-walking ancestor components per file or re-opening identical
`source_parent == destination_parent` directories.
- After `fstat`/`file.metadata()`, require `Vec::with_capacity(len + 1)` before
`read_to_end` so small metadata reads do not trigger 6–7 exponential-growth
`read()` syscalls.
- Avoid redundant `store.load()` or `root_missing()` round-trips under the same
exclusive lock when the state or original bytes were just loaded or persisted.
one small operation into many `spawn_blocking` round trips.
- Measure lock hold time across I/O and compare acquisition order for ABBA.
- Keep cleanup, extra stat/rename, and diagnostics out of the PUT commit critical
section when they need not be there.
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-linux=6190506b9e4bdb60fd392544a7bc47be708077bb4fe7a2ef689d047455e1af86
sha256-darwin=6190506b9e4bdb60fd392544a7bc47be708077bb4fe7a2ef689d047455e1af86
sha256-linux=9bd4b5ed4d135c6f61e0d144ddeb6b897fd5a6037b51f31088693cf1b23e0718
sha256-darwin=9bd4b5ed4d135c6f61e0d144ddeb6b897fd5a6037b51f31088693cf1b23e0718
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=a67a347f26be186c598b5beaf68e6bcbe1855350a35e1423dd5b8a81bcba06a5
sha256-linux=ad4fabe811c6cca1aba89fad34ccd404b5832a6a955875c2826f6e0fa1ea4f1f
sha256-darwin=da00472db3d8181ed413b8e9919f43e86fe644ccee444d92035ce189a33ab037
sha256-linux=42e95c9480e5bfa0d717be6652c6500b43e3a337cb63907869104bb155b5bd91
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81
sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db
sha256-darwin=347a0d4e5e547d5388703062ce66a66167a845b6e57ec591ccb4c732a567c68d
sha256-linux=4f9efa47194279185bb68249e82c788fa147abe944b6c764050e3d47f31e7926
+1 -1
View File
@@ -29,7 +29,7 @@
{
"invariant": "list-uncommitted-version",
"suite": "rustfs-filemeta",
"name": "metacache::tests::resolve_merge_keeps_the_newest_version_at_listing_quorum"
"name": "metacache::tests::resolve_with_write_quorum_slack_keeps_partial_latest_hidden_during_merge"
},
{
"invariant": "minio-object-fixture",
+2 -1
View File
@@ -52,7 +52,6 @@ script-tests: ## Run shell script tests
./scripts/check_embedded_secrets.sh --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_glibc.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_upgrade_workflow.py
@@ -64,6 +63,8 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
-22
View File
@@ -68,17 +68,6 @@ command = ['sh', '-c', 'echo RUST_MIN_STACK=33554432 >> "$NEXTEST_ENV"']
# boundaries. Reserve the run's capacity so unrelated fixtures cannot starve
# their existing deadlines and typed-failure assertions (rustfs/backlog#2706).
# Keep each test's internal concurrency and zero-retry policy unchanged.
[[profile.default.overrides]]
filter = """
package(rustfs-ecstore) & (
test(/^set_disk::ops::multipart::tests::complete_multipart_(quorum_success_releases_logical_guards_before_physical_tail|waits_for_tail_before_releasing_guards_and_marking_capacity)$/)
| test(=store::heal::tests::full_format_heal_lost_after_last_save_does_not_publish_or_renew_and_latches)
| test(/^store::init::tests::(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page|dispatch_manifest_shutdown_(drains_admitted_delete_batch_without_tail_admission|stops_aborting_validation_tail_reads|stops_authorized_tail_reads_before_receipt_or_cas))$/)
| (binary(metadata_read_lock_reuse_test) & test(=replica_multipart_completes_while_metadata_writer_waits_for_its_snapshot))
)
"""
threads-required = "num-test-threads"
[[profile.default.overrides]]
filter = 'package(rustfs) & test(/^connect::diagnostics::schedule::tests::/)'
threads-required = "num-test-threads"
@@ -362,17 +351,6 @@ path = "junit.xml"
# Match the local durable-progress reservations. All members, deadlines,
# assertions, and internal concurrency remain enabled without retries.
[[profile.ci.overrides]]
filter = """
package(rustfs-ecstore) & (
test(/^set_disk::ops::multipart::tests::complete_multipart_(quorum_success_releases_logical_guards_before_physical_tail|waits_for_tail_before_releasing_guards_and_marking_capacity)$/)
| test(=store::heal::tests::full_format_heal_lost_after_last_save_does_not_publish_or_renew_and_latches)
| test(/^store::init::tests::(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page|dispatch_manifest_shutdown_(drains_admitted_delete_batch_without_tail_admission|stops_aborting_validation_tail_reads|stops_authorized_tail_reads_before_receipt_or_cas))$/)
| (binary(metadata_read_lock_reuse_test) & test(=replica_multipart_completes_while_metadata_writer_waits_for_its_snapshot))
)
"""
threads-required = "num-test-threads"
[[profile.ci.overrides]]
filter = 'package(rustfs) & test(/^connect::diagnostics::schedule::tests::/)'
threads-required = "num-test-threads"
@@ -74,6 +74,7 @@ Notes:
- `.github/workflows/e2e-s3tests.yml` (weekly full s3-tests sweep)
- `.github/workflows/mint.yml` (weekly multi-SDK mint run)
- `.github/workflows/fuzz.yml` (nightly fuzz corpus jobs)
- `.github/workflows/performance-ab.yml` (nightly warp A/B gate)
- **Pending:** the ci-7 e2e nightly-full workflow does not exist yet
(backlog#1149 ci-7). When it lands, wire it up with the snippet above.
+1
View File
@@ -18,6 +18,7 @@
"never_ran_grace_until": "2026-09-08T00:00:00Z"
},
{ "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
{ "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
{
"workflow": ".github/workflows/runner-hygiene.yml",
"max_age_hours": 792,
+4 -4
View File
@@ -7,7 +7,7 @@
This repository is public. For a public repository, GitHub's standard hosted runners (`ubuntu-latest`, `windows-latest`, `macos-latest`) cost nothing and have no minute cap, whatever plan the organisation is on. GitHub bills only two things: larger hosted runners, which have custom labels and are not used here, and standard runners in private repositories.
Every other label in this repository is capacity RustFS pays for. ARC pods and lab machines accrue cost while the machine exists. A Ubicloud label is metered for the minutes a job holds its VM:
Every other label in this repository is capacity RustFS pays for by the hour whether or not a job is running well:
| Label | Paid by | What it is | Use it for |
|---|---|---|---|
@@ -15,7 +15,7 @@ Every other label in this repository is capacity RustFS pays for. ARC pods and l
| `windows-latest`, `macos-latest` | nobody | GitHub-hosted standard runners | Windows and macOS targets |
| `sm-standard-2` | RustFS | ARC pod, Ubuntu 24.04, minimal image: no `gh`, no C toolchain; Docker only on the `dind-` label | only a job that a hosted runner cannot run (see the placement rule) |
| `sm-standard-4` | RustFS | ARC pod, Ubuntu 24.04, 14 CPUs and 28 GB (the label is not the core count), large disk | workspace-scale Cargo builds |
| `ubicloud-standard-16-arm` | RustFS | Ubicloud VM, alias of `ubicloud-standard-16-arm-ubuntu-2404`: Ubuntu 24.04 arm64, 16 vCPU, 48 GB, 300 GB disk. Reached through the `build.yml` matrix, so `actionlint.yaml` does not list it | native aarch64 release builds (`linux-aarch64-musl`, `linux-aarch64-gnu`) |
| `sm-standard-4-arm` | RustFS | ARC pod, aarch64; reached through the `build.yml` matrix, so `actionlint.yaml` does not list it | native aarch64 release builds |
| `dind-sm-standard-2` | RustFS | ARC pod with Docker-in-Docker | image builds that need `buildx` and a registry push |
| `smoke-testing`, `pf-testing` | RustFS | long-lived lab machines | the packaged functional chain and performance suites |
@@ -40,7 +40,7 @@ A hosted runner is also the only choice for a few things the pods cannot do: `mo
- Every `shared-key` has exactly one writer. `cache-warm.yml` owns `ci-dev`, `ci-feat-rio`, `ci-feat-proto` and `ci-uring`; every other `./.github/actions/setup` call states `cache-save-if` explicitly, and `scripts/security/check_cache_save_if.sh` fails the build when one does not.
- All readers of a key run on the same image as its writer. Keys that live on hosted runners say so in their name (`ci-uring` is warmed on `ubuntu-latest`; `ci-e2e-distributed-hosted`, `e2e-upgrade-server-hosted`, `e2e-upgrade-tests-hosted`).
- Moving a job between pools therefore means renaming its key. The new key is empty until a run from `main` (a scheduled run, or a manual dispatch of that workflow from `main`) saves it; until then that lane does a cold build. The `build.yml` aarch64 release legs append `-ubicloud-arm` for this reason. `cache-warm.yml` does not write those keys.
- Moving a job between pools therefore means renaming its key. The new key is empty until a run from `main` (a scheduled run, or a manual dispatch of that workflow from `main`) saves it; until then that lane does a cold build.
- `CARGO_BUILD_JOBS` and every other `CARGO*`, `CC*`, `CFLAGS*`, `CXX*`, `CMAKE*` and `RUST*` variable is hashed into the key. `ci.yml` sets `CARGO_BUILD_JOBS` at step level for that reason; promoting it to job level rotates the key for every lane that shares it.
## Where self-hosted minutes leak
@@ -62,7 +62,7 @@ gh api "repos/rustfs/rustfs/actions/runs/<run_id>/jobs?per_page=100" \
--jq '.jobs[] | "\(.conclusion)\t\(.labels | join(","))\t\(.started_at)\t\(.completed_at)\t\(.name)"'
```
Only rows whose label starts with `sm-`, `dind-`, `ubicloud-`, `smoke-` or `pf-` cost anything. At the October 2026 review, a full pull-request run of `ci.yml` spent about 150 self-hosted minutes, a third of it in jobs that never compiled; treat that as the scale of what a placement mistake costs, not as a current figure.
Only rows whose label starts with `sm-`, `dind-`, `smoke-` or `pf-` cost anything. At the October 2026 review, a full pull-request run of `ci.yml` spent about 150 self-hosted minutes, a third of it in jobs that never compiled; treat that as the scale of what a placement mistake costs, not as a current figure.
## Checklist for a new or moved job
+5
View File
@@ -29,6 +29,7 @@ on:
- 'scripts/test_package_service_scripts.sh'
- 'scripts/test_package_versions.sh'
- 'scripts/test_docker_workflow.py'
- 'scripts/security/check_performance_ab_workflow.sh'
- 'scripts/security/check_preview_release_workflow.sh'
- 'scripts/security/check_tier_artifact_workflow.sh'
- 'scripts/security/check_workflow_pins.sh'
@@ -47,6 +48,7 @@ on:
- 'scripts/test_package_service_scripts.sh'
- 'scripts/test_package_versions.sh'
- 'scripts/test_docker_workflow.py'
- 'scripts/security/check_performance_ab_workflow.sh'
- 'scripts/security/check_preview_release_workflow.sh'
- 'scripts/security/check_tier_artifact_workflow.sh'
- 'scripts/security/check_workflow_pins.sh'
@@ -153,6 +155,9 @@ jobs:
- name: Check preview release workflow policy
run: ./scripts/security/check_preview_release_workflow.sh
- name: Check performance A/B workflow trust boundary
run: ./scripts/security/check_performance_ab_workflow.sh
- name: Check tier evidence workflow isolation
run: ./scripts/security/check_tier_artifact_workflow.sh
@@ -1,70 +0,0 @@
name: BTF prototype reference
on:
pull_request:
paths:
- "scripts/inspect_btf_prototypes.py"
- "scripts/test_inspect_btf_prototypes.py"
- "scripts/fixtures/btf_prototypes.c"
- ".github/workflows/btf-prototype-reference.yml"
workflow_dispatch:
permissions:
contents: read
concurrency:
group: btf-prototype-reference-${{ github.ref }}
cancel-in-progress: true
jobs:
reference:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
- name: Check prototype and CLI contracts
run: python3 -B scripts/test_inspect_btf_prototypes.py
- name: Install only missing native reference tools
run: |
select_bpftool() {
local candidate
local candidates=()
if command -v bpftool >/dev/null; then
candidates+=("$(command -v bpftool)")
fi
for candidate in /usr/lib/linux-tools-*/bpftool /usr/lib/linux-tools/*/bpftool; do
if [[ -x "$candidate" ]]; then
candidates+=("$candidate")
fi
done
while IFS= read -r candidate; do
if [[ -x "$candidate" ]] && file -Lb "$candidate" | grep -q 'ELF' \
&& "$candidate" version >/dev/null 2>&1; then
printf '%s\n' "$candidate"
return 0
fi
done < <(printf '%s\n' "${candidates[@]}" | sort -Vr)
return 1
}
packages=()
command -v clang >/dev/null || packages+=(clang)
command -v llvm-objcopy >/dev/null || packages+=(llvm)
select_bpftool >/dev/null || packages+=(linux-tools-generic)
if (( ${#packages[@]} )); then
sudo apt-get update
sudo apt-get install --no-install-recommends -y "${packages[@]}"
fi
# File parsing needs a working binary, not a host-kernel wrapper.
selected_bpftool="$(select_bpftool)"
"$selected_bpftool" version
printf 'BPFTOOL=%s\n' "$selected_bpftool" >> "$GITHUB_ENV"
- name: Compare compiled declarations with bpftool
run: |
clang -target bpf -O2 -g -c scripts/fixtures/btf_prototypes.c -o "$RUNNER_TEMP/prototypes.o"
llvm-objcopy --dump-section ".BTF=$RUNNER_TEMP/prototypes.btf" "$RUNNER_TEMP/prototypes.o"
python3 -B scripts/test_inspect_btf_prototypes.py --native-reference "$RUNNER_TEMP/prototypes.btf"
+7 -17
View File
@@ -181,18 +181,12 @@ jobs:
selected="all"
fi
# linux-aarch64-* need a real aarch64 host (placement rule: hardware)
# so the packaged binary and console smoke execute the artifact.
# ubicloud-standard-16-arm is that host: Ubuntu 24.04 arm64, alias of
# ubicloud-standard-16-arm-ubuntu-2404 (16 vCPU, 48 GB). linux-x86_64-* stay on the
# ARC sm-standard-4 pool. GNU rows keep cross=true so cargo-zigbuild
# pins the glibc 2.31 floor on the same architecture; musl stays a
# native cargo build.
all='{"include":[
{"target_id":"linux-x86_64-musl","os":"sm-standard-4","target":"x86_64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
{"target_id":"linux-aarch64-musl","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
{"target_id":"linux-aarch64-musl","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
{"target_id":"linux-x86_64-gnu","os":"sm-standard-4","target":"x86_64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
{"target_id":"linux-aarch64-gnu","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
{"target_id":"linux-aarch64-gnu","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
{"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
{"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":"","features":"","glibc":""}
]}'
@@ -250,11 +244,7 @@ jobs:
with:
rust-version: stable
target: ${{ matrix.target }}
# rust-cache keys include OS and architecture but not the runner image.
# The Ubicloud aarch64 legs therefore use their own suffix so they
# cannot restore a cache written on a different arm64 image.
# x86_64, macOS, and Windows keys stay unsuffixed.
cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}${{ matrix.os == 'ubicloud-standard-16-arm' && '-ubicloud-arm' || '' }}
cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}
# main only. A cache saved on refs/tags/X is scoped to that tag: no
# other tag, no main run and no PR can restore it, so every release
# cycle wrote up to 12 entries of 1-2GB (preview tag plus final tag,
@@ -774,9 +764,9 @@ jobs:
exit 0
fi
# ARC pods (the x86_64 legs) do not ship the aws CLI. Ubicloud arm64
# images follow the GitHub runner image and often do. Install the
# official CLI bundle when it is missing so R2 uploads work on either.
# The self-hosted Linux runners do not ship the aws CLI (GitHub-hosted
# images did). Install the official CLI bundle on demand so R2 uploads
# survive a fresh runner without relying on a distro package.
if ! command -v aws >/dev/null 2>&1; then
echo "aws CLI not found on runner; installing..."
if command -v apt-get >/dev/null 2>&1; then
+1 -1
View File
@@ -45,7 +45,7 @@ on:
# 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
# build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
# (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
# and e2e-replication-nightly (04:00) lanes.
# e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
- cron: "43 7 * * 0"
concurrency:
@@ -38,8 +38,8 @@ name: e2e-nightly
on:
workflow_dispatch:
schedule:
# 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00)
# and stale (01:30).
# 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00),
# stale (01:30) and performance-ab (06:00).
- cron: "29 4 * * *"
# Only alert-on-failure needs more than read access; it declares its own
+384
View File
@@ -0,0 +1,384 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Warp A/B budget gate for the hotpath series (rustfs/backlog#935 HP-14, item 4).
#
# Two entry points, honestly scoped:
# * schedule (nightly, on main): post-merge detection — catches a regression
# within 24h of landing, not before merge.
# * workflow_dispatch: an explicitly selected trusted ref.
# The dispatch input can run the gate with --allow-regression so a deliberate
# correctness cost (e.g. the #4221 fsync durability fix) is recorded, not
# blocked (rustfs/backlog#935 correction 1).
name: Performance A/B
on:
schedule:
- cron: "31 6 * * *" # 06:31 UTC nightly, against main
workflow_dispatch:
inputs:
duration:
description: "warp duration per round"
required: false
default: "12s"
type: string
allow_regression:
description: "Pass the gate despite a FAIL (deliberate tradeoff)"
required: false
default: false
type: boolean
permissions:
actions: read
contents: read
env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1
jobs:
warp-ab:
if: github.repository == 'rustfs/rustfs'
name: Warp A/B budget gate
runs-on: sm-standard-2
# A normal nightly restores the last successful binary and builds only the
# candidate; daily access keeps that cache warm. A cache miss may build both
# and needs room for the A/B run plus artifact and cache publication.
timeout-minutes: 180
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
fetch-depth: 0 # baseline may be an earlier successful scheduled head
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install warp
run: |
set -euo pipefail
WARP_VERSION="1.0.0"
curl -fsSL "https://github.com/minio/warp/releases/download/v${WARP_VERSION}/warp_Linux_x86_64.tar.gz" \
| sudo tar -xz -C /usr/local/bin warp
warp --version
- name: Decide exemption
id: exempt
env:
INPUT_ALLOW_REGRESSION: ${{ github.event.inputs.allow_regression }}
run: |
allow="false"
if [[ "$INPUT_ALLOW_REGRESSION" == "true" ]]; then
allow="true"
fi
echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"
# A failed regression run must keep comparing against the last known-good
# scheduled head. Otherwise the next nightly would absorb the regression
# into its baseline and turn green without a fix.
- name: Find last successful scheduled baseline
id: scheduled_baseline
if: github.event_name == 'schedule'
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
result-encoding: string
script: |
const { data } = await github.rest.actions.listWorkflowRuns({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: "performance-ab.yml",
event: "schedule",
status: "success",
per_page: 1,
});
return data.workflow_runs[0]?.head_sha ?? "";
# Manual runs compare a selected ref with current main. Scheduled runs
# compare current main with the last successful scheduled head. With no
# history, the first run measures the candidate against itself and seeds
# that head only if the complete rig succeeds.
- name: Resolve baseline / candidate commits
id: commits
env:
SCHEDULED_BASELINE_SHA: ${{ steps.scheduled_baseline.outputs.result }}
run: |
set -euo pipefail
candidate_sha="$(git rev-parse HEAD)"
if [[ "${{ github.event_name }}" == "schedule" ]]; then
baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}"
else
baseline_sha="$(git rev-parse origin/main)"
fi
git cat-file -e "${baseline_sha}^{commit}"
if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
echo "::error::baseline $baseline_sha is not an ancestor of candidate $candidate_sha; update the selected ref before comparing" >&2
exit 1
fi
echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
echo "baseline commit: $baseline_sha"
echo "candidate commit: $candidate_sha"
# Exact-key restore of the candidate binary saved by its successful
# scheduled run. A miss leaves cache-hit unset and falls back to a source
# build of that known-good head.
- name: Restore cached baseline binary
id: baseline_cache
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
with:
path: baseline-bin/rustfs
key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}
# Self-heal: on a nightly/dispatch run where the candidate commit IS the
# baseline commit, a cache miss would make the rig build the same commit
# twice (~65min per side with the post-#4806 LTO profile — no job budget
# fits that). Build it once here, reuse it for both phases, and save it
# back to the cache so the next run hits.
- name: Build baseline on cache miss (same-commit self-heal)
id: selfheal
if: >-
steps.baseline_cache.outputs.cache-hit != 'true' &&
steps.commits.outputs.baseline_sha == steps.commits.outputs.candidate_sha
run: |
set -euo pipefail
cargo build --release --bin rustfs
mkdir -p baseline-bin
cp target/release/rustfs baseline-bin/rustfs
echo "built=true" >> "$GITHUB_OUTPUT"
- name: Build baseline on cache miss (different candidate)
id: baseline_build
if: >-
steps.baseline_cache.outputs.cache-hit != 'true' &&
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
run: |
set -euo pipefail
baseline_root="$RUNNER_TEMP/rustfs-baseline-${{ github.run_id }}"
baseline_target="$RUNNER_TEMP/rustfs-baseline-target-${{ github.run_id }}"
git worktree add --detach "$baseline_root" "${{ steps.commits.outputs.baseline_sha }}"
cargo build --release --manifest-path "$baseline_root/Cargo.toml" --bin rustfs --target-dir "$baseline_target"
mkdir -p baseline-bin
cp "$baseline_target/release/rustfs" baseline-bin/rustfs
git worktree remove --force "$baseline_root"
echo "built=true" >> "$GITHUB_OUTPUT"
- name: Build candidate binary
id: candidate_build
if: steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
run: |
set -euo pipefail
cargo build --release --bin rustfs
mkdir -p candidate-bin
cp target/release/rustfs candidate-bin/rustfs
echo "built=true" >> "$GITHUB_OUTPUT"
- name: Save self-healed baseline to cache
if: steps.selfheal.outputs.built == 'true' || steps.baseline_build.outputs.built == 'true'
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
with:
path: baseline-bin/rustfs
key: rustfs-baseline-${{ steps.commits.outputs.baseline_sha }}
- name: Run warp A/B and gate
id: ab
env:
INPUT_DURATION: ${{ github.event.inputs.duration }}
run: |
set -euo pipefail
# The formal runner executes A1 baseline -> B1 candidate -> B2 candidate
# -> A2 baseline for each workload and drive-sync cell. It requires three
# rounds per leg to emit tail latency and error-rate evidence.
# --health-timeout 180 outlasts the server's own 120s startup-readiness
# budget, which the rig's previous 60s health poll undershot (the first
# two nightly failures). perf-6 recalibrates these once the noise study
# lands.
duration="${INPUT_DURATION:-12s}"
baseline_sha="${{ steps.commits.outputs.baseline_sha }}"
candidate_sha="${{ steps.commits.outputs.candidate_sha }}"
baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}"
selfheal_built="${{ steps.selfheal.outputs.built }}"
baseline_built="${{ steps.baseline_build.outputs.built }}"
candidate_built="${{ steps.candidate_build.outputs.built }}"
args=(--duration "$duration" --rounds 3 --cooldown 5 --health-timeout 180 --baseline-revision "$baseline_sha" --candidate-revision "$candidate_sha")
if [[ "$baseline_hit" == "true" || "$selfheal_built" == "true" || "$baseline_built" == "true" ]]; then
chmod +x baseline-bin/rustfs
base_bin="$PWD/baseline-bin/rustfs"
args+=(--baseline-bin "$base_bin")
if [[ "$baseline_hit" == "true" ]]; then
base_src="actions-cache (rustfs-baseline-$baseline_sha)"
elif [[ "$selfheal_built" == "true" ]]; then
base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
else
base_src="isolated baseline source build (saved as rustfs-baseline-$baseline_sha)"
fi
if [[ "$candidate_sha" == "$baseline_sha" ]]; then
# No commits landed since the last successful baseline, so reuse
# the one binary for both phases and measure only rig drift.
args+=(--candidate-bin "$base_bin")
cand_src="same binary as baseline (same commit)"
elif [[ "$candidate_built" == "true" ]]; then
chmod +x candidate-bin/rustfs
args+=(--candidate-bin "$PWD/candidate-bin/rustfs")
cand_src="source build of the checked-out ref"
else
echo "::error::candidate binary was not built" >&2
exit 2
fi
else
echo "::error::baseline binary was not restored or built" >&2
exit 2
fi
echo "baseline binary: $base_src"
echo "candidate binary: $cand_src"
if [[ "${{ steps.exempt.outputs.allow_regression }}" == "true" ]]; then
args+=(--allow-regression --exemption-reason "workflow dispatch override")
fi
# Do not let a gate FAIL abort the job here; capture status and surface
# it after the step summary is written.
set +e
bash scripts/run_hotpath_warp_abba.sh "${args[@]}"
echo "status=$?" >> "$GITHUB_OUTPUT"
set -e
# Locate the newest run dir + gate.md for the summary/comment/artifact
# steps. On a startup failure there is no gate.md, but the run dir still
# holds server-logs/ for diagnosis.
# Run dirs are UTC-timestamp names (no special chars); ls is safe here.
# shellcheck disable=SC2012
run_dir="$(ls -td target/hotpath-abba/*/ 2>/dev/null | head -n1 || true)"
echo "run_dir=${run_dir%/}" >> "$GITHUB_OUTPUT"
# shellcheck disable=SC2012
gate_md="$(ls -t target/hotpath-abba/*/candidate_gate.md 2>/dev/null | head -n1 || true)"
echo "gate_md=$gate_md" >> "$GITHUB_OUTPUT"
- name: Upload A/B results
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: hotpath-warp-ab-${{ github.run_number }}
# Includes per-cell median_summary.csv / baseline_compare.csv, both gates,
# and server-logs/ (rustfs.log + startup env per phase) so a failed run
# is diagnosable. Short retention: this is churny nightly debug data.
path: target/hotpath-abba/
if-no-files-found: warn
retention-days: 14
- name: Write gate summary
if: always()
run: |
set -euo pipefail
status="${{ steps.ab.outputs.status }}"
gate_md="${{ steps.ab.outputs.gate_md }}"
run_dir="${{ steps.ab.outputs.run_dir }}"
{
echo "## Hotpath warp A/B — run ${{ github.run_number }}"
echo
if [[ "$status" == "0" ]]; then
echo "Rig/gate exit: \`0\` (pass or warn)."
else
echo "Rig/gate exit: \`${status:-unknown}\` — **FAILED**."
fi
echo
if [[ -n "$gate_md" && -f "$gate_md" ]]; then
cat "$gate_md"
else
echo "No \`gate.md\` produced — the rig failed **before** the gate"
echo "(most likely server startup / health). Failing phase(s) below;"
echo "full logs in the \`hotpath-warp-ab-${{ github.run_number }}\` artifact."
if [[ -n "$run_dir" && -d "$run_dir/server-logs" ]]; then
for f in "$run_dir"/server-logs/*.log; do
[[ -f "$f" ]] || continue
echo
echo "<details><summary>$(basename "$f")</summary>"
echo
echo '```'
tail -n 30 "$f"
echo '```'
echo "</details>"
done
fi
fi
} >> "$GITHUB_STEP_SUMMARY"
- name: Stage successful candidate baseline
if: >-
steps.ab.outputs.status == '0' &&
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
run: |
set -euo pipefail
cp candidate-bin/rustfs baseline-bin/rustfs
- name: Cache successful candidate baseline
if: >-
steps.ab.outputs.status == '0' &&
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
with:
path: baseline-bin/rustfs
key: rustfs-baseline-${{ steps.commits.outputs.candidate_sha }}
# Scheduled failure alerting is handled by the alert-on-failure job below
# (perf-2 consuming ci-8's schedule-failure-issue composite action).
- name: Enforce gate
if: always()
run: |
status="${{ steps.ab.outputs.status }}"
if [[ -z "$status" ]]; then
echo "::error::warp A/B setup failed before the rig ran. Check the first failed workflow step." >&2
exit 1
fi
if [[ "$status" != "0" ]]; then
echo "::error::warp A/B budget gate failed (exit $status). See the step summary / gate.md artifact." >&2
exit "$status"
fi
echo "warp A/B budget gate passed."
alert-on-failure:
name: Alert on scheduled failure
needs: [warp-ab]
# `always()` is required: without it this job is skipped when a needed
# job fails. Alerts only for scheduled (nightly) runs (backlog#1149
# ci-8); manual dispatch failures are already watched by a human.
# `cancelled` is included alongside `failure` on purpose: a job that hits
# timeout-minutes ends as `cancelled`, and the 2026-07-11..07-14 nightly
# timeouts went silent precisely because the guard was failure-only. The
# composite action already reports cancelled/timed-out jobs in the issue
# body.
if: >-
github.repository == 'rustfs/rustfs' && (
always() && github.event_name == 'schedule' &&
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
)
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
+563 -5
View File
@@ -20,13 +20,91 @@ on:
workflow_dispatch:
inputs:
build_run_id:
description: Successful nightly build run on main
description: Successful nightly or validation package run; empty for package-only build
type: string
required: true
required: false
default: ''
build_run_attempt:
description: Exact successful build attempt
type: string
required: true
required: false
default: ''
validation_package_sha:
description: Exact source SHA for an isolated package-only build
type: string
required: false
default: ''
validation_case:
description: Canonical case ID, environment.inspect for read-only resources, or empty for functional chain
type: string
required: false
default: ''
i05_diagnostic_branch:
description: I05 diagnostic-only branch; queued-successor requires the exact native evidence bundle
type: choice
options: [timeout, queued-successor]
required: false
default: timeout
auto_testing_sha:
description: Exact auto-testing commit for a validation case
type: string
required: false
default: ''
validation_retain_evidence_run_id:
description: Upload existing persistent evidence from one interrupted validation run without fleet access
type: string
required: false
default: ''
validation_retain_evidence_run_attempt:
description: Exact interrupted run attempt to retain
type: string
required: false
default: '1'
validation_runner_inspect_only:
description: Inspect GitHub runner registration and queued jobs on a hosted runner without fleet access
type: boolean
required: false
default: false
validation_tools_only:
description: Build candidate-bound validation tools without running a fleet case
type: boolean
required: false
default: false
validation_native_tools:
description: Include exact-source native regressions in a tools-only build
type: boolean
required: false
default: false
validation_shared_tools:
description: Include the controlled C10 shared-reference encoder in a tools-only build
type: boolean
required: false
default: false
validation_manager_tools:
description: Include exact checkpoint/MRF/incarnation regressions in a tools-only build
type: boolean
required: false
default: false
validation_cleanup_run_id:
description: Failed validation run whose supported retained environment needs evidence-based finalization
type: string
required: false
default: ''
validation_cleanup_retained_evidence:
description: Recover from verified persistent runner-local evidence when the failed run uploaded no artifact
type: boolean
required: false
default: false
validation_tools_run_id:
description: Successful tool build run; required by Heal physical metadata checks
type: string
required: false
default: ''
validation_tools_run_attempt:
description: Exact tool build attempt
type: string
required: false
default: '1'
workflow_run:
workflows: ["Nightly GNU Build"]
types: [completed]
@@ -36,12 +114,103 @@ permissions:
actions: read
concurrency:
group: rustfs-functional-chain-runs
group: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_runner_inspect_only && 'rustfs-functional-validation-runner-inspection' || github.event_name == 'workflow_dispatch' && inputs.validation_package_sha != '' && format('rustfs-functional-validation-package-{0}', inputs.validation_package_sha) || github.event_name == 'workflow_dispatch' && inputs.validation_tools_only && 'rustfs-functional-validation-tools' || 'rustfs-functional-chain-runs' }}
cancel-in-progress: false
jobs:
validation-runner-inspection:
if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_runner_inspect_only }}
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Inspect shared runner registration and queue without fleet access
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
RUNNER_INSPECTION_INPUTS: ${{ toJSON(inputs) }}
run: >-
python3 -B scripts/inspect_validation_runner.py
--output "${RUNNER_TEMP}/validation-runner-inspection/report.json"
- name: Upload read-only runner inspection including incomplete reports
if: ${{ always() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: validation-runner-inspection-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/validation-runner-inspection/report.json
if-no-files-found: error
validation-evidence-retention:
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id != '' }}
# The original interrupted evidence exists only on this long-lived runner.
runs-on: smoke-testing
timeout-minutes: 15
concurrency:
group: rustfs-shared-functional-tests-v2
cancel-in-progress: false
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Verify existing interrupted evidence without changing it
env:
GH_TOKEN: ${{ github.token }}
EVIDENCE_RETAIN_INPUTS: ${{ toJSON(inputs) }}
run: >-
python3 -B scripts/retain_validation_evidence.py
--report "${RUNNER_TEMP}/validation-evidence-retention/report.json"
- name: Upload admitted original evidence under collection provenance
if: ${{ env.VALIDATION_RETAIN_ROOT != '' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: rustfs-validation-retained-${{ env.VALIDATION_RETAIN_SOURCE_RUN }}-${{ env.VALIDATION_RETAIN_SOURCE_ATTEMPT }}-by-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ env.VALIDATION_RETAIN_ROOT }}
if-no-files-found: error
include-hidden-files: true
- name: Upload retention receipt including incomplete reports
if: ${{ always() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: validation-evidence-retention-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/validation-evidence-retention/report.json
if-no-files-found: error
validation-package-request:
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha != '' }}
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Validate exclusive package-only request
env:
PACKAGE_INPUTS: ${{ toJSON(inputs) }}
run: |
python3 - <<'PY'
import json
import os
import re
value = json.loads(os.environ["PACKAGE_INPUTS"])
if re.fullmatch(r"[0-9a-f]{40}", value["validation_package_sha"]) is None:
raise ValueError("package build requires an exact source SHA")
exclusive = ("validation_retain_evidence_run_id", "validation_runner_inspect_only", "build_run_id", "build_run_attempt", "validation_case", "auto_testing_sha",
"validation_tools_only", "validation_native_tools", "validation_shared_tools",
"validation_manager_tools", "validation_cleanup_run_id", "validation_cleanup_retained_evidence", "validation_tools_run_id")
if any(value.get(name) for name in exclusive):
raise ValueError("package-only build cannot select existing candidate, tools, fleet or recovery execution")
PY
validation-package:
needs: validation-package-request
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha != '' }}
uses: ./.github/workflows/rustfs-validation-package.yml
with:
source_sha: ${{ inputs.validation_package_sha }}
secrets:
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
prepare:
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule' }}
if: ${{ (github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && inputs.validation_case == '' && !inputs.validation_tools_only && !inputs.validation_native_tools && !inputs.validation_shared_tools && !inputs.validation_manager_tools && !inputs.validation_cleanup_retained_evidence && inputs.validation_cleanup_run_id == '') || github.event.workflow_run.event == 'schedule' }}
runs-on: smoke-testing
timeout-minutes: 10
outputs:
@@ -89,6 +258,395 @@ jobs:
echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
fi
validation:
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '' || inputs.validation_cleanup_retained_evidence) }}
runs-on: ${{ inputs.validation_tools_only && 'ubuntu-latest' || 'smoke-testing' }}
timeout-minutes: 90
outputs:
source_sha: ${{ steps.candidate.outputs.source_sha }}
concurrency:
group: ${{ inputs.validation_tools_only && 'rustfs-validation-tools-resolver' || 'rustfs-shared-functional-tests-v2' }}
cancel-in-progress: false
env:
RUSTFS_ACCESS_KEY: ${{ !inputs.validation_tools_only && secrets.RUSTFS_ACCESS_KEY || '' }}
RUSTFS_SECRET_KEY: ${{ !inputs.validation_tools_only && secrets.RUSTFS_SECRET_KEY || '' }}
RUSTFS_API_ENDPOINT: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT) || '' }}
RUSTFS_NODES: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_NODES || vars.RUSTFS_NODES) || '' }}
RUSTFS_SSH_USER: ${{ !inputs.validation_tools_only && (secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER) || '' }}
steps:
- name: Checkout candidate resolver
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Initialize validation evidence
env:
VALIDATION_TOOLS_ONLY: ${{ inputs.validation_tools_only }}
run: |
set -euo pipefail
umask 077
python3 -B scripts/initialize_validation_evidence.py
- name: Resolve immutable candidate
id: candidate
env:
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
BUILD_RUN_ID: ${{ inputs.build_run_id }}
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
CLEANUP_RUN_ID: ${{ inputs.validation_cleanup_run_id }}
CLEANUP_RETAINED_EVIDENCE: ${{ inputs.validation_cleanup_retained_evidence }}
SELECTED_CASE: ${{ inputs.validation_case }}
TOOLS_ONLY: ${{ inputs.validation_tools_only }}
NATIVE_TOOLS: ${{ inputs.validation_native_tools }}
SHARED_TOOLS: ${{ inputs.validation_shared_tools }}
MANAGER_TOOLS: ${{ inputs.validation_manager_tools }}
I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}
run: |
set -euo pipefail
python3 - <<'PY'
import json
import os
from pathlib import Path
import re
from scripts.resolve_functional_candidate import resolve
revision = os.environ["AUTO_TESTING_SHA"]
cleanup = os.environ["CLEANUP_RUN_ID"]
if os.environ["CLEANUP_RETAINED_EVIDENCE"] == "true" and not cleanup:
raise ValueError("persistent recovery input requires one exact cleanup run")
if os.environ["TOOLS_ONLY"] == "true" and (cleanup or os.environ["SELECTED_CASE"]):
raise ValueError("tools-only execution cannot select a fleet case or cleanup")
if os.environ["NATIVE_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
raise ValueError("native regressions require tools-only execution, without a fleet case or cleanup")
if os.environ["SHARED_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
raise ValueError("shared fixture builds require tools-only execution, without a fleet case or cleanup")
if os.environ["MANAGER_TOOLS"] == "true" and (os.environ["TOOLS_ONLY"] != "true" or cleanup or os.environ["SELECTED_CASE"]):
raise ValueError("manager regressions require tools-only execution, without a fleet case or cleanup")
if cleanup and (not cleanup.isdigit() or int(cleanup) <= 0 or os.environ["SELECTED_CASE"] or os.environ["TOOLS_ONLY"] == "true"):
raise ValueError("cleanup requires one exact failed run and is exclusive with case/tool execution")
diagnostic_branch = os.environ["I05_DIAGNOSTIC_BRANCH"] or "timeout"
if diagnostic_branch not in {"timeout", "queued-successor"}:
raise ValueError("I05 diagnostic branch is not registered")
if diagnostic_branch == "queued-successor" and os.environ["SELECTED_CASE"] != "heal.lock-timeout-and-caller-abort":
raise ValueError("queued-successor is available only for the I05 diagnostic case")
if re.fullmatch(r"[0-9a-f]{40}", revision) is None:
raise ValueError("validation requires an exact auto-testing commit SHA")
candidate = resolve(int(os.environ["BUILD_RUN_ID"]), int(os.environ["BUILD_RUN_ATTEMPT"]), allow_validation=True)
manifest = candidate["manifest"]
root = Path(os.environ["VALIDATION_ROOT"])
(root / "candidate-source.json").write_text(json.dumps(candidate, sort_keys=True) + "\n")
(root / "candidate.json").write_text(json.dumps({
"package_url": manifest["package_url"],
"package_sha256": manifest["package_sha256"],
"source_sha": manifest["source_sha"],
}, sort_keys=True) + "\n")
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
output.write("source_sha=" + manifest["source_sha"] + "\n")
PY
- name: Checkout exact auto-testing revision
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
repository: rustfs/auto-testing
ref: ${{ inputs.auto_testing_sha }}
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
path: auto-testing
persist-credentials: false
- name: Prepare validation Python runtime
if: ${{ !inputs.validation_tools_only && (startsWith(inputs.validation_case, 'ilm.') || startsWith(inputs.validation_case, 'pool.') || startsWith(inputs.validation_case, 'bucket-replication.') || startsWith(inputs.validation_case, 'site-replication.') || startsWith(inputs.validation_case, 'heal.')) }}
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
- name: Install validation Python client
if: ${{ !inputs.validation_tools_only && (startsWith(inputs.validation_case, 'ilm.') || startsWith(inputs.validation_case, 'pool.') || startsWith(inputs.validation_case, 'bucket-replication.') || startsWith(inputs.validation_case, 'site-replication.') || startsWith(inputs.validation_case, 'heal.')) }}
run: |
set -euo pipefail
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
test ! -e "${validation_python}"
python3 -m venv "${validation_python}"
"${validation_python}/bin/python" -m pip install \
-r auto-testing/scripts/validation_ilm/requirements.txt
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
- name: Validate Heal tool build identity
if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
env:
GH_TOKEN: ${{ github.token }}
TOOL_RUN_ID: ${{ inputs.validation_tools_run_id }}
TOOL_RUN_ATTEMPT: ${{ inputs.validation_tools_run_attempt }}
run: |
set -euo pipefail
[[ "${TOOL_RUN_ID}" =~ ^[1-9][0-9]*$ ]]
[[ "${TOOL_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]
gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${TOOL_RUN_ID}/attempts/${TOOL_RUN_ATTEMPT}" \
> "${VALIDATION_ROOT}/tool-run.json"
python3 - <<'PY'
import json
import os
from pathlib import Path
value = json.loads((Path(os.environ["VALIDATION_ROOT"]) / "tool-run.json").read_text())
if (value.get("conclusion") != "success" or value.get("status") != "completed"
or value.get("run_attempt") != int(os.environ["TOOL_RUN_ATTEMPT"])
or value.get("path") not in {".github/workflows/rustfs-functional-chain.yml", ".github/workflows/rustfs-validation-tools.yml"}):
raise ValueError("Heal tool run is not an exact successful validation build attempt")
PY
- name: Download pinned Heal metadata inspector
if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
github-token: ${{ github.token }}
run-id: ${{ inputs.validation_tools_run_id }}
name: rustfs-validation-tools-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
path: ${{ runner.temp }}/rustfs-validation-tools-${{ github.run_id }}-${{ github.run_attempt }}
- name: Verify Heal metadata inspector
if: ${{ !inputs.validation_tools_only && startsWith(inputs.validation_case, 'heal.') }}
run: |
set -euo pipefail
TOOL_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-tools-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
python3 -B auto-testing/scripts/validation_heal/inspector.py \
--directory "${TOOL_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
> "${VALIDATION_ROOT}/tool-manifest.json"
printf 'VALIDATION_HEAL_TOOL_DIRECTORY=%s\n' "${TOOL_DIRECTORY}" >> "${GITHUB_ENV}"
- name: Select exact-source race native evidence
id: race_native
if: ${{ inputs.validation_case == 'heal.race-unversioned-overwrite' || inputs.validation_case == 'heal.race-versioned-put' || inputs.validation_case == 'heal.race-version-delete' || inputs.validation_case == 'heal.race-markers-and-batch-delete' || inputs.validation_case == 'heal.race-multipart-complete-abort' || inputs.validation_case == 'heal.admin-nolock-safety' || inputs.validation_case == 'heal.race-copy-and-metadata-update' }}
env:
CANDIDATE_SHA: ${{ steps.candidate.outputs.source_sha }}
VALIDATION_CASE_ID: ${{ inputs.validation_case }}
run: |
set -euo pipefail
required=false
if [ -f auto-testing/scripts/validation_heal/race_native.py ]; then
status=0
python3 -B auto-testing/scripts/validation_heal/race_native.py supports \
--commit "${CANDIDATE_SHA}" --case-id "${VALIDATION_CASE_ID}" || status=$?
if [ "${status}" -eq 0 ]; then
required=true
elif [ "${status}" -ne 2 ]; then
exit "${status}"
fi
fi
reviewed=false
case "${CANDIDATE_SHA}:${VALIDATION_CASE_ID}" in
6afdec9fe1b0b88007f2e5b445c19bf73d51d3ca:heal.race-unversioned-overwrite | \
bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-unversioned-overwrite | \
bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-versioned-put | \
bd0671ba781d8ebf509f4f03533228dfb44496f1:heal.race-version-delete | \
8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-unversioned-overwrite | \
8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-versioned-put | \
8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-version-delete | \
8380466f3a27c3c3ef4115084178a0ab99a78f32:heal.race-markers-and-batch-delete | \
757b751482bb970bc321e92059690a7698554358:heal.admin-nolock-safety | \
757b751482bb970bc321e92059690a7698554358:heal.race-multipart-complete-abort | \
757b751482bb970bc321e92059690a7698554358:heal.race-copy-and-metadata-update) reviewed=true ;;
esac
if [ "${reviewed}" = true ] && [ "${required}" != true ]; then
printf 'Race candidate requires its reviewed native collector before fleet execution\n' >&2
exit 1
fi
printf 'required=%s\n' "${required}" >> "${GITHUB_OUTPUT}"
- name: Download pinned native storage evidence
if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || steps.race_native.outputs.required == 'true' || (inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch == 'queued-successor') }}
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
github-token: ${{ github.token }}
run-id: ${{ inputs.validation_tools_run_id }}
name: rustfs-validation-native-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
path: ${{ runner.temp }}/rustfs-validation-native-${{ github.run_id }}-${{ github.run_attempt }}
- name: Verify C07 native source and exact test evidence
if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' }}
run: |
set -euo pipefail
NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
python3 -B auto-testing/scripts/validation_heal/authority.py \
--directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
> "${VALIDATION_ROOT}/native-authority-manifest.json"
printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
- name: Verify C11 physical storage and exact final-rename barrier
if: ${{ inputs.validation_case == 'heal.bucket-incarnation-recreation' }}
run: |
set -euo pipefail
NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
python3 -B auto-testing/scripts/validation_heal/incarnation_native.py \
--directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
--testing-commit "${{ inputs.auto_testing_sha }}" \
> "${VALIDATION_ROOT}/native-incarnation-manifest.json"
printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
- name: Verify exact-source race native and runtime evidence
if: ${{ steps.race_native.outputs.required == 'true' }}
env:
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
VALIDATION_CASE_ID: ${{ inputs.validation_case }}
run: |
set -euo pipefail
NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \
--directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
--testing-commit "${AUTO_TESTING_SHA}" --case-id "${VALIDATION_CASE_ID}" \
> "${VALIDATION_ROOT}/native-race-manifest.json"
runtime_driver=race.py
if [ "${VALIDATION_CASE_ID}" = 'heal.admin-nolock-safety' ]; then
runtime_driver=nolock.py
fi
VALIDATION_HEAL_NATIVE_DIRECTORY="${NATIVE_DIRECTORY}" \
python3 -B "auto-testing/scripts/validation_heal/${runtime_driver}" preflight \
--candidate "${VALIDATION_ROOT}/candidate.json" --testing-commit "${AUTO_TESTING_SHA}" \
--case-id "${VALIDATION_CASE_ID}" \
> "${VALIDATION_ROOT}/native-race-runtime-preflight.json"
printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
- name: Verify I05 queued-successor native prerequisite
if: ${{ inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch == 'queued-successor' }}
env:
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
run: |
set -euo pipefail
NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \
--directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
--testing-commit "${AUTO_TESTING_SHA}" --case-id heal.lock-timeout-and-caller-abort \
> "${VALIDATION_ROOT}/native-i05-prerequisite.json"
printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
- name: Download pinned manager persistence and incarnation evidence
if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || inputs.validation_case == 'heal.outcome-counters-and-error-truth' }}
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
github-token: ${{ github.token }}
run-id: ${{ inputs.validation_tools_run_id }}
name: rustfs-validation-manager-native-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
path: ${{ runner.temp }}/rustfs-validation-manager-native-${{ github.run_id }}-${{ github.run_attempt }}
- name: Verify exact manager native prerequisites before fleet mutation
if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || inputs.validation_case == 'heal.outcome-counters-and-error-truth' }}
env:
VALIDATION_CASE: ${{ inputs.validation_case }}
run: |
set -euo pipefail
MANAGER_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-manager-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
cp "${VALIDATION_ROOT}/tool-run.json" "${MANAGER_DIRECTORY}/tool-run.json"
python3 -B auto-testing/scripts/validation_heal/manager_fixture.py \
--directory "${MANAGER_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
--case-id "${VALIDATION_CASE}" > "${VALIDATION_ROOT}/manager-prerequisite-manifest.json"
printf 'VALIDATION_HEAL_MANAGER_DIRECTORY=%s\n' "${MANAGER_DIRECTORY}" >> "${GITHUB_ENV}"
- name: Download pinned C10 shared-reference encoder
if: ${{ inputs.validation_case == 'heal.dangling-and-orphan-safety' }}
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
github-token: ${{ github.token }}
run-id: ${{ inputs.validation_tools_run_id }}
name: rustfs-validation-shared-${{ inputs.validation_tools_run_id }}-${{ inputs.validation_tools_run_attempt }}
path: ${{ runner.temp }}/rustfs-validation-shared-${{ github.run_id }}-${{ github.run_attempt }}
- name: Verify C10 exact-source codec and original tests
if: ${{ inputs.validation_case == 'heal.dangling-and-orphan-safety' }}
run: |
set -euo pipefail
SHARED_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-shared-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
cp "${VALIDATION_ROOT}/tool-run.json" "${SHARED_DIRECTORY}/tool-run.json"
python3 -B auto-testing/scripts/validation_heal/shared_fixture.py \
--directory "${SHARED_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
> "${VALIDATION_ROOT}/shared-fixture-manifest.json"
printf 'VALIDATION_HEAL_SHARED_DIRECTORY=%s\n' "${SHARED_DIRECTORY}" >> "${GITHUB_ENV}"
- name: Run selected canonical validation case
if: ${{ !inputs.validation_tools_only && inputs.validation_cleanup_run_id == '' && inputs.validation_case != 'environment.inspect' }}
env:
VALIDATION_CASE: ${{ inputs.validation_case }}
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}
run: |
set -euo pipefail
test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
./auto-testing/scripts/run_validation_case.sh \
"${VALIDATION_CASE}" "${VALIDATION_ROOT}/candidate.json" \
"${AUTO_TESTING_SHA}" "${VALIDATION_ROOT}/run"
- name: Inspect current Heal fleet resources without mutation
if: ${{ !inputs.validation_tools_only && inputs.validation_case == 'environment.inspect' }}
env:
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
run: |
set -euo pipefail
test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
python3 -B auto-testing/scripts/validation_heal/environment.py \
--candidate "${VALIDATION_ROOT}/candidate.json" \
--testing-commit "${AUTO_TESTING_SHA}" --evidence "${VALIDATION_ROOT}/environment"
- name: Audit and finalize retained validation environment
if: ${{ inputs.validation_cleanup_run_id != '' }}
env:
GH_TOKEN: ${{ github.token }}
FAILED_RUN_ID: ${{ inputs.validation_cleanup_run_id }}
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
CLEANUP_RETAINED_EVIDENCE: ${{ inputs.validation_cleanup_retained_evidence }}
run: |
set -euo pipefail
test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
gh run view "${FAILED_RUN_ID}" --repo "${GITHUB_REPOSITORY}" \
--json status,conclusion,headBranch,workflowName > "${VALIDATION_ROOT}/failed-run.json"
python3 - <<'PY'
import json
import os
from pathlib import Path
run = json.loads((Path(os.environ["VALIDATION_ROOT"]) / "failed-run.json").read_text())
if (run["status"] != "completed" or run["conclusion"] not in {"failure", "cancelled"}
or run["workflowName"] != "RustFS Functional Chain" or run["headBranch"] != os.environ["GITHUB_REF_NAME"]):
raise ValueError("cleanup source must be a completed failed or cancelled Functional Chain run")
PY
if [ "${CLEANUP_RETAINED_EVIDENCE}" = true ]; then
python3 -B scripts/prepare_retained_validation_recovery.py \
--report "${VALIDATION_ROOT}/retained-recovery-input.json"
else
gh run download "${FAILED_RUN_ID}" --repo "${GITHUB_REPOSITORY}" \
--pattern "rustfs-validation-${FAILED_RUN_ID}-*" --dir "${VALIDATION_ROOT}/failed-artifact"
fi
python3 -B auto-testing/scripts/validation_heal/recover_environment.py \
--failed-run-id "${FAILED_RUN_ID}" --artifact "${VALIDATION_ROOT}/failed-artifact" \
--candidate "${VALIDATION_ROOT}/candidate.json" --evidence "${VALIDATION_ROOT}/recovery"
- name: Show validation report
if: ${{ always() }}
run: |
if [ -f "${VALIDATION_ROOT}/run/report.md" ]; then
cat "${VALIDATION_ROOT}/run/report.md" >> "${GITHUB_STEP_SUMMARY}"
fi
- name: Upload validation evidence
if: ${{ always() && env.VALIDATION_ROOT != '' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: rustfs-validation-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ env.VALIDATION_ROOT }}
if-no-files-found: error
validation-tools:
needs: [validation]
if: ${{ github.event_name == 'workflow_dispatch' && inputs.validation_package_sha == '' && inputs.validation_tools_only }}
uses: ./.github/workflows/rustfs-validation-tools.yml
with:
candidate_sha: ${{ needs.validation.outputs.source_sha }}
auto_testing_sha: ${{ inputs.auto_testing_sha }}
build_native_heal: ${{ inputs.validation_native_tools }}
build_shared_heal: ${{ inputs.validation_shared_tools }}
build_manager_native: ${{ inputs.validation_manager_tools }}
secrets: inherit
upgrade:
needs: [prepare]
if: ${{ always() && needs.prepare.result == 'success' }}
@@ -0,0 +1,244 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
name: RustFS Validation Package
on:
workflow_call:
inputs:
source_sha:
description: Exact RustFS commit SHA to build
required: true
type: string
secrets:
R2_ACCESS_KEY_ID:
required: true
R2_SECRET_ACCESS_KEY:
required: true
R2_ENDPOINT:
required: true
R2_BUCKET:
required: true
workflow_dispatch:
inputs:
source_sha:
description: Exact RustFS commit SHA to build
required: true
type: string
permissions:
contents: read
concurrency:
group: rustfs-validation-package-${{ inputs.source_sha }}
cancel-in-progress: false
jobs:
resolve-source:
if: github.repository == 'rustfs/rustfs'
name: Resolve exact validation source
runs-on: ubuntu-latest
timeout-minutes: 10
outputs:
source_sha: ${{ steps.source.outputs.sha }}
source_ref: ${{ steps.source.outputs.ref }}
steps:
- name: Validate requested source
id: source
env:
REQUESTED_SHA: ${{ inputs.source_sha }}
run: |
set -euo pipefail
[[ "${REQUESTED_SHA}" =~ ^[0-9a-f]{40}$ ]]
source_directory="$(mktemp -d "${RUNNER_TEMP}/validation-source.XXXXXX")"
trap 'rm -rf "${source_directory}"' EXIT
git init -q "${source_directory}"
git -C "${source_directory}" fetch --no-tags --depth=1 https://github.com/rustfs/rustfs.git "${REQUESTED_SHA}"
test "$(git -C "${source_directory}" rev-parse FETCH_HEAD)" = "${REQUESTED_SHA}"
echo "sha=${REQUESTED_SHA}" >> "${GITHUB_OUTPUT}"
echo "ref=${REQUESTED_SHA}" >> "${GITHUB_OUTPUT}"
build:
needs: resolve-source
name: Build immutable validation DEB
runs-on: ubuntu-latest
timeout-minutes: 150
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }}
steps:
- name: Checkout exact source
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
ref: ${{ needs.resolve-source.outputs.source_sha }}
- name: Confirm checked out source
env:
EXPECTED_SHA: ${{ needs.resolve-source.outputs.source_sha }}
run: test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}"
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
cache-shared-key: validation-package-hosted-x86_64-unknown-linux-gnu
cache-save-if: "false"
install-build-packaging-tools: "false"
install-test-tools: "false"
- name: Build RustFS
run: cargo build --release --locked --target x86_64-unknown-linux-gnu -p rustfs --bins
- name: Build validation DEB
shell: bash
run: |
set -euo pipefail
command -v fakeroot >/dev/null 2>&1 || sudo apt-get install -y -qq fakeroot
package_dir="rustfs-validation-${VALIDATION_SOURCE_SHA}"
package_file="${RUNNER_TEMP}/validation-package/rustfs.deb"
mkdir -p "${RUNNER_TEMP}/validation-package"
binary="target/x86_64-unknown-linux-gnu/release/rustfs"
test -x "${binary}"
mkdir -p "${package_dir}/DEBIAN" "${package_dir}/usr/bin" "${package_dir}/etc/default" \
"${package_dir}/lib/systemd/system" "${package_dir}/usr/share/doc/rustfs"
cp "${binary}" "${package_dir}/usr/bin/rustfs"
chmod 755 "${package_dir}/usr/bin/rustfs"
cp deploy/build/rustfs.service "${package_dir}/lib/systemd/system/"
cat > "${package_dir}/etc/default/rustfs" <<'ENVEOF'
# RustFS Environment Configuration
# RUSTFS_ACCESS_KEY=<changeme>
# RUSTFS_SECRET_KEY=<changeme>
# RUSTFS_VOLUMES=""
# RUSTFS_ADDRESS=":9000"
ENVEOF
cat > "${package_dir}/DEBIAN/control" <<'CONTROLEOF'
Package: rustfs
Version: 0.0.0~validation
Section: utils
Priority: optional
Architecture: amd64
Depends: libc6 (>= 2.31)
Maintainer: RustFS Team <support@rustfs.com>
Description: RustFS immutable validation package
Candidate package for RustFS validation only.
CONTROLEOF
printf '%s\n' /etc/default/rustfs > "${package_dir}/DEBIAN/conffiles"
./scripts/release/package_service_scripts.sh after-install > "${package_dir}/DEBIAN/postinst"
./scripts/release/package_service_scripts.sh before-remove > "${package_dir}/DEBIAN/prerm"
./scripts/release/package_service_scripts.sh after-remove > "${package_dir}/DEBIAN/postrm"
chmod 755 "${package_dir}/DEBIAN/postinst" "${package_dir}/DEBIAN/prerm" "${package_dir}/DEBIAN/postrm"
cp LICENSE README.md "${package_dir}/usr/share/doc/rustfs/"
fakeroot dpkg-deb --build "${package_dir}" "${package_file}"
test -s "${package_file}"
- name: Upload opaque validation package
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: validation-package-bytes-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/validation-package/rustfs.deb
if-no-files-found: error
publish:
needs: [resolve-source, build]
name: Publish immutable validation DEB
# Candidate-controlled build scripts never run on this fresh hosted runner.
runs-on: ubuntu-latest
timeout-minutes: 20
env:
VALIDATION_SOURCE_SHA: ${{ needs.resolve-source.outputs.source_sha }}
VALIDATION_SOURCE_REF: ${{ needs.resolve-source.outputs.source_ref }}
steps:
- name: Download opaque validation package
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: validation-package-bytes-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/validation-package-input
- name: Validate and stage opaque package bytes
shell: bash
run: |
set -euo pipefail
python3 -I - <<'PYTHON'
import os
from pathlib import Path
import shutil
import stat
root = Path(os.environ["RUNNER_TEMP"])
incoming = root / "validation-package-input"
if incoming.is_symlink() or not incoming.is_dir() or sorted(p.name for p in incoming.iterdir()) != ["rustfs.deb"]:
raise SystemExit("Validation artifact must contain exactly one rustfs.deb file")
fd = os.open(incoming / "rustfs.deb", os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
with os.fdopen(fd, "rb") as source:
info = os.fstat(source.fileno())
if not stat.S_ISREG(info.st_mode) or info.st_size == 0:
raise SystemExit("Validation package must be a nonempty regular file")
staged = root / "validation-publication"
staged.mkdir(mode=0o700)
with (staged / "rustfs.deb").open("xb") as target:
shutil.copyfileobj(source, target)
PYTHON
- name: Prepare trusted publication tools
id: tools
env:
AWS_EC2_METADATA_DISABLED: true
shell: bash
run: |
set -euo pipefail
AWS_CLI=aws
if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then
sudo apt-get update
sudo apt-get install -y -qq python3-venv
AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/validation-awscli.XXXXXX")"
python3 -m venv "${AWS_CLI_DIR}"
"${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79'
AWS_CLI="${AWS_CLI_DIR}/bin/aws"
fi
"${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null
echo "aws_cli=${AWS_CLI}" >> "${GITHUB_OUTPUT}"
- name: Upload validation package to immutable R2 path
id: publish
env:
PACKAGE_FILE: ${{ runner.temp }}/validation-publication/rustfs.deb
AWS_CLI: ${{ steps.tools.outputs.aws_cli }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
AWS_EC2_METADATA_DISABLED: true
shell: bash
run: |
set -euo pipefail
if [[ -z "${R2_ACCESS_KEY_ID}" || -z "${R2_SECRET_ACCESS_KEY}" || -z "${R2_ENDPOINT}" || -z "${R2_BUCKET}" ]]; then
echo "Validation package publication requires complete R2 configuration" >&2
exit 1
fi
[[ "${VALIDATION_SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]]
test "${VALIDATION_SOURCE_REF}" = "${VALIDATION_SOURCE_SHA}"
test -s "${PACKAGE_FILE}" && test ! -L "${PACKAGE_FILE}"
export AWS_ACCESS_KEY_ID="${R2_ACCESS_KEY_ID}" AWS_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY}" AWS_DEFAULT_REGION=auto
package_sha256="$(sha256sum "${PACKAGE_FILE}" | cut -d ' ' -f 1)"
key="artifacts/rustfs/packages/validation/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${package_sha256}/rustfs.deb"
url="https://dl.rustfs.com/${key}"
"${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${key}" --body "${PACKAGE_FILE}" \
--if-none-match '*' --endpoint-url "${R2_ENDPOINT}"
downloaded_sha256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${url}" | sha256sum | cut -d ' ' -f 1)"
test "${downloaded_sha256}" = "${package_sha256}"
manifest="${RUNNER_TEMP}/validation-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
jq -n --arg workflow_sha "${GITHUB_SHA}" --arg source_ref "${VALIDATION_SOURCE_REF}" \
--arg source_sha "${VALIDATION_SOURCE_SHA}" --arg channel validation \
--argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \
--arg package_url "${url}" --arg package_sha256 "${package_sha256}" \
'{schema: 3, workflow_sha: $workflow_sha, source_ref: $source_ref, source_sha: $source_sha, channel: $channel, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' > "${manifest}"
echo "candidate_file=${manifest}" >> "${GITHUB_OUTPUT}"
- name: Upload validation candidate manifest
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: validation-candidate-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ steps.publish.outputs.candidate_file }}
if-no-files-found: error
@@ -0,0 +1,343 @@
# Build source-bound helpers separately so live validation needs only artifacts.
name: RustFS Validation Tools
on:
workflow_call:
inputs:
candidate_sha:
type: string
required: true
auto_testing_sha:
type: string
required: true
build_native_heal:
type: boolean
required: false
default: false
build_shared_heal:
type: boolean
required: false
default: false
build_manager_native:
type: boolean
required: false
default: false
secrets:
PF_TESTING_GH_TOKEN:
required: true
workflow_dispatch:
inputs:
candidate_sha:
description: Exact RustFS source SHA from the validation candidate manifest
type: string
required: true
auto_testing_sha:
description: Exact auto-testing SHA containing the Heal metadata inspector
type: string
required: true
build_native_heal:
description: Also compile and run separate exact-source native Heal regressions
type: boolean
required: false
default: false
build_shared_heal:
description: Build the controlled C10 shared-reference fixture encoder
type: boolean
required: false
default: false
build_manager_native:
description: Build and run exact candidate checkpoint/MRF/incarnation regressions
type: boolean
required: false
default: false
permissions:
contents: read
jobs:
manager-native-regressions:
if: ${{ inputs.build_manager_native }}
runs-on: ubuntu-22.04
timeout-minutes: 180
env:
CANDIDATE_SHA: ${{ inputs.candidate_sha }}
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
steps:
- name: Validate immutable manager native inputs
run: |
set -euo pipefail
[[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
[[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
- name: Checkout exact manager candidate source
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
ref: ${{ inputs.candidate_sha }}
path: candidate
persist-credentials: false
- name: Checkout exact manager collector source
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
repository: rustfs/auto-testing
ref: ${{ inputs.auto_testing_sha }}
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
path: auto-testing
persist-credentials: false
- name: Install manager native build dependencies
run: |
set -euo pipefail
sudo apt-get update
sudo apt-get install -y clang lld cmake pkg-config libssl-dev protobuf-compiler
df -h .
- name: Build exact unmodified candidate manager test executable
run: |
set -euo pipefail
test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
python3 -B auto-testing/scripts/validation_heal/manager_native.py build \
--source candidate --commit "${CANDIDATE_SHA}" --out manager-tools
- name: Run exact manager regressions independently
run: |
set -euo pipefail
python3 -B auto-testing/scripts/validation_heal/manager_native.py run \
--directory manager-tools --out manager-evidence
- name: Upload manager executable identities and original diagnostics
if: ${{ always() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: rustfs-validation-manager-native-${{ github.run_id }}-${{ github.run_attempt }}
path: |
manager-tools/heal-manager-native-tests
manager-tools/*.json
manager-tools/*.jsonl
manager-tools/*.stderr
manager-tools/*.log
manager-evidence/
if-no-files-found: warn
native-heal-regressions:
if: ${{ inputs.build_native_heal }}
runs-on: ubuntu-22.04
timeout-minutes: 180
env:
CANDIDATE_SHA: ${{ inputs.candidate_sha }}
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
steps:
- name: Validate immutable native source inputs
run: |
set -euo pipefail
[[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
[[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
- name: Checkout exact native candidate source
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
ref: ${{ inputs.candidate_sha }}
path: candidate
persist-credentials: false
- name: Checkout exact native collector source
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
repository: rustfs/auto-testing
ref: ${{ inputs.auto_testing_sha }}
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
path: auto-testing
persist-credentials: false
- name: Install native build dependencies
run: |
set -euo pipefail
sudo apt-get update
sudo apt-get install -y clang lld cmake pkg-config libssl-dev protobuf-compiler
df -h .
- name: Build and enumerate the unmodified candidate test executable
run: |
set -euo pipefail
test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
use_race_native=false
if [[ -f auto-testing/scripts/validation_heal/race_native.py ]]; then
support_rc=0
python3 -B auto-testing/scripts/validation_heal/race_native.py supports \
--commit "${CANDIDATE_SHA}" || support_rc=$?
case "${support_rc}" in
0) use_race_native=true ;;
2) ;; # The original exact-source collector handles legacy candidates.
*) exit "${support_rc}" ;;
esac
fi
if [[ "${use_race_native}" == true ]]; then
python3 -B auto-testing/scripts/validation_heal/race_native.py build \
--source candidate --commit "${CANDIDATE_SHA}" --testing-commit "${AUTO_TESTING_SHA}" --out native-tools
else
python3 -B auto-testing/scripts/validation_heal/native_regressions.py build \
--source candidate --commit "${CANDIDATE_SHA}" --out native-tools
fi
- name: Run exact native regressions separately from fleet validation
run: |
set -euo pipefail
if [[ -f native-tools/race-native.json ]]; then
test ! -e native-tools/native-regressions.json
python3 -B auto-testing/scripts/validation_heal/race_native.py run \
--directory native-tools --out native-evidence
else
python3 -B auto-testing/scripts/validation_heal/native_regressions.py run \
--directory native-tools --out native-evidence
fi
- name: Run existing F07 directory fsync failure regressions
if: ${{ inputs.candidate_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac' }}
run: |
set -euo pipefail
# Older immutable harness pins retain their original native suite;
# absence of this additive collector must never fabricate its proof.
if [[ ! -f auto-testing/scripts/validation_heal/persistence_native.py ]]; then
mkdir native-persistence-evidence
echo '{"complete":false,"reason":"pinned harness predates F07 directory-fsync collector","requiredNativeCoverageComplete":false,"canonicalCasePassClaimed":false}' \
> native-persistence-evidence/not-available.json
exit 0
fi
python3 -B auto-testing/scripts/validation_heal/persistence_native.py run \
--directory native-tools --native-evidence native-evidence --out native-persistence-evidence \
--testing-commit "${AUTO_TESTING_SHA}"
python3 -B auto-testing/scripts/validation_heal/persistence_native.py verify \
--directory native-tools --native-evidence native-evidence --out native-persistence-evidence \
--testing-commit "${AUTO_TESTING_SHA}"
- name: Upload native executable identities and original diagnostics
if: ${{ always() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: rustfs-validation-native-${{ github.run_id }}-${{ github.run_attempt }}
path: |
native-tools/ecstore-native-tests
native-tools/ecstore-race-native-tests
native-tools/*.json
native-tools/*.jsonl
native-tools/*.stderr
native-tools/*.log
native-evidence/
native-persistence-evidence/
if-no-files-found: warn
heal-metadata-inspector:
runs-on: ubuntu-22.04
timeout-minutes: 30
env:
CANDIDATE_SHA: ${{ inputs.candidate_sha }}
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
steps:
- name: Validate immutable source inputs
run: |
set -euo pipefail
[[ "${CANDIDATE_SHA}" =~ ^[0-9a-f]{40}$ ]]
[[ "${AUTO_TESTING_SHA}" =~ ^[0-9a-f]{40}$ ]]
- name: Checkout exact candidate source
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
ref: ${{ inputs.candidate_sha }}
path: candidate
persist-credentials: false
- name: Checkout exact inspector source
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
repository: rustfs/auto-testing
ref: ${{ inputs.auto_testing_sha }}
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
path: auto-testing
persist-credentials: false
- name: Build the read-only candidate metadata decoder
run: |
set -euo pipefail
test "$(git -C candidate rev-parse HEAD)" = "${CANDIDATE_SHA}"
test "$(git -C auto-testing rev-parse HEAD)" = "${AUTO_TESTING_SHA}"
df -h .
python3 auto-testing/scripts/validation_heal/prepare_heal_meta_inspector.py \
--source candidate --commit "${CANDIDATE_SHA}" --out inspector-build
# Seed the helper's separate graph from the candidate's frozen graph.
# Cargo adds the helper package; both lock identities are retained.
cp candidate/Cargo.lock inspector-build/Cargo.lock
cargo build --release --manifest-path inspector-build/Cargo.toml
python3 auto-testing/scripts/validation_heal/check_bitrot_tool.py \
--binary inspector-build/target/release/heal-meta-inspect --source candidate
mkdir validation-tools
cp inspector-build/target/release/heal-meta-inspect validation-tools/
cp inspector-build/Cargo.lock validation-tools/inspector-Cargo.lock
python3 - <<'PY'
import hashlib
import json
import os
from pathlib import Path
import subprocess
source = json.loads(Path("inspector-build/source.json").read_text())
binary = Path("validation-tools/heal-meta-inspect")
source.update(built=True, testingCommit=os.environ["AUTO_TESTING_SHA"],
binarySha256=hashlib.sha256(binary.read_bytes()).hexdigest(),
inspectorLockSha256=hashlib.sha256(Path("validation-tools/inspector-Cargo.lock").read_bytes()).hexdigest(),
toolchain=subprocess.check_output(["rustc", "-vV"], text=True),
target="x86_64-unknown-linux-gnu", runId=os.environ["GITHUB_RUN_ID"],
runAttempt=os.environ["GITHUB_RUN_ATTEMPT"])
Path("validation-tools/heal-meta-inspect.json").write_text(json.dumps(source, indent=2) + "\n")
PY
git -C candidate diff --exit-code
- name: Build and test the controlled C10 metadata encoder
if: ${{ inputs.build_shared_heal }}
run: |
set -euo pipefail
python3 -B auto-testing/scripts/validation_heal/prepare_heal_shared_fixture.py \
--source candidate --commit "${CANDIDATE_SHA}" --out shared-build
cp candidate/Cargo.lock shared-build/Cargo.lock
export CARGO_TARGET_DIR="${GITHUB_WORKSPACE}/inspector-build/target"
cargo build --release --manifest-path shared-build/Cargo.toml
mkdir shared-tools
if ! cargo test --release --manifest-path shared-build/Cargo.toml -- --test-threads=1 \
> shared-tools/shared-fixture-tests.log 2>&1; then
cat shared-tools/shared-fixture-tests.log
exit 1
fi
cp "${CARGO_TARGET_DIR}/release/heal-shared-fixture" shared-tools/
cp shared-build/Cargo.lock shared-tools/shared-Cargo.lock
python3 - <<'PY'
import hashlib
import json
import os
from pathlib import Path
import subprocess
source = json.loads(Path("shared-build/source.json").read_text())
source.update(schemaVersion=1, built=True, sourceUnmodified=True,
candidateSourceTree=subprocess.check_output(["git", "-C", "candidate", "rev-parse", "HEAD^{tree}"], text=True).strip(),
testingCommit=os.environ["AUTO_TESTING_SHA"],
binarySha256=hashlib.sha256(Path("shared-tools/heal-shared-fixture").read_bytes()).hexdigest(),
sharedLockSha256=hashlib.sha256(Path("shared-tools/shared-Cargo.lock").read_bytes()).hexdigest(),
toolchain=subprocess.check_output(["rustc", "-vV"], text=True), target="x86_64-unknown-linux-gnu",
runId=os.environ["GITHUB_RUN_ID"], runAttempt=os.environ["GITHUB_RUN_ATTEMPT"])
Path("shared-tools/heal-shared-fixture.json").write_text(json.dumps(source, indent=2) + "\n")
PY
git -C candidate diff --exit-code
- name: Upload controlled C10 fixture encoder evidence
if: ${{ always() && inputs.build_shared_heal }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: rustfs-validation-shared-${{ github.run_id }}-${{ github.run_attempt }}
path: shared-tools/
if-no-files-found: warn
- name: Upload immutable validation tools
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: rustfs-validation-tools-${{ github.run_id }}-${{ github.run_attempt }}
path: validation-tools/
if-no-files-found: error
@@ -16,8 +16,8 @@
#
# Forces a job failure and then runs .github/actions/schedule-failure-issue
# through the exact `needs` + `always() && contains(needs.*.result,
# 'failure')` wiring used by the real consumers (e2e-s3tests, mint, fuzz).
# Dispatch it twice to verify both the issue-creation and
# 'failure')` wiring used by the real consumers (e2e-s3tests, mint, fuzz,
# performance-ab). Dispatch it twice to verify both the issue-creation and
# the dedupe-comment paths, then close the resulting
# "[scheduled-failure] Schedule Failure Alert Drill" issue in rustfs/backlog
# (the default target repository). Requires the BACKLOG_ISSUE_TOKEN secret
@@ -28,6 +28,7 @@ on:
- "mint"
- "minio-interop"
- "Nightly GNU Build"
- "Performance A/B"
- "Runner Hygiene"
types: [completed]
Generated
+86 -169
View File
@@ -365,9 +365,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
[[package]]
name = "arrow"
version = "59.3.2"
version = "59.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f04e0a89cc175eaf3740282537851e6fd9d7276f97730b9d0ea2859c11a85df"
checksum = "4c25317ebddb8f2e6f006759336c95e5ded847a446f3524c27853bfe7d77a69b"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -386,9 +386,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
version = "59.3.2"
version = "59.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "157943c720919e8f5b6731407ce5cd4c9a5ef47a75bf10edaf24b7081b5763ef"
checksum = "2ab555c0ffe9cb817041c0cee80baa9663a22225d0a035693579771f2693cadc"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -400,9 +400,9 @@ dependencies = [
[[package]]
name = "arrow-array"
version = "59.3.2"
version = "59.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b5b1682924f11ecf6929689c16cbc69efbb6dd141b06ad98ee1cb10d858c43"
checksum = "2dae1d91737421e92c6dd7c7f9d0f2d25198fd6fc539738b628f4dad10adf533"
dependencies = [
"ahash",
"arrow-buffer",
@@ -419,9 +419,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
version = "59.3.2"
version = "59.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5d687afc6c927965a8a759f3b68de91933d9dfa2e1bb6e156bac84b85665e92"
checksum = "1fe9cd0064f2d7cf8ecfaf77cd70e67a980bc71aea92b63102bd37a7d3379a09"
dependencies = [
"bytes",
"half",
@@ -431,9 +431,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
version = "59.3.2"
version = "59.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f23c3379feede42135ef27ca60e67f6e643cd4343ae74e4c8b16c83f55afd85"
checksum = "ee510bc252781d700400979a96bd1eadebeae23901132bbabd92118da1b376c5"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -453,9 +453,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
version = "59.3.2"
version = "59.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85b1984628b091b8319b3c305fce21d04f9e9b2578bb733b0d0108462ae1d698"
checksum = "ace83d5935421b950fea3a0a3f9972c0837c297ef98ade57127a5661b681112c"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -468,9 +468,9 @@ dependencies = [
[[package]]
name = "arrow-data"
version = "59.3.2"
version = "59.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0a031c7ff5e2bdb443117054fb466af1d4df5b7f0ecbe197713dc8b113b1ba"
checksum = "54f128c73bb7fe9f7137218f910edd5b436f390e0f040b1956f5028390787897"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -481,9 +481,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
version = "59.3.2"
version = "59.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5dd1afadda2eb821aaf851bb4c0adef60c30cc0f1a29bd02fb99dd07fb9f58d"
checksum = "8e244f39b82fa7a823e8b5984a0bd988234f9c92c1df79378fb85090e6e4c821"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -497,9 +497,9 @@ dependencies = [
[[package]]
name = "arrow-json"
version = "59.3.2"
version = "59.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2bc4d56a0dad84cc51c5812ccff48b642047a1f3be80349cd84318983e79edb"
checksum = "727b21ebf8894dcd9504d6ab66bffdac338391e1f85cde594526a0210299ea40"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -522,9 +522,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
version = "59.3.2"
version = "59.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "561b8209faa2cd489764b7fd07d7cb9e8d274d78048c38de59f77c41140cb834"
checksum = "70d376b9b883239ae0f9963cdeeb59126ccaac08d60adf208a6ad3e2dd4048c3"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -535,9 +535,9 @@ dependencies = [
[[package]]
name = "arrow-row"
version = "59.3.2"
version = "59.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d8c68b87a569f92b90c46cb0a09d86bd9609ddc9b1bf1fc5a503c8c6907c14e"
checksum = "48096811f92b5c3a0c59d33cfecd7d7d2ed037d81e54a27a42d232e7700e6de2"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -548,9 +548,9 @@ dependencies = [
[[package]]
name = "arrow-schema"
version = "59.3.2"
version = "59.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e87acb0c6bf6237fca7a50d7b51e801c98a18a5fe6f3bb34f7d77b85ca165388"
checksum = "37b6c8a6b12cf3869d299ce36ed8e1cd6fe9836264b68fb6c6538473a36124a1"
dependencies = [
"serde_core",
"serde_json",
@@ -558,9 +558,9 @@ dependencies = [
[[package]]
name = "arrow-select"
version = "59.3.2"
version = "59.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c844141958d805b47d1208e0d12aac53fcf388ce8011639fe82aa492adde0bb4"
checksum = "e523e85fa9fb3181a391bf0edfa6a1ae0dc46df8cbf7f6b96034dcb1e2a7a04a"
dependencies = [
"ahash",
"arrow-array",
@@ -572,9 +572,9 @@ dependencies = [
[[package]]
name = "arrow-string"
version = "59.3.2"
version = "59.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc130892b1a9552da77971924b390e06cf375d1f7e4344a7b7b40ca4426fa470"
checksum = "b931a905a0e4b8832d9f54fb369cf6e545f9dc03cb3d9611216a7bd2653df60a"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -1514,23 +1514,6 @@ dependencies = [
"serde",
]
[[package]]
name = "bindgen"
version = "0.73.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "787ef8ef523575546b106a58213d6e6b06198a05c2f757258c68a74273670cfa"
dependencies = [
"bitflags 2.13.2",
"cexpr",
"clang-sys",
"proc-macro2",
"quote",
"regex",
"rustc-hash",
"shlex",
"syn 3.0.9",
]
[[package]]
name = "binrw"
version = "0.15.2"
@@ -1743,13 +1726,13 @@ dependencies = [
[[package]]
name = "brotli"
version = "9.1.0"
version = "9.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6737ce8b81460bf9407909f73ff50bf13f7cb8577e1e76715f57ed0eba58ba44"
checksum = "f8b851b75c23ca7873623d612fe49bd1989aeb03d08fb9432187eb253d3d4c6b"
dependencies = [
"alloc-no-stdlib 3.0.0",
"alloc-stdlib 0.3.0",
"brotli-decompressor 6.2.0",
"brotli-decompressor 6.0.1",
]
[[package]]
@@ -1764,9 +1747,9 @@ dependencies = [
[[package]]
name = "brotli-decompressor"
version = "6.2.0"
version = "6.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34c6783f42c21c952ad7112b354f0ba2f10760e9decea5b1b60523d6e270faaf"
checksum = "941cd9bd4ddab83cb46fa5a2d428f1c857b24ac78cb876cf7beb710840934bd7"
dependencies = [
"alloc-no-stdlib 3.0.0",
"alloc-stdlib 0.3.0",
@@ -1984,15 +1967,6 @@ dependencies = [
"shlex",
]
[[package]]
name = "cexpr"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
dependencies = [
"nom 7.1.3",
]
[[package]]
name = "cfg-if"
version = "1.0.5"
@@ -2109,17 +2083,6 @@ dependencies = [
"zeroize",
]
[[package]]
name = "clang-sys"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a"
dependencies = [
"glob",
"libc",
"libloading",
]
[[package]]
name = "clap"
version = "4.6.7"
@@ -2236,7 +2199,7 @@ version = "0.4.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe"
dependencies = [
"brotli 9.1.0",
"brotli 9.0.0",
"bzip2",
"compression-core",
"flate2",
@@ -2244,7 +2207,7 @@ dependencies = [
"lz4",
"memchr",
"zstd 0.14.3",
"zstd-safe",
"zstd-safe 8.2.0",
]
[[package]]
@@ -2579,9 +2542,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
[[package]]
name = "crypto-bigint"
version = "0.7.10"
version = "0.7.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c035cb64cba03dad2fb921f3e218785e8cf9a41790da5eb3a1400f62fa2be11"
checksum = "cc907c45bb7e9ef8fc3778defd0f3cd8c81288f1b4e748eba064948f20c47122"
dependencies = [
"cpubits",
"ctutils",
@@ -2657,9 +2620,9 @@ dependencies = [
[[package]]
name = "ctutils"
version = "0.4.6"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe3e9d21d39e6f39ff46e03d5b621424f88ee23d8a1672c6bfc9ccc7f2da4f6a"
checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0"
dependencies = [
"cmov",
"subtle",
@@ -3669,9 +3632,9 @@ dependencies = [
[[package]]
name = "der"
version = "0.8.3"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a996a108feb8b427a6e12fcf6af4aa83409b79d72c15337d7741368131ee004"
checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a"
dependencies = [
"const-oid 0.10.2",
"pem-rfc7468 1.0.0",
@@ -4077,7 +4040,6 @@ dependencies = [
"rustfs-utils",
"rustls",
"s3s",
"same-file",
"serde",
"serde_json",
"serde_urlencoded",
@@ -4104,7 +4066,7 @@ version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0"
dependencies = [
"der 0.8.3",
"der 0.8.2",
"digest 0.11.3",
"elliptic-curve",
"rfc6979",
@@ -4169,9 +4131,9 @@ checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
[[package]]
name = "elliptic-curve"
version = "0.14.3"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "900f37e7e18d74a2347389af6cd69e3a30d5cc9e697f40004865679aa75e9c7c"
checksum = "0ad4496221fcfc21015fd8c0e26fcfab00504832dc0604f52acdd7bad2a8dc81"
dependencies = [
"base16ct",
"crypto-bigint",
@@ -5273,9 +5235,9 @@ dependencies = [
[[package]]
name = "hotpath"
version = "0.29.2"
version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b4af2b5ecf2650278fe33b0e49b68c8c484d2306475c127628316dea740cf5c"
checksum = "62f2351902fa259b758c7303505273393f6b08172ab5bdd5739a2d2bc6d554e8"
dependencies = [
"arc-swap",
"async-channel",
@@ -5308,15 +5270,15 @@ dependencies = [
[[package]]
name = "hotpath-drain"
version = "0.29.2"
version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5cceb6c15fbaaeb00b51acc7be7fad533d0b56cb422b1cfc16cf9ca1624d33a"
checksum = "83dab2340134d9976d62f9f5172f9c6584c7f521b092f258e0e58e412e160b53"
[[package]]
name = "hotpath-macros"
version = "0.29.2"
version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08c0a42eb299ebb7ec6b9161489e5bf132603a34ca12cecadb8490937aa0492a"
checksum = "a04bbe5701b097f224a08187a78b75bb9d9b4bbaff6a279c2a50ba73edbb7aac"
dependencies = [
"proc-macro2",
"quote",
@@ -6436,7 +6398,7 @@ dependencies = [
"hashbrown 0.16.1",
"indexmap 2.14.2",
"metrics",
"ordered-float 5.5.1",
"ordered-float 5.5.0",
"quanta",
"radix_trie",
"rand 0.9.5",
@@ -7370,9 +7332,9 @@ dependencies = [
[[package]]
name = "ordered-float"
version = "5.5.1"
version = "5.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f59006ce1e204c30956638535b74d317bca99477b9356dfaec54c37f6f600ab"
checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0"
dependencies = [
"num-traits",
]
@@ -7489,9 +7451,9 @@ dependencies = [
[[package]]
name = "pageant"
version = "0.2.5"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0bc922d9aa8cfcf598de27b3c6122cb2eeda9d6fe71527a22ebcb7f633540ff"
checksum = "f0786409b5aaa884655c9b4065681db0835db0cc185a6fb19fb17a5d38656866"
dependencies = [
"base16ct",
"byteorder",
@@ -7560,9 +7522,9 @@ dependencies = [
[[package]]
name = "parquet"
version = "59.3.2"
version = "59.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8eb260d566f2fbdabe356d26aaa5d13403aac29b5b4e44463b8f9bd7b2b12680"
checksum = "8b3d2ef0fb136e9341570f219ecb168902e76079c6d81ad555f0a4840b1e08a3"
dependencies = [
"ahash",
"arrow-array",
@@ -7847,7 +7809,7 @@ version = "0.8.0-rc.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "986d2e952779af96ea048f160fd9194e1751b4faea78bcf3ceb456efe008088e"
dependencies = [
"der 0.8.3",
"der 0.8.2",
"spki 0.8.1",
]
@@ -7890,7 +7852,7 @@ dependencies = [
"aes 0.9.3",
"aes-gcm",
"cbc 0.2.1",
"der 0.8.3",
"der 0.8.2",
"pbkdf2 0.13.0",
"rand_core 0.10.1",
"scrypt 0.12.0",
@@ -7914,7 +7876,7 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7"
dependencies = [
"der 0.8.3",
"der 0.8.2",
"pkcs5 0.8.1",
"rand_core 0.10.1",
"spki 0.8.1",
@@ -9176,9 +9138,9 @@ dependencies = [
[[package]]
name = "russh"
version = "0.65.0"
version = "0.64.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3bb77a4b259312b2abbc541bca1c13cb4e9b61c560a7fd894068c3b92e0c9dc3"
checksum = "ba61e87b9ec9a39a59a6bbed4c0b8ff7fb07073405b24a767a1f2c8560ac8f58"
dependencies = [
"aes 0.9.3",
"aws-lc-rs",
@@ -9193,7 +9155,7 @@ dependencies = [
"curve25519-dalek 5.0.0",
"data-encoding",
"delegate",
"der 0.8.3",
"der 0.8.2",
"digest 0.11.3",
"ecdsa",
"ed25519-dalek 3.0.0",
@@ -10884,7 +10846,7 @@ version = "1.0.2"
dependencies = [
"base64-simd",
"blake2",
"brotli 9.1.0",
"brotli 9.0.0",
"bytes",
"convert_case 0.12.0",
"crc-fast",
@@ -11345,7 +11307,7 @@ checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d"
dependencies = [
"base16ct",
"ctutils",
"der 0.8.3",
"der 0.8.2",
"hybrid-array",
"subtle",
"zeroize",
@@ -11511,15 +11473,6 @@ dependencies = [
"syn 3.0.9",
]
[[package]]
name = "serde_spanned"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c"
dependencies = [
"serde_core",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
@@ -11799,9 +11752,9 @@ checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256"
[[package]]
name = "sketches-ddsketch"
version = "0.3.2"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45f82fa313d625dd72aaa65ec0c9f8c95407add0510c1bef3c18ea7257ec34ce"
checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
[[package]]
name = "slab"
@@ -11930,7 +11883,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9"
dependencies = [
"base64ct",
"der 0.8.3",
"der 0.8.2",
]
[[package]]
@@ -12679,30 +12632,6 @@ dependencies = [
"webpki-roots 0.26.11",
]
[[package]]
name = "toml"
version = "0.9.12+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"
dependencies = [
"indexmap 2.14.2",
"serde_core",
"serde_spanned",
"toml_datetime 0.7.5+spec-1.1.0",
"toml_parser",
"toml_writer",
"winnow 0.7.15",
]
[[package]]
name = "toml_datetime"
version = "0.7.5+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347"
dependencies = [
"serde_core",
]
[[package]]
name = "toml_datetime"
version = "1.1.2+spec-1.1.0"
@@ -12719,9 +12648,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c"
dependencies = [
"indexmap 2.14.2",
"toml_datetime 1.1.2+spec-1.1.0",
"toml_datetime",
"toml_parser",
"winnow 1.0.4",
"winnow",
]
[[package]]
@@ -12730,15 +12659,9 @@ version = "1.1.5+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c"
dependencies = [
"winnow 1.0.4",
"winnow",
]
[[package]]
name = "toml_writer"
version = "1.1.3+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770"
[[package]]
name = "tonic"
version = "0.14.6"
@@ -13765,12 +13688,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winnow"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
[[package]]
name = "winnow"
version = "1.0.4"
@@ -14037,9 +13954,9 @@ dependencies = [
[[package]]
name = "zip"
version = "9.1.2"
version = "9.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec2e3161de27091f18a831d9494e9c53378a34833af5c1dda35ad85849da18d6"
checksum = "2886b00003a72c4ccee3f42cfc34908d33b2508a74d827dc2a4244706475c704"
dependencies = [
"aes 0.9.3",
"bzip2",
@@ -14091,18 +14008,10 @@ dependencies = [
[[package]]
name = "zstd"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
"apache-avro",
"arrow-array",
"arrow-ipc",
"arrow-schema",
"bytes",
"kafka-protocol",
"parquet",
"semver",
"toml",
"zstd 0.14.3",
"zstd-safe",
"zstd-safe 7.3.0",
]
[[package]]
@@ -14111,7 +14020,16 @@ version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70"
dependencies = [
"zstd-safe",
"zstd-safe 8.2.0",
]
[[package]]
name = "zstd-safe"
version = "7.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882"
dependencies = [
"zstd-sys",
]
[[package]]
@@ -14129,7 +14047,6 @@ version = "2.1.2+zstd.1.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3"
dependencies = [
"bindgen",
"cc",
"pkg-config",
]
+4 -7
View File
@@ -66,7 +66,6 @@ members = [
"crates/io-metrics", # I/O metrics collection for performance analysis
"crates/io-core", # Buffered Bytes, mmap-then-copy, and aligned pread I/O helpers
"crates/zip", # ZIP file handling and compression
"crates/zstd-compat", # Fixed codec for dependencies requiring zstd 0.13
]
resolver = "3"
@@ -261,7 +260,7 @@ aws-smithy-http-client = { default-features = false, version = "1.5.1" }
aws-smithy-runtime-api = { version = "1.19.1" }
aws-smithy-types = { version = "1.9.0" }
base64-simd = "0.8.0"
brotli = "9.1.0"
brotli = "9.0.0"
clap = { version = "4.6.7" }
const-str = { version = "1.1.0" }
convert_case = "0.12.0"
@@ -303,7 +302,6 @@ num_cpus = { version = "1.17.0" }
nvml-wrapper = "0.13.0"
object = { version = "=0.40.0", default-features = false }
parking_lot = "0.13.0"
same-file = "1.0.6"
path-absolutize = "4.0.1"
percent-encoding = "2.3.2"
# Server-side QR rendering for TOTP enrollment, so neither the console nor the
@@ -356,7 +354,7 @@ winapi-util = "0.1.11"
windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.19" }
zip = "9.1.2"
zip = "9.1.0"
zstd = "0.14.3"
# Observability and Metrics
@@ -377,7 +375,7 @@ libunftp = { version = "0.23.1" }
unftp-core = "0.1.0"
suppaftp = { version = "12.2.1" }
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.65.0" }
russh = { version = "0.64.1" }
russh-sftp = "3.0.1"
# WebDAV
@@ -386,7 +384,7 @@ dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.7.2" }
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
hotpath = { version = "0.29.2", default-features = false }
hotpath = { version = "0.29.1", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.49" }
@@ -426,7 +424,6 @@ strip = "none"
# Pin upstream fixes until compatible crates.io releases are available.
[patch.crates-io]
zstd = { path = "crates/zstd-compat" }
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
-1
View File
@@ -146,7 +146,6 @@ zip.workspace = true
clap = { workspace = true, features = ["derive", "env"] }
[dev-dependencies]
same-file.workspace = true
aws-smithy-http-client = { workspace = true, default-features = false, features = ["test-util"] }
rustfs-heal.workspace = true
tempfile.workspace = true
@@ -20,14 +20,15 @@
//! offline before heal converges, a strict latest-listing quorum of 3 can only
//! ever observe 2 copies, so ListObjectsV2 silently dropped the object even
//! though GetObject (read quorum 2) still succeeded. Exposed by the flaky
//! "Mixed-version rolling upgrade from rc.2" CI lane (run 33478999853). The
//! listing now follows MinIO's rule: an entry needs `ceil(asked/2)` copies
//! among the online drives it asked, so three online drives need two copies
//! (see `SetDisks::list_path` in `crates/ecstore/src/store/list_objects.rs`).
//! "Mixed-version rolling upgrade from rc.2" CI lane (run 33478999853); the
//! product fix relaxes the listing's required object quorum by the number of
//! set drives the listing could not consult (see
//! `latest_listing_required_object_quorum` in
//! `crates/ecstore/src/store/list_objects.rs`).
#[cfg(test)]
mod tests {
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment, init_logging};
use crate::common::{RustFSTestClusterEnvironment, init_logging};
use aws_sdk_s3::Client;
use bytes::Bytes;
use std::collections::HashSet;
@@ -50,13 +51,13 @@ mod tests {
format!("degraded-object-{idx:02}")
}
async fn list_all_keys(client: &Client, bucket: &str) -> Result<HashSet<String>, Box<dyn Error + Send + Sync>> {
async fn list_all_keys(client: &Client) -> Result<HashSet<String>, Box<dyn Error + Send + Sync>> {
let mut keys = HashSet::new();
let mut continuation_token: Option<String> = None;
loop {
let response = client
.list_objects_v2()
.bucket(bucket)
.bucket(BUCKET)
.set_continuation_token(continuation_token.clone())
.send()
.await?;
@@ -146,7 +147,7 @@ mod tests {
let expected: HashSet<String> = (0..OBJECT_COUNT).map(object_key).collect();
let deadline = Instant::now() + LISTING_DEADLINE;
let listed = loop {
let listed = match list_all_keys(&client, BUCKET).await {
let listed = match list_all_keys(&client).await {
Ok(keys) => keys,
Err(error) if Instant::now() < deadline => {
info!("retrying degraded listing: {error}");
@@ -170,97 +171,4 @@ mod tests {
Ok(())
}
/// 4-node x 4-drive cluster (one 16-drive set), #8828 shape:
/// 1. PUT objects while the storage class is EC:8, so each is 8+8.
/// 2. Restart with the EC:4 default, as after a storage-class change.
/// 3. Stop two nodes (8 drives). Each object keeps eight data shards, so
/// GET succeeds and ListObjectsV2 must list every key: the listing
/// quorum is half of the eight drives it can ask.
/// 4. Stop a third node. Four online drives are below the set's read
/// quorum, so LIST must report 503 instead of a partial page.
#[tokio::test]
async fn ec8_objects_stay_listable_with_half_of_an_ec4_set_offline() -> TestResult {
const EC8_BUCKET: &str = "degraded-listing-ec8";
init_logging();
let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, 4)).await?;
cluster.set_env("RUSTFS_HEAL_ENABLED", "false");
cluster.set_env("RUSTFS_SCANNER_ENABLED", "false");
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:8");
cluster.start().await?;
cluster.create_test_bucket(EC8_BUCKET).await?;
let client = cluster.create_s3_client(0)?;
for idx in 0..OBJECT_COUNT {
let key = object_key(idx);
client
.put_object()
.bucket(EC8_BUCKET)
.key(&key)
.body(Bytes::from(format!("ec8 listing payload {idx}")).into())
.send()
.await
.map_err(|error| format!("EC:8 PUT for {key} failed: {error}"))?;
}
info!("restarting with the EC:4 default storage class");
cluster.stop();
// A later entry for the same variable wins when the node process starts.
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4");
cluster.start().await?;
let client = cluster.create_s3_client(0)?;
info!("stopping nodes 2 and 3 (eight of sixteen drives)");
cluster.stop_node(2)?;
cluster.stop_node(3)?;
for idx in 0..OBJECT_COUNT {
let key = object_key(idx);
let deadline = Instant::now() + GET_RETRY_DEADLINE;
loop {
match client.get_object().bucket(EC8_BUCKET).key(&key).send().await {
Ok(response) => {
let body = response.body.collect().await?.into_bytes();
assert_eq!(body.as_ref(), format!("ec8 listing payload {idx}").as_bytes());
break;
}
Err(error) if Instant::now() < deadline => {
info!("retrying EC:8 GET for {key}: {error}");
tokio::time::sleep(Duration::from_millis(500)).await;
}
Err(error) => return Err(format!("EC:8 object {key} failed GET with eight drives: {error}").into()),
}
}
}
let expected: HashSet<String> = (0..OBJECT_COUNT).map(object_key).collect();
let deadline = Instant::now() + LISTING_DEADLINE;
loop {
match list_all_keys(&client, EC8_BUCKET).await {
Ok(listed) if listed == expected => break,
Ok(listed) => assert!(
Instant::now() < deadline,
"EC:8 objects readable with eight drives stayed missing from ListObjectsV2: listed={listed:?}"
),
Err(error) => assert!(
Instant::now() < deadline,
"ListObjectsV2 kept failing with eight of sixteen drives online: {error}"
),
}
tokio::time::sleep(Duration::from_millis(500)).await;
}
info!("stopping node 1 (four of sixteen drives remain)");
cluster.stop_node(1)?;
let deadline = Instant::now() + LISTING_DEADLINE;
loop {
match client.list_objects_v2().bucket(EC8_BUCKET).send().await {
Err(error) if error.raw_response().is_some_and(|response| response.status().as_u16() == 503) => break,
Err(error) => assert!(Instant::now() < deadline, "below the set read quorum LIST must report 503, got {error}"),
Ok(response) => panic!("below the set read quorum LIST must fail, but listed {} keys", response.contents().len()),
}
tokio::time::sleep(Duration::from_millis(500)).await;
}
Ok(())
}
}
+3 -132
View File
@@ -13,20 +13,18 @@
// limitations under the License.
use super::harness::{
DistCluster, DistLayout, TestResult, assert_object_bytes, cluster_admin_ok, get_object_bytes, payload_for, put_object,
retrying_get_equals, retrying_put, s3_probe_client, s3_probe_error_code, unique_bucket, wait_for_ready, wait_until,
DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, retrying_get_equals, unique_bucket,
wait_for_ready, wait_until,
};
use crate::chaos::{census_object_version_on_disk, signed_admin_post};
use crate::common::{build_test_s3_config, init_logging};
use crate::fault_proxy::FaultMode;
use aws_sdk_s3::Client;
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::operation::{get_object::GetObjectError, put_object::PutObjectError};
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::{Barrier, mpsc};
use tokio::time::{Instant, timeout};
use tokio::time::timeout;
#[tokio::test]
async fn kill_and_restart_node_preserves_objects() -> TestResult {
@@ -46,133 +44,6 @@ async fn kill_and_restart_node_preserves_objects() -> TestResult {
Ok(())
}
#[tokio::test]
async fn ec4_partial_node_return_restores_read_write_quorum() -> TestResult {
init_logging();
let mut dist = DistCluster::start_with_env(
DistLayout::FourByFour,
&[
("RUSTFS_ERASURE_SET_DRIVE_COUNT", "16"),
("RUSTFS_STORAGE_CLASS_STANDARD", "EC:4"),
],
)
.await?;
let storage = cluster_admin_ok(&dist.cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
let storage: serde_json::Value = serde_json::from_str(&storage)?;
let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
assert_eq!(storage.disks.len(), 16);
assert_eq!(storage.backend.total_sets, vec![1]);
assert_eq!(storage.backend.drives_per_set, vec![16]);
assert_eq!(storage.backend.standard_sc_parity, Some(4));
let bucket = unique_bucket("partial-ec4");
dist.create_bucket(&bucket).await?;
let client = s3_probe_client(&dist.client(0)?);
let baseline = payload_for("partial-recovery/baseline", 8 * 1024 * 1024);
put_object(&client, &bucket, "baseline.bin", baseline.clone()).await?;
wait_until(
Duration::from_secs(30),
|| async {
for node in &dist.cluster.nodes {
for directory in &node.data_dirs {
let census = census_object_version_on_disk(std::path::Path::new(directory), &bucket, "baseline.bin", None)?;
if !census.is_complete() {
return Ok(false);
}
assert_eq!(census.data_blocks, Some(12), "the object must require twelve data shards");
assert_eq!(census.parity_blocks, Some(4), "the object must have four parity shards");
assert!(!census.expected_part_numbers.is_empty(), "the baseline must use physical shards");
}
}
Ok(true)
},
"the baseline is complete on all sixteen EC:4 drives",
)
.await?;
dist.cluster.stop_node(2)?;
dist.cluster.stop_node(3)?;
let read_error = timeout(Duration::from_secs(60), get_object_bytes(&client, &bucket, "baseline.bin"))
.await?
.expect_err("eight unavailable drives must prevent the baseline GET");
assert!(
matches!(
s3_probe_error_code(read_error.as_ref()),
Some("SlowDown" | "SlowDownRead" | "ServiceUnavailable")
),
"quorum loss must produce an availability error: {read_error:?}"
);
assert_eq!(
read_error
.downcast_ref::<SdkError<GetObjectError>>()
.and_then(SdkError::raw_response)
.map(|response| response.status().as_u16()),
Some(503),
"read quorum loss must remain a service availability failure"
);
let write_error = timeout(Duration::from_secs(60), put_object(&client, &bucket, "unavailable.bin", baseline.clone()))
.await?
.expect_err("eight unavailable drives must prevent a successful PUT");
assert!(
matches!(s3_probe_error_code(write_error.as_ref()), Some("SlowDown" | "ServiceUnavailable")),
"quorum loss must produce an availability error: {write_error:?}"
);
assert_eq!(
write_error
.downcast_ref::<SdkError<PutObjectError>>()
.and_then(SdkError::raw_response)
.map(|response| response.status().as_u16()),
Some(503),
"write quorum loss must remain a service availability failure"
);
// Recover only four drives. The remaining node must stay stopped while
// every live coordinator serves old data and acknowledges new writes.
dist.cluster.start_node_process(2).await?;
let recovery_deadline = Instant::now() + Duration::from_secs(90);
let mut recovered_objects = Vec::new();
for node in 0..3 {
assert!(dist.cluster.nodes[3].process.is_none(), "the fourth node must remain unavailable");
let client = s3_probe_client(&dist.client(node)?);
retrying_get_equals(
&client,
&bucket,
"baseline.bin",
&baseline,
recovery_deadline.saturating_duration_since(Instant::now()),
)
.await?;
let key = format!("partial-{node}.bin");
let body = payload_for(&key, 1024 * 1024);
retrying_put(
&client,
&bucket,
&key,
body.clone(),
recovery_deadline.saturating_duration_since(Instant::now()),
)
.await?;
recovered_objects.push((key, body));
}
for node in 0..3 {
assert!(dist.cluster.nodes[3].process.is_none(), "reads must complete before all drives return");
let client = s3_probe_client(&dist.client(node)?);
assert_object_bytes(&client, &bucket, "baseline.bin", &baseline).await?;
for (key, body) in &recovered_objects {
assert_object_bytes(&client, &bucket, key, body).await?;
}
}
dist.cluster.start_node(3).await?;
wait_for_ready(&dist.cluster).await?;
let client = s3_probe_client(&dist.client(3)?);
assert_object_bytes(&client, &bucket, "baseline.bin", &baseline).await?;
for (key, body) in &recovered_objects {
assert_object_bytes(&client, &bucket, key, body).await?;
}
Ok(())
}
#[tokio::test]
async fn full_cluster_restart_preserves_objects() -> TestResult {
init_logging();
+5 -66
View File
@@ -1181,16 +1181,11 @@ pub(crate) async fn retrying_get_equals(
)
.into()),
Err(error) => {
let retryable = match s3_probe_error_code(error.as_ref()) {
Some("SlowDown" | "ServiceUnavailable") => true,
// GET preserves this compatibility code for a lost data quorum.
Some("SlowDownRead") => error
.downcast_ref::<SdkError<GetObjectError>>()
.and_then(SdkError::raw_response)
.is_some_and(|response| response.status().as_u16() == 503),
_ => false,
};
if retryable { Ok(false) } else { Err(error) }
if matches!(s3_probe_error_code(error.as_ref()), Some("SlowDown" | "ServiceUnavailable")) {
Ok(false)
} else {
Err(error)
}
}
}
},
@@ -1329,62 +1324,6 @@ mod retry_tests {
Ok(())
}
#[tokio::test(start_paused = true)]
async fn retrying_get_allows_read_quorum_recovery() -> TestResult {
use aws_sdk_s3::config::Region;
use aws_smithy_http_client::test_util::{ReplayEvent, StaticReplayClient};
use aws_smithy_types::body::SdkBody;
let endpoint = "http://read-quorum.example.invalid";
let event = |status: u16, body: &'static str| {
ReplayEvent::new(
http::Request::builder()
.method(Method::GET)
.uri(format!("{endpoint}/retry-probe/key?x-id=GetObject"))
.body(SdkBody::empty())
.expect("build replay request"),
http::Response::builder()
.status(status)
.header("content-type", "application/xml")
.header("content-length", body.len())
.body(SdkBody::from(body))
.expect("build replay response"),
)
};
for status in [503, 403, 500] {
let transport = StaticReplayClient::new(vec![
event(status, "<Error><Code>SlowDownRead</Code></Error>"),
event(200, "expected bytes"),
]);
let client = Client::from_conf(
aws_sdk_s3::Config::builder()
.behavior_version_latest()
.credentials_provider(Credentials::new(FAKE_ACCESS_KEY, FAKE_SECRET_KEY, None, None, "read-quorum-test"))
.region(Region::new("us-east-1"))
.endpoint_url(endpoint)
.force_path_style(true)
.retry_config(RetryConfig::disabled())
.http_client(transport.clone())
.build(),
);
let result = retrying_get_equals(&client, "retry-probe", "key", b"expected bytes", Duration::from_secs(5)).await;
if status == 503 {
result?;
assert_eq!(
transport.actual_requests().count(),
2,
"the probe must retry the read-quorum error exactly once"
);
transport.assert_requests_match(&[]);
} else {
let error = result.expect_err("the compatibility code must not hide a non-503 failure");
assert_eq!(s3_probe_error_code(error.as_ref()), Some("SlowDownRead"));
assert_eq!(transport.actual_requests().count(), 1, "non-availability errors must fail immediately");
}
}
Ok(())
}
#[tokio::test]
async fn object_polls_distinguish_committed_and_replicated_keys() -> TestResult {
let target = FakeS3Target::start().await?;
-1
View File
@@ -31,7 +31,6 @@ mod multipart_fence_acceptance_test;
mod multipart_rpc_delay_test;
mod object_lock_test;
mod observability_test;
mod publication_log;
mod replication_delete_marker_test;
mod replication_quota_test;
mod s3_basic_test;
@@ -1638,191 +1638,6 @@ async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation(
"refused late packets must not recreate staging"
);
}
late_part_packets_preserve_retired_and_fresh_uploads().await?;
Ok(())
}
async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
for (method, settles) in [("PreparePartTransaction", false), ("SettlePartTransaction", true)] {
for recreate_bucket in [false, true] {
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
let key = "late-part-transaction";
let previous = payload_for("committed-before-late-part", 4096);
clients[0]
.put_object()
.bucket(&bucket)
.key(key)
.body(ByteStream::from(previous.clone()))
.send()
.await?;
let upload = clients[0].create_multipart_upload().bucket(&bucket).key(key).send().await?;
let upload_id = upload.upload_id().ok_or("missing original upload ID")?;
clients[0]
.upload_part()
.bucket(&bucket)
.key(key)
.upload_id(upload_id)
.part_number(1)
.body(ByteStream::from(vec![0x41; 9 * 1024 * 1024]))
.send()
.await?;
let path = upload_path_scope(&bucket, key, upload_id)?;
let hash = sha256_hex(format!("{bucket}/{key}").as_bytes());
let roots = cluster
.nodes
.iter()
.flat_map(|node| &node.data_dirs)
.map(|root| Path::new(root).join(RUSTFS_META_MULTIPART_BUCKET).join(&path))
.collect::<Vec<_>>();
assert_eq!(roots.len(), 4);
let original = multipart_sources(&cluster, Some(&hash))?;
assert_eq!(original.len(), 4);
assert!(
original.values().all(|source| source
.files
.keys()
.any(|path| path.file_name().and_then(|name| name.to_str()) == Some("part.1"))),
"all selected peers originally own the acknowledged part"
);
let scope = upload_scope(upload_id)?;
// Retain only the original attempt; recovery of the same upload
// must be able to issue fresh settlement requests through the gate.
let pauses = proxies[1..3]
.iter()
.map(|proxy| proxy.pause_late(&[method], 1, &scope))
.collect::<Result<Vec<_>, _>>()?;
let replacement = clients[0]
.upload_part()
.bucket(&bucket)
.key(key)
.upload_id(upload_id)
.part_number(1)
.body(ByteStream::from(vec![0x62; 8 * 1024 * 1024]));
let caller = tokio::spawn(async move { replacement.send().await });
wait_for_faults(&pauses, &[method]).await?;
let failure = tokio::time::timeout(DEADLINE, caller)
.await??
.expect_err("two retained peers cannot acknowledge part replacement");
assert_unavailable(&failure, "storage_write_quorum", None);
assert!(pauses.iter().all(|pause| pause.hits() == 1 && pause.responses() == 0));
let expected_actions = if settles { (1, 0) } else { (0, 0) };
assert!(
pauses.iter().all(|pause| pause.retained_settle_actions() == expected_actions),
"settlement capture must be a real Commit, not Rollback"
);
abort_after_fault(&clients[0], &bucket, key, upload_id, false).await?;
assert!(
roots.iter().all(|root| !root.exists()),
"ordinary Abort physically retires all original roots"
);
assert!(multipart_sources(&cluster, Some(&hash))?.is_empty());
for client in &clients {
assert_object_bytes_after_fault(client, &bucket, key, &previous).await?;
}
let protected = if recreate_bucket {
clients[0].delete_object().bucket(&bucket).key(key).send().await?;
clients[0].delete_bucket().bucket(&bucket).send().await?;
clients[1].create_bucket().bucket(&bucket).send().await?;
let fresh = payload_for("recreated-before-late-part", 4096);
clients[1]
.put_object()
.bucket(&bucket)
.key(key)
.body(ByteStream::from(fresh.clone()))
.send()
.await?;
fresh
} else {
previous
};
let fresh = clients[2].create_multipart_upload().bucket(&bucket).key(key).send().await?;
let fresh_id = fresh.upload_id().ok_or("missing fresh upload ID")?;
let fresh_body = payload_for("fresh-active-part", 4096);
let fresh_part = clients[2]
.upload_part()
.bucket(&bucket)
.key(key)
.upload_id(fresh_id)
.part_number(1)
.body(ByteStream::from(fresh_body.clone()))
.send()
.await?;
let before = multipart_sources(&cluster, Some(&hash))?;
assert_eq!(before.len(), 4, "the new upload exists on every original participant");
for pause in &pauses {
pause.release();
}
wait_until(
DEADLINE,
|| async {
Ok(pauses.iter().all(|pause| {
pause.responses() == pause.hits()
&& if settles {
pause.successful_mutation_replies() == pause.hits() && pause.refused_mutations() == 0
} else {
pause.refused_mutations() == pause.hits() && pause.successful_mutation_replies() == 0
}
}))
},
"every original part packet returns its typed authenticated terminal outcome",
)
.await?;
assert!(roots.iter().all(|root| !root.exists()), "late part packets cannot recreate retired roots");
assert!(
multipart_sources(&cluster, Some(&hash))? == before,
"late packets cannot change a fresh upload's shard or control bytes"
);
for client in &clients {
assert_object_bytes_after_fault(client, &bucket, key, &protected).await?;
let listed = client
.list_parts()
.bucket(&bucket)
.key(key)
.upload_id(fresh_id)
.send()
.await?;
assert_eq!(listed.parts().len(), 1);
assert_eq!(listed.parts()[0].e_tag(), fresh_part.e_tag());
let missing = client
.list_parts()
.bucket(&bucket)
.key(key)
.upload_id(upload_id)
.send()
.await
.expect_err("old upload remains retired");
assert_eq!(missing.raw_response().expect("old upload service response").status().as_u16(), 404);
assert_eq!(missing.as_service_error().and_then(|error| error.meta().code()), Some("NoSuchUpload"));
}
clients[2]
.complete_multipart_upload()
.bucket(&bucket)
.key(key)
.upload_id(fresh_id)
.multipart_upload(
CompletedMultipartUpload::builder()
.parts(
CompletedPart::builder()
.part_number(1)
.e_tag(fresh_part.e_tag().ok_or("missing fresh part ETag")?)
.build(),
)
.build(),
)
.send()
.await?;
for client in &clients {
assert_object_bytes_after_fault(client, &bucket, key, &fresh_body).await?;
}
assert!(multipart_sources(&cluster, Some(&hash))?.is_empty());
assert_no_uploads_after_fault(&clients, &bucket).await?;
eprintln!(
"MPU late part: method={method}, recreate_bucket={recreate_bucket}, retired_roots=4, delivered=2, refused={}, successful_replies={}, fresh_sources_unchanged=4, verified_readers=4",
pauses.iter().map(RpcPause::refused_mutations).sum::<usize>(),
pauses.iter().map(RpcPause::successful_mutation_replies).sum::<usize>()
);
}
}
Ok(())
}
@@ -1,413 +0,0 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! Follow the same append-only log files without rescanning previous JSON rows.
use same_file::Handle;
use serde::{Deserialize, Deserializer};
use serde_json::Value;
use std::fs::File;
use std::io::{self, Read};
use std::path::PathBuf;
// Captured RPC spans already produce valid rows above 2 MiB. Keep complete
// rows bounded independently from the much smaller selected-event history.
const MAX_LINE_BYTES: usize = 4 << 20;
const MAX_EVENTS: usize = 128;
const MAX_EVENT_BYTES: usize = 1 << 20;
fn invalid(reason: &'static str) -> io::Error {
io::Error::new(io::ErrorKind::InvalidData, reason)
}
#[derive(Default, Deserialize)]
struct Selector {
#[serde(default)]
event: Value,
#[serde(default)]
op: Value,
#[serde(default)]
bucket: Value,
#[serde(default)]
object: Value,
}
impl Selector {
fn matches(&self, bucket: &str, object: &str) -> bool {
self.event.as_str() == Some("object_namespace_lock")
&& self.op.as_str() == Some("data_movement_publication")
&& self.bucket.as_str() == Some(bucket)
&& self.object.as_str() == Some(object)
}
}
#[derive(Deserialize)]
struct Probe {
#[serde(default)]
event: Value,
#[serde(default)]
op: Value,
#[serde(default)]
bucket: Value,
#[serde(default)]
object: Value,
// A present null is not a missing fields object. The old reader uses the
// root only when this key is absent, including for non-object JSON values.
#[serde(default, deserialize_with = "present_fields")]
fields: Option<Option<Selector>>,
}
fn present_fields<'de, D: Deserializer<'de>>(d: D) -> Result<Option<Option<Selector>>, D::Error> {
Option::<Selector>::deserialize(d).map(Some)
}
impl Probe {
fn matches(&self, bucket: &str, object: &str) -> bool {
match &self.fields {
Some(Some(fields)) => fields.matches(bucket, object),
Some(None) => false,
None => {
self.event.as_str() == Some("object_namespace_lock")
&& self.op.as_str() == Some("data_movement_publication")
&& self.bucket.as_str() == Some(bucket)
&& self.object.as_str() == Some(object)
}
}
}
}
fn select(line: &[u8], bucket: &str, object: &str, full_decodes: &mut u64) -> io::Result<Option<Value>> {
let text = std::str::from_utf8(line).map_err(|_| invalid("invalid UTF-8 in a completed log row"))?;
// Skip unknown fields without allocating a Value tree for every RPC trace.
// A shape/duplicate-field error falls back to the original Value parser;
// it must not discard valid escaped strings or last-key-wins JSON.
if let Ok(probe) = serde_json::from_str::<Probe>(text)
&& !probe.matches(bucket, object)
{
return Ok(None);
}
*full_decodes += 1;
let Ok(row) = serde_json::from_str::<Value>(text) else {
return Ok(None);
};
let fields = row.get("fields").unwrap_or(&row);
if fields.get("event").and_then(Value::as_str) != Some("object_namespace_lock")
|| fields.get("op").and_then(Value::as_str) != Some("data_movement_publication")
|| fields.get("bucket").and_then(Value::as_str) != Some(bucket)
|| fields.get("object").and_then(Value::as_str) != Some(object)
{
return Ok(None);
}
let mut event = fields.clone();
event["timestamp"] = row["timestamp"].clone();
Ok(Some(event))
}
struct Tail {
path: PathBuf,
handle: Handle,
offset: u64,
pending: Vec<u8>,
}
impl Tail {
fn open(path: PathBuf) -> io::Result<Self> {
let file = File::open(&path)?;
if !file.metadata()?.is_file() {
return Err(invalid("publication log is not a regular file"));
}
Ok(Self {
path,
handle: Handle::from_file(file)?,
offset: 0,
pending: Vec::new(),
})
}
fn check_identity(&self) -> io::Result<()> {
if Handle::from_path(&self.path)? != self.handle {
return Err(invalid("publication log file was replaced"));
}
Ok(())
}
fn poll(&mut self, bytes_read: &mut u64, mut accept: impl FnMut(&[u8]) -> io::Result<()>) -> io::Result<()> {
self.check_identity()?;
let end = self.handle.as_file().metadata()?.len();
if end < self.offset {
return Err(invalid("publication log file was truncated"));
}
// A fixed high-water snapshot bounds each poll even if writers append
// faster than the reader. Bytes after it remain for the next poll.
let mut chunk = [0_u8; 64 * 1024];
while self.offset < end {
let size = usize::try_from((end - self.offset).min(64 * 1024)).expect("a log chunk fits usize");
self.handle.as_file_mut().read_exact(&mut chunk[..size])?;
self.offset += u64::try_from(size).expect("a bounded log read fits u64");
*bytes_read += u64::try_from(size).expect("a bounded log read fits u64");
let appended_at = self.pending.len();
self.pending.extend_from_slice(&chunk[..size]);
let mut start = 0;
for end in self.pending[appended_at..]
.iter()
.enumerate()
.filter_map(|(i, byte)| (*byte == b'\n').then_some(appended_at + i + 1))
{
if end - start > MAX_LINE_BYTES {
return Err(invalid("publication log row exceeds its fixed bound"));
}
accept(&self.pending[start..end])?;
start = end;
}
self.pending.drain(..start);
if self.pending.len() > MAX_LINE_BYTES {
return Err(invalid("unfinished publication log row exceeds its fixed bound"));
}
}
self.check_identity()
}
}
pub(super) struct PublicationEvents {
tails: Vec<Tail>,
bucket: String,
object: String,
events: Vec<Value>,
per_file: Vec<Vec<Value>>,
event_count: usize,
event_bytes: usize,
bytes_read: u64,
full_decodes: u64,
failed: bool,
}
impl PublicationEvents {
pub(super) fn new(paths: Vec<PathBuf>, bucket: &str, object: &str) -> io::Result<Self> {
if paths.is_empty() || paths.len() > 4 {
return Err(invalid("publication capture requires one to four log files"));
}
let tails: Vec<Tail> = paths.into_iter().map(Tail::open).collect::<io::Result<_>>()?;
let mut reader = Self {
per_file: vec![Vec::new(); tails.len()],
tails,
bucket: bucket.to_owned(),
object: object.to_owned(),
events: Vec::new(),
event_bytes: 0,
event_count: 0,
bytes_read: 0,
full_decodes: 0,
failed: false,
};
// Prime before holding remote DELETE or starting a movement attempt.
// Retain earlier matching events too: the caller still applies its
// original timestamp boundary, and evidence keeps the complete history.
reader.poll()?;
Ok(reader)
}
pub(super) fn poll(&mut self) -> io::Result<&[Value]> {
if self.failed {
return Err(invalid("publication log reader was already invalidated"));
}
let result = (|| {
for (tail, history) in self.tails.iter_mut().zip(&mut self.per_file) {
tail.poll(&mut self.bytes_read, |line| {
if let Some(event) = select(line, &self.bucket, &self.object, &mut self.full_decodes)? {
let bytes = serde_json::to_vec(&event).map_err(io::Error::other)?.len();
if self.event_count >= MAX_EVENTS || bytes > MAX_EVENT_BYTES - self.event_bytes {
return Err(invalid("publication event history exceeds its fixed bound"));
}
self.event_bytes += bytes;
self.event_count += 1;
history.push(event);
}
Ok(())
})?;
}
Ok(())
})();
// Preserve the old file-then-row order, even when a later poll appends
// to an earlier file. Only bounded selected history is copied here.
self.events = self.per_file.iter().flatten().cloned().collect();
if let Err(error) = result {
self.failed = true;
return Err(error);
}
Ok(&self.events)
}
/// Diagnostics only: after a read error this history can be incomplete.
pub(super) fn cached(&self) -> &[Value] {
&self.events
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
use std::io::Write;
fn event(id: &str) -> Value {
json!({"timestamp":"2026-10-26T00:00:00Z","event":"object_namespace_lock","op":"data_movement_publication","bucket":"bucket","object":"object","lock_attempt_id":id,"state":"acquiring","extra":{"kept":[1,2]}})
}
fn append(path: &std::path::Path, raw: &[u8]) {
std::fs::OpenOptions::new()
.append(true)
.open(path)
.expect("open append log")
.write_all(raw)
.expect("append log bytes");
}
fn legacy(raw: &[u8]) -> Vec<Value> {
std::str::from_utf8(raw)
.expect("fixture UTF-8")
.lines()
.filter_map(|line| {
let row: Value = serde_json::from_str(line).ok()?;
let fields = row.get("fields").unwrap_or(&row);
if fields.get("event").and_then(Value::as_str) == Some("object_namespace_lock")
&& fields.get("op").and_then(Value::as_str) == Some("data_movement_publication")
&& fields.get("bucket").and_then(Value::as_str) == Some("bucket")
&& fields.get("object").and_then(Value::as_str) == Some("object")
{
let mut result = fields.clone();
result["timestamp"] = row["timestamp"].clone();
Some(result)
} else {
None
}
})
.collect()
}
#[test]
fn appended_events_keep_full_fields_without_history_rescans() {
let file = tempfile::NamedTempFile::new().expect("log file");
let mut raw = (json!({"message":"unrelated trace","payload":"x".repeat(4096)}).to_string() + "\n")
.repeat(64)
.into_bytes();
raw.extend_from_slice(
format!("{}\n", json!({"event":"object_namespace_lock","op":"get_object","bucket":"bucket","object":"object","spans":"x".repeat(2401832)})).as_bytes(),
);
raw.extend_from_slice(format!("{}\n", event("first")).as_bytes());
std::fs::write(file.path(), &raw).expect("seed log");
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("prime log");
assert_eq!(reader.cached(), legacy(&raw));
assert_eq!(reader.full_decodes, 1);
let bytes = reader.bytes_read;
assert_eq!(reader.poll().expect("unchanged poll"), legacy(&raw));
assert_eq!(reader.bytes_read, bytes);
let next = format!("{}\n", event("second"));
append(file.path(), next.as_bytes());
raw.extend_from_slice(next.as_bytes());
assert_eq!(reader.poll().expect("new rows"), legacy(&raw));
assert_eq!(reader.bytes_read, u64::try_from(raw.len()).expect("fixture length"));
assert_eq!(reader.full_decodes, 2);
}
#[test]
fn later_appends_preserve_the_old_file_then_row_order() {
let first = tempfile::NamedTempFile::new().expect("first log");
let second = tempfile::NamedTempFile::new().expect("second log");
std::fs::write(first.path(), format!("{}\n", event("a1"))).expect("seed first");
std::fs::write(second.path(), format!("{}\n", event("b1"))).expect("seed second");
let mut reader = PublicationEvents::new(vec![first.path().to_owned(), second.path().to_owned()], "bucket", "object")
.expect("prime both");
append(first.path(), format!("{}\n", event("a2")).as_bytes());
let mut expected = legacy(&std::fs::read(first.path()).expect("first bytes"));
expected.extend(legacy(&std::fs::read(second.path()).expect("second bytes")));
assert_eq!(reader.poll().expect("append first after second was read"), expected);
}
#[test]
fn root_nested_escaped_and_duplicate_fields_match_the_old_reader() {
let file = tempfile::NamedTempFile::new().expect("log file");
let root = event("root");
let mut nested = json!({"timestamp":root["timestamp"],"fields":root});
nested["fields"]["extra"] = json!({"unicode":"é"});
let null = json!({"fields":null,"event":"object_namespace_lock","op":"data_movement_publication","bucket":"bucket","object":"object"});
let escaped = root
.to_string()
.replace("object_namespace_lock", "\\u006fbject_namespace_lock");
let duplicate = root.to_string().replacen("{", "{\"event\":\"earlier-value\",", 1);
let raw = format!("{root}\n{nested}\n{null}\n{escaped}\n{duplicate}\nnot json\n{{\"fields\":[]}}\n");
std::fs::write(file.path(), &raw).expect("mixed log");
let reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("probe and fallback");
assert_eq!(reader.cached(), legacy(raw.as_bytes()));
assert_eq!(reader.cached().len(), 4);
}
#[test]
fn incomplete_rows_and_split_utf8_wait_for_the_newline() {
let file = tempfile::NamedTempFile::new().expect("log file");
let mut row = event("partial");
row["extra"] = json!("é");
let raw = format!("{row}\n").into_bytes();
let cut = raw.iter().position(|b| *b == 0xc3).expect("UTF-8 prefix") + 1;
std::fs::write(file.path(), &raw[..cut]).expect("partial code point");
let mut reader =
PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("incomplete row is pending");
assert!(reader.cached().is_empty());
append(file.path(), &raw[cut..raw.len() - 1]);
assert!(reader.poll().expect("complete JSON without delimiter").is_empty());
append(file.path(), b"\n");
assert_eq!(reader.poll().expect("published row"), legacy(&raw));
}
#[test]
fn truncation_is_sticky_even_after_the_file_grows_again() {
let file = tempfile::NamedTempFile::new().expect("log file");
std::fs::write(file.path(), format!("{}\n", event("first"))).expect("seed log");
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("prime log");
file.as_file().set_len(0).expect("truncate");
assert!(reader.poll().is_err());
append(file.path(), format!("{}\n", event("longer replacement")).as_bytes());
assert!(reader.poll().is_err());
assert_eq!(reader.cached().len(), 1);
}
#[cfg(unix)]
#[test]
fn a_replaced_path_does_not_switch_to_the_new_log() {
let dir = tempfile::tempdir().expect("log dir");
let path = dir.path().join("node.log");
std::fs::write(&path, b"").expect("log file");
let mut reader = PublicationEvents::new(vec![path.clone()], "bucket", "object").expect("open log");
std::fs::rename(&path, dir.path().join("retired.log")).expect("replace log");
std::fs::write(&path, format!("{}\n", event("other-generation"))).expect("replacement");
assert!(reader.poll().is_err());
assert!(reader.cached().is_empty());
}
#[test]
fn invalid_utf8_and_oversized_or_excessive_records_fail_closed() {
let file = tempfile::NamedTempFile::new().expect("log file");
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("empty log");
append(file.path(), b"\xff\n");
assert!(reader.poll().is_err());
let file = tempfile::NamedTempFile::new().expect("log file");
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("empty log");
append(file.path(), &vec![b'x'; MAX_LINE_BYTES + 1]);
assert!(reader.poll().is_err());
let file = tempfile::NamedTempFile::new().expect("log file");
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("empty log");
for _ in 0..=MAX_EVENTS {
append(file.path(), format!("{}\n", event("attempt")).as_bytes());
}
assert!(reader.poll().is_err());
assert_eq!(reader.cached().len(), MAX_EVENTS);
assert!(reader.poll().is_err());
let file = tempfile::NamedTempFile::new().expect("log file");
let mut reader = PublicationEvents::new(vec![file.path().to_owned()], "bucket", "object").expect("empty log");
let mut large = event("large");
large["extra"] = json!("x".repeat(600 * 1024));
for _ in 0..2 {
append(file.path(), format!("{large}\n").as_bytes());
}
assert!(reader.poll().is_err());
assert_eq!(reader.cached().len(), 1);
assert!(reader.poll().is_err());
}
}
@@ -21,7 +21,6 @@ use super::harness::{
rebalance_status_json, sha256_hex, start_decommission, start_rebalance, unique_bucket, wait_for_decommission_complete,
wait_for_rebalance_complete,
};
use super::publication_log::PublicationEvents;
use super::scanner_heal_topology_fixture::{
MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL, STAGE_AFTER_RESPONSIBILITY_PERSISTED, STAGE_BEFORE_SOURCE_CLEANUP,
TopologyRun, assert_bucket_durability_strict, assert_process_durability_strict, require_test_hooks,
@@ -175,6 +174,29 @@ async fn transition_version(
}
}
fn lock_events(run: &TopologyRun, bucket: &str) -> TestResult<Vec<Value>> {
let mut events = Vec::new();
for path in run.node_log_paths() {
let log = std::fs::read_to_string(path)?;
for line in log.lines() {
let Ok(row) = serde_json::from_str::<Value>(line) else {
continue;
};
let fields = row.get("fields").unwrap_or(&row);
if fields.get("event").and_then(Value::as_str) == Some("object_namespace_lock")
&& fields.get("op").and_then(Value::as_str) == Some("data_movement_publication")
&& fields.get("bucket").and_then(Value::as_str) == Some(bucket)
&& fields.get("object").and_then(Value::as_str) == Some(KEY)
{
let mut event = fields.clone();
event["timestamp"] = row["timestamp"].clone();
events.push(event);
}
}
}
Ok(events)
}
fn event_time(event: &Value) -> TestResult<OffsetDateTime> {
Ok(OffsetDateTime::parse(
event["timestamp"].as_str().ok_or("lock event omitted its timestamp")?,
@@ -182,10 +204,10 @@ fn event_time(event: &Value) -> TestResult<OffsetDateTime> {
)?)
}
async fn pending_publication(events: &mut PublicationEvents, started: OffsetDateTime) -> TestResult<Value> {
async fn pending_publication(run: &TopologyRun, bucket: &str, started: OffsetDateTime) -> TestResult<Value> {
loop {
let observed = events.poll()?;
for event in observed {
let events = lock_events(run, bucket)?;
for event in &events {
if event_time(event)? < started {
continue;
}
@@ -194,7 +216,7 @@ async fn pending_publication(events: &mut PublicationEvents, started: OffsetDate
"mover acquired its publication lock while remote DELETE was held"
);
if event["state"] == "acquiring"
&& !observed
&& !events
.iter()
.any(|later| later["lock_attempt_id"] == event["lock_attempt_id"] && later["state"] != "acquiring")
{
@@ -206,14 +228,14 @@ async fn pending_publication(events: &mut PublicationEvents, started: OffsetDate
}
}
async fn acquired_publication(events: &mut PublicationEvents, pending: &Value) -> TestResult<Value> {
async fn acquired_publication(run: &TopologyRun, bucket: &str, pending: &Value) -> TestResult<Value> {
loop {
for event in events.poll()? {
for event in lock_events(run, bucket)? {
if event["lock_attempt_id"] != pending["lock_attempt_id"] {
continue;
}
match event["state"].as_str() {
Some("acquired") => return Ok(event.clone()),
Some("acquired") => return Ok(event),
Some("failed" | "cancelled") => return Err(format!("held publication attempt did not acquire: {event}").into()),
_ => {}
}
@@ -381,7 +403,6 @@ async fn tier_delete_during_movement(decommission: bool) -> TestResult {
// started a worker. Install a separate release nonce before holding DELETE.
let cleanup_barrier = run.movement_barrier.next_phase();
cleanup_barrier.arm_movement(operation, &bucket, KEY, STAGE_BEFORE_SOURCE_CLEANUP)?;
let mut publication_events = PublicationEvents::new(run.node_log_paths(), &bucket, KEY)?;
let mut delete_gate = target.hold_delete_object(&remote_bucket, &old_remote.key);
// The public DELETE persists a free owner and enqueues its remote cleanup.
let client = run.dist.client(0)?;
@@ -449,11 +470,11 @@ async fn tier_delete_during_movement(decommission: bool) -> TestResult {
})?;
drop(decommission_start);
let movement_acknowledged_at = OffsetDateTime::now_utc();
let pending = timeout_at(overlap_deadline, pending_publication(&mut publication_events, movement_started))
let pending = timeout_at(overlap_deadline, pending_publication(&run, &bucket, movement_started))
.await
.map_err(|error| format!(
"G-TIER {operation} publication pending timed out within DELETE overlap: {error}; startup={startup}; lock_events={:?}; cleanup_barrier={:?}",
publication_events.cached(), cleanup_barrier.reached_observations()
lock_events(&run, &bucket), cleanup_barrier.reached_observations()
))?
.map_err(|error| format!("G-TIER {operation} publication pending failed within DELETE overlap: {error}"))?;
// Positive gate entry, exact physical ownership and the pending lock attempt
@@ -463,11 +484,11 @@ async fn tier_delete_during_movement(decommission: bool) -> TestResult {
let released_at = OffsetDateTime::now_utc();
let delete_overlap_elapsed_ms = overlap_started.elapsed().as_millis();
drop(delete_gate);
let acquired = timeout(DELETE_OVERLAP_TIMEOUT, acquired_publication(&mut publication_events, &pending))
let acquired = timeout(DELETE_OVERLAP_TIMEOUT, acquired_publication(&run, &bucket, &pending))
.await
.map_err(|error| format!(
"G-TIER {operation} publication acquired after DELETE release timed out: {error}; pending={pending}; lock_events={:?}; cleanup_barrier={:?}",
publication_events.cached(), cleanup_barrier.reached_observations()
lock_events(&run, &bucket), cleanup_barrier.reached_observations()
))?
.map_err(|error| format!("G-TIER {operation} publication acquired after DELETE release failed: {error}"))?;
assert!(event_time(&pending)? >= movement_started && event_time(&pending)? < released_at);
@@ -475,7 +496,6 @@ async fn tier_delete_during_movement(decommission: bool) -> TestResult {
event_time(&acquired)? >= released_at,
"the same publication attempt must acquire after DELETE is released"
);
drop(publication_events);
let reached = cleanup_barrier
.wait_reached_during(
&run.dist.cluster,
@@ -1129,14 +1129,12 @@ mod tests {
/// and the marker plus the cursor tag.
///
/// Bug Reference: backlog#1047
/// Earlier releases appended a `[rustfs_cache:...]` cursor tag to the last
/// returned key inside the V2 continuation token. The orchestration layer
/// passed that raw string to `forward_past`, so any key whose byte after the
/// shared stem sorts below `[` (0x5B) — `.`, `/`, `-`, digits, uppercase —
/// was silently dropped on the next page: with keys `a`, `a.txt`, `zz` and
/// max_keys=1, page 2 returned `zz` and `a.txt` was never listed. The token
/// is now base64 of the plain key, and this test keeps guarding the
/// continuation boundary.
/// The V2 continuation token appends a `[rustfs_cache:...]` cursor tag to the
/// last returned key. The orchestration layer passed that raw string to
/// `forward_past`, so any key whose byte after the shared stem sorts below
/// `[` (0x5B) — `.`, `/`, `-`, digits, uppercase — was silently dropped on
/// the next page: with keys `a`, `a.txt`, `zz` and max_keys=1, page 2
/// returned `zz` and `a.txt` was never listed.
#[tokio::test]
async fn test_list_objects_v2_continuation_keeps_keys_after_marker_stem() {
init_logging();
+94 -201
View File
@@ -19,19 +19,6 @@ use http::{Method, StatusCode};
use tokio::time::{Duration, sleep, timeout};
use tracing::{debug, info};
/// A successful reporting response can still have unknown scanner usage.
fn reported_quota_usage(response: &serde_json::Value) -> std::io::Result<Option<u64>> {
let value = response
.get("current_usage")
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "quota response is missing current_usage"))?;
if value.is_null() {
return Ok(None);
}
value.as_u64().map(Some).ok_or_else(|| {
std::io::Error::new(std::io::ErrorKind::InvalidData, "quota current_usage is not an unsigned integer or null")
})
}
/// Test environment setup for quota tests
pub struct QuotaTestEnv {
pub env: RustFSTestEnvironment,
@@ -100,69 +87,35 @@ impl QuotaTestEnv {
}
}
async fn wait_for_usage_response(
&self,
method: Method,
path: &str,
body: Option<String>,
expected_usage: u64,
) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
let convergence = async {
loop {
let (status, response) = admin_request(
&self.env.url,
method.clone(),
path,
body.clone(),
&self.env.access_key,
&self.env.secret_key,
)
.await?;
if status.is_success() {
let response: serde_json::Value = serde_json::from_str(&response)?;
if let Some(usage) = reported_quota_usage(&response)?
&& usage == expected_usage
{
return Ok::<_, Box<dyn std::error::Error + Send + Sync>>(response);
}
} else if status != StatusCode::SERVICE_UNAVAILABLE {
return Err(format!("quota usage request failed for {path}: {status} {response}").into());
}
sleep(Duration::from_millis(100)).await;
}
};
match timeout(Duration::from_secs(30), convergence).await {
Ok(result) => result,
Err(_) => Err(format!("quota usage did not converge to {expected_usage} for {path} within 30 seconds").into()),
pub async fn get_bucket_quota_stats(&self) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
let url = format!("{}/rustfs/admin/v3/quota-stats/{}", self.env.url, self.bucket_name);
let response = awscurl_get(&url, &self.env.access_key, &self.env.secret_key).await?;
if response.contains("error") {
Err(format!("Failed to get quota stats: {}", response).into())
} else {
Ok(serde_json::from_str(&response)?)
}
}
async fn wait_for_bucket_quota_stats(
&self,
bucket: &str,
expected_usage: u64,
) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
self.wait_for_usage_response(Method::GET, &format!("/rustfs/admin/v3/quota-stats/{bucket}"), None, expected_usage)
.await
}
async fn wait_for_bucket_quota_check(
pub async fn check_bucket_quota(
&self,
operation_type: &str,
operation_size: u64,
expected_usage: u64,
) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
let url = format!("{}/rustfs/admin/v3/quota-check/{}", self.env.url, self.bucket_name);
let check_request = serde_json::json!({
"operation_type": operation_type,
"operation_size": operation_size
});
self.wait_for_usage_response(
Method::POST,
&format!("/rustfs/admin/v3/quota-check/{}", self.bucket_name),
Some(check_request.to_string()),
expected_usage,
)
.await
let response = awscurl_post(&url, &check_request.to_string(), &self.env.access_key, &self.env.secret_key).await?;
if response.contains("error") {
Err(format!("Failed to check quota: {}", response).into())
} else {
Ok(serde_json::from_str(&response)?)
}
}
pub async fn upload_object(&self, key: &str, size_bytes: usize) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -192,9 +145,30 @@ impl QuotaTestEnv {
}
}
pub async fn get_bucket_usage(&self) -> Result<u64, Box<dyn std::error::Error + Send + Sync>> {
let stats = self.get_bucket_quota_stats().await?;
stats
.get("current_usage")
.and_then(|value| value.as_u64())
.ok_or_else(|| format!("bucket quota usage is not authoritative for {}", self.bucket_name).into())
}
async fn wait_for_bucket_usage(&self, expected: u64) -> Result<u64, Box<dyn std::error::Error + Send + Sync>> {
let stats = self.wait_for_bucket_quota_stats(&self.bucket_name, expected).await?;
reported_quota_usage(&stats)?.ok_or_else(|| "converged quota usage is missing".into())
let convergence = async {
loop {
let stats = self.get_bucket_quota_stats().await?;
if let Some(usage) = stats.get("current_usage").and_then(|value| value.as_u64())
&& usage == expected
{
return Ok::<u64, Box<dyn std::error::Error + Send + Sync>>(usage);
}
sleep(Duration::from_millis(100)).await;
}
};
match timeout(Duration::from_secs(30), convergence).await {
Ok(result) => result,
Err(_) => Err(format!("bucket usage did not converge to {expected} bytes within 30 seconds").into()),
}
}
pub async fn set_bucket_quota_for(
@@ -202,6 +176,8 @@ impl QuotaTestEnv {
bucket: &str,
quota_bytes: u64,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
self.wait_for_quota_usage_for(bucket).await?;
let quota_path = format!("/rustfs/admin/v3/quota/{bucket}");
let quota_config = serde_json::json!({
"quota": quota_bytes,
@@ -235,6 +211,48 @@ impl QuotaTestEnv {
}
}
pub async fn wait_for_quota_usage_for(&self, bucket: &str) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let stats_path = format!("/rustfs/admin/v3/quota-stats/{bucket}");
let readiness = async {
loop {
let (status, response) =
admin_request(&self.env.url, Method::GET, &stats_path, None, &self.env.access_key, &self.env.secret_key)
.await?;
if status.is_success() {
let stats: serde_json::Value = serde_json::from_str(&response)?;
if stats.get("current_usage").and_then(|value| value.as_u64()).is_some() {
return Ok::<(), Box<dyn std::error::Error + Send + Sync>>(());
}
} else if status != StatusCode::SERVICE_UNAVAILABLE {
return Err(format!("quota usage readiness failed for {bucket}: {status} {response}").into());
}
sleep(Duration::from_secs(1)).await;
}
};
match timeout(Duration::from_secs(30), readiness).await {
Ok(result) => result,
Err(_) => Err(format!("quota usage did not become authoritative for {bucket} within 30 seconds").into()),
}
}
/// Get bucket quota statistics for specific bucket
pub async fn get_bucket_quota_stats_for(
&self,
bucket: &str,
) -> Result<serde_json::Value, Box<dyn std::error::Error + Send + Sync>> {
debug!("Getting quota stats for bucket: {}", bucket);
let url = format!("{}/rustfs/admin/v3/quota-stats/{}", self.env.url, bucket);
let response = awscurl_get(&url, &self.env.access_key, &self.env.secret_key).await?;
if response.contains("error") {
Err(format!("Failed to get quota stats: {}", response).into())
} else {
let stats: serde_json::Value = serde_json::from_str(&response)?;
Ok(stats)
}
}
/// Upload an object to specific bucket
pub async fn upload_object_to_bucket(
&self,
@@ -259,132 +277,6 @@ impl QuotaTestEnv {
}
}
#[cfg(test)]
mod usage_response_tests {
use super::*;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
use tokio::task::JoinSet;
#[test]
fn unknown_usage_is_not_zero() {
assert_eq!(
reported_quota_usage(&serde_json::json!({"current_usage": null})).expect("valid unknown usage"),
None
);
assert_eq!(
reported_quota_usage(&serde_json::json!({"current_usage": 0})).expect("known zero usage"),
Some(0)
);
}
#[test]
fn missing_or_invalid_usage_is_not_readiness() {
for response in [
serde_json::json!({}),
serde_json::json!({"current_usage": -1}),
serde_json::json!({"current_usage": "0"}),
serde_json::json!({"current_usage": 1.5}),
serde_json::json!({"current_usage": false}),
] {
let error = reported_quota_usage(&response).expect_err("malformed usage must fail");
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
}
}
async fn mock_environment(
responses: Vec<(StatusCode, &'static str)>,
) -> Result<(QuotaTestEnv, JoinSet<()>), Box<dyn std::error::Error + Send + Sync>> {
let listener = TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)).await?;
let address = listener.local_addr()?;
let mut tasks = JoinSet::new();
tasks.spawn(async move {
for (status, body) in responses {
let (mut stream, _) = listener.accept().await.expect("accept quota reporting request");
let mut request = Vec::new();
while !request.windows(4).any(|part| part == b"\r\n\r\n") {
let mut buffer = [0_u8; 1024];
let read = stream.read(&mut buffer).await.expect("read reporting request");
assert!(read > 0 && request.len() + read <= 8192, "bounded HTTP request headers");
request.extend_from_slice(&buffer[..read]);
}
let response = format!(
"HTTP/1.1 {} Test\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
status.as_u16(),
body.len(),
body
);
stream
.write_all(response.as_bytes())
.await
.expect("write quota reporting response");
stream.shutdown().await.expect("close reporting response");
}
});
let mut env = RustFSTestEnvironment::new().await?;
env.url = format!("http://{address}");
let client = env.create_s3_client();
Ok((
QuotaTestEnv {
env,
client,
bucket_name: "quota-reporting-fixture".to_owned(),
},
tasks,
))
}
#[tokio::test]
async fn reporting_waits_for_known_matching_usage_without_selecting_allowed()
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let (env, mut tasks) = mock_environment(vec![
(StatusCode::SERVICE_UNAVAILABLE, "unavailable"),
(StatusCode::OK, r#"{"current_usage":null}"#),
(StatusCode::OK, r#"{"current_usage":1}"#),
(StatusCode::OK, r#"{"current_usage":0,"allowed":false}"#),
])
.await?;
let response = env.wait_for_usage_response(Method::GET, "/reporting", None, 0).await?;
assert_eq!(response["current_usage"], 0);
assert_eq!(response["allowed"], false, "readiness must not select a desired admission outcome");
timeout(Duration::from_secs(1), tasks.join_next())
.await?
.expect("reporting mock task")
.expect("all responses consumed");
Ok(())
}
#[tokio::test]
async fn reporting_rejects_malformed_success_instead_of_retrying() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let (env, mut tasks) = mock_environment(vec![(StatusCode::OK, r#"{"current_usage":"unknown"}"#)]).await?;
let error = env
.wait_for_usage_response(Method::GET, "/reporting", None, 0)
.await
.expect_err("malformed usage must fail");
assert!(error.to_string().contains("not an unsigned integer or null"));
timeout(Duration::from_secs(1), tasks.join_next())
.await?
.expect("reporting mock task")
.expect("response consumed");
Ok(())
}
#[tokio::test]
async fn reporting_does_not_retry_authorization_failure() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let (env, mut tasks) = mock_environment(vec![(StatusCode::FORBIDDEN, "denied")]).await?;
let error = env
.wait_for_usage_response(Method::GET, "/reporting", None, 0)
.await
.expect_err("authorization error must fail");
assert!(error.to_string().contains("403"));
timeout(Duration::from_secs(1), tasks.join_next())
.await?
.expect("reporting mock task")
.expect("response consumed");
Ok(())
}
}
#[cfg(test)]
mod integration_tests {
use super::*;
@@ -597,7 +489,7 @@ mod integration_tests {
env.upload_object("file2.txt", 256 * 1024).await?;
// Check usage
let usage = env.wait_for_bucket_usage((512 + 256) * 1024).await?;
let usage = env.get_bucket_usage().await?;
assert_eq!(usage, (512 + 256) * 1024);
// Delete a file
@@ -632,7 +524,7 @@ mod integration_tests {
env.upload_object("file2.txt", 512 * 1024).await?;
// Get detailed quota statistics
let stats = env.wait_for_bucket_quota_stats(&env.bucket_name, (1024 + 512) * 1024).await?;
let stats = env.get_bucket_quota_stats().await?;
assert_eq!(stats.get("bucket").unwrap().as_str().unwrap(), env.bucket_name);
assert_eq!(stats.get("quota_limit").unwrap().as_u64().unwrap(), 2 * 1024 * 1024);
@@ -661,22 +553,22 @@ mod integration_tests {
env.upload_object("existing_file.txt", 512 * 1024).await?;
// Check if we can upload another 512KB (should succeed, exactly fill quota)
let check_result = env.wait_for_bucket_quota_check("PUT", 512 * 1024, 512 * 1024).await?;
let check_result = env.check_bucket_quota("PUT", 512 * 1024).await?;
assert!(check_result.get("allowed").unwrap().as_bool().unwrap());
assert_eq!(check_result.get("remaining_quota").unwrap().as_u64().unwrap(), 0);
// Note: we haven't actually uploaded the second file yet, so current_usage is still 512KB
// Check if we can upload 1KB (should succeed - we haven't used the full quota yet)
let check_result = env.wait_for_bucket_quota_check("PUT", 1024, 512 * 1024).await?;
let check_result = env.check_bucket_quota("PUT", 1024).await?;
assert!(check_result.get("allowed").unwrap().as_bool().unwrap());
assert_eq!(check_result.get("remaining_quota").unwrap().as_u64().unwrap(), 523264); // 511 * 1024
// Check if we can upload 600KB (should fail - would exceed quota)
let check_result = env.wait_for_bucket_quota_check("PUT", 600 * 1024, 512 * 1024).await?;
let check_result = env.check_bucket_quota("PUT", 600 * 1024).await?;
assert!(!check_result.get("allowed").unwrap().as_bool().unwrap());
// Check delete operation (should always be allowed)
let check_result = env.wait_for_bucket_quota_check("DELETE", 512 * 1024, 512 * 1024).await?;
let check_result = env.check_bucket_quota("DELETE", 512 * 1024).await?;
assert!(check_result.get("allowed").unwrap().as_bool().unwrap());
env.cleanup_bucket().await?;
@@ -708,8 +600,8 @@ mod integration_tests {
env.upload_object_to_bucket(&bucket2, "another_file.txt", 512 * 1024).await?;
// Verify statistics are independent
let stats1 = env.wait_for_bucket_quota_stats(&bucket1, 1024 * 1024).await?;
let stats2 = env.wait_for_bucket_quota_stats(&bucket2, (1024 + 512) * 1024).await?;
let stats1 = env.get_bucket_quota_stats_for(&bucket1).await?;
let stats2 = env.get_bucket_quota_stats_for(&bucket2).await?;
assert_eq!(stats1.get("current_usage").unwrap().as_u64().unwrap(), 1024 * 1024);
assert_eq!(stats2.get("current_usage").unwrap().as_u64().unwrap(), (1024 + 512) * 1024);
@@ -770,6 +662,7 @@ mod integration_tests {
let env = QuotaTestEnv::new().await?;
env.create_bucket().await?;
env.wait_for_quota_usage_for(&env.bucket_name).await?;
// Test 1: GET quota for bucket without quota config
let url = format!("{}/rustfs/admin/v3/quota/{}", env.env.url, env.bucket_name);
+11 -175
View File
@@ -27,8 +27,8 @@ use hyper_util::{
};
use prost::Message as _;
use rustfs_protos::proto_gen::node_service::{
BatchGenerallyLockRequest, DeleteRequest, GenerallyLockRequest, PreparePartTransactionResponse, ReadVersionRequest,
RecoverMultipartSourceRequest, RenameDataResponse, SettlePartTransactionRequest, SettlePartTransactionResponse,
BatchGenerallyLockRequest, DeleteRequest, GenerallyLockRequest, ReadVersionRequest, RecoverMultipartSourceRequest,
RenameDataResponse,
};
use std::{
collections::HashSet,
@@ -56,35 +56,6 @@ enum Disposition {
Discard,
}
#[derive(Clone, Copy)]
enum MutationReply {
None,
Rename,
PreparePart,
SettlePart,
}
impl MutationReply {
fn outcome(self, bytes: &[u8]) -> io::Result<Option<(bool, bool)>> {
let result = match self {
Self::None => return Ok(None),
Self::Rename => {
let reply = RenameDataResponse::decode(unary_payload(bytes)?).map_err(io::Error::other)?;
(reply.success, reply.error.is_some())
}
Self::PreparePart => {
let reply = PreparePartTransactionResponse::decode(unary_payload(bytes)?).map_err(io::Error::other)?;
(reply.success, reply.error.is_some())
}
Self::SettlePart => {
let reply = SettlePartTransactionResponse::decode(unary_payload(bytes)?).map_err(io::Error::other)?;
(reply.success, reply.error.is_some())
}
};
Ok(Some(result))
}
}
struct PauseState {
methods: Vec<&'static str>,
scope: Option<Vec<u8>>,
@@ -93,9 +64,6 @@ struct PauseState {
limit: usize,
retain_after_disconnect: bool,
refused_mutations: AtomicUsize,
successful_mutation_replies: AtomicUsize,
settle_commits: AtomicUsize,
settle_rollbacks: AtomicUsize,
responses: AtomicUsize,
matched: AtomicUsize,
blocked: AtomicUsize,
@@ -154,17 +122,6 @@ impl RpcPause {
self.state.refused_mutations.load(Ordering::SeqCst)
}
pub(crate) fn successful_mutation_replies(&self) -> usize {
self.state.successful_mutation_replies.load(Ordering::SeqCst)
}
pub(crate) fn retained_settle_actions(&self) -> (usize, usize) {
(
self.state.settle_commits.load(Ordering::SeqCst),
self.state.settle_rollbacks.load(Ordering::SeqCst),
)
}
pub(crate) fn unsupported_source_inspection_routes(&self) -> HashSet<String> {
self.state
.unsupported_inspection_routes
@@ -437,9 +394,6 @@ impl RpcFaultProxy {
limit,
retain_after_disconnect,
refused_mutations: AtomicUsize::new(0),
successful_mutation_replies: AtomicUsize::new(0),
settle_commits: AtomicUsize::new(0),
settle_rollbacks: AtomicUsize::new(0),
responses: AtomicUsize::new(0),
matched: AtomicUsize::new(0),
blocked: AtomicUsize::new(0),
@@ -555,7 +509,6 @@ async fn forward_request(
if let Some(method) = method.filter(|name| pause.methods.contains(name)) {
let mut route = None;
let mut lock_owners = Vec::new();
let mut settle_action = None;
let matches = if let Some(scope) = pause.scope.as_deref() {
// Inspect bounded control routing while preserving the exact
// signed bytes and trailers forwarded to the backend.
@@ -569,19 +522,12 @@ async fn forward_request(
inspection = Some((pause.clone(), disk));
}
let mut matches = scope_matches(method, &bytes, scope)?;
if matches && method == "SettlePartTransaction" {
settle_action = Some(
SettlePartTransactionRequest::decode(unary_payload(&bytes)?)
.map_err(io::Error::other)?
.rollback,
);
}
if matches && (pause.record_lock_owners || pause.lock_owner.is_some()) {
lock_owners = scoped_lock_owners(method, &bytes, scope)?;
if let Some(owner) = &pause.lock_owner {
matches = lock_owners.iter().any(|candidate| candidate == owner);
if !matches && pause.record_lock_owners {
observed = Some((pause.clone(), None, std::mem::take(&mut lock_owners), MutationReply::None));
observed = Some((pause.clone(), None, std::mem::take(&mut lock_owners), false));
}
}
}
@@ -603,14 +549,6 @@ async fn forward_request(
true
};
if matches && pause.matched.fetch_add(1, Ordering::SeqCst) < pause.limit {
if let Some(rollback) = settle_action {
let counter = if rollback {
&pause.settle_rollbacks
} else {
&pause.settle_commits
};
counter.fetch_add(1, Ordering::SeqCst);
}
let mut released = pause.release.subscribe();
pause.blocked.fetch_add(1, Ordering::SeqCst);
pause.reached.notify_one();
@@ -630,14 +568,10 @@ async fn forward_request(
pause,
route,
lock_owners,
match method {
"RenameData" | "RenameMultipartData" | "RenameDataAtIncarnation" | "RenameDataWithQuotaFence" => {
MutationReply::Rename
}
"PreparePartTransaction" => MutationReply::PreparePart,
"SettlePartTransaction" => MutationReply::SettlePart,
_ => MutationReply::None,
},
matches!(
method,
"RenameData" | "RenameMultipartData" | "RenameDataAtIncarnation" | "RenameDataWithQuotaFence"
),
));
}
}
@@ -670,7 +604,7 @@ async fn forward_request(
.expect("inspection response routes")
.insert(disk);
}
if let Some((pause, route, lock_owners, mutation_reply)) = observed {
if let Some((pause, route, lock_owners, rename)) = observed {
pause.responses.fetch_add(1, Ordering::SeqCst);
if response.status().is_success() && response.headers().get("grpc-status").is_none_or(|status| status == "0") {
pause.forwarded.fetch_add(1, Ordering::SeqCst);
@@ -692,14 +626,10 @@ async fn forward_request(
.headers
.get("grpc-status")
.or_else(|| trailers.as_ref()?.get("grpc-status"));
if parts.status.is_success()
&& status.is_some_and(|status| status == "0")
&& let Some((success, has_error)) = mutation_reply.outcome(&bytes)?
{
if !success && has_error {
if rename && parts.status.is_success() && status.is_none_or(|status| status == "0") {
let result = RenameDataResponse::decode(unary_payload(&bytes)?).map_err(io::Error::other)?;
if !result.success && result.error.is_some() {
pause.refused_mutations.fetch_add(1, Ordering::SeqCst);
} else if success && !has_error {
pause.successful_mutation_replies.fetch_add(1, Ordering::SeqCst);
}
}
let body = Full::new(bytes)
@@ -746,10 +676,6 @@ mod tests {
}
async fn backend_with_status(status: Option<&'static str>) -> io::Result<Backend> {
backend_with_reply(status, None).await
}
async fn backend_with_reply(status: Option<&'static str>, reply: Option<Bytes>) -> io::Result<Backend> {
let listener = TcpListener::bind("127.0.0.1:0").await?;
let address = listener.local_addr()?;
let received = Arc::new(AtomicUsize::new(0));
@@ -761,11 +687,9 @@ mod tests {
accepted = listener.accept() => {
let Ok((socket, _)) = accepted else { break };
let received = received_task.clone();
let reply = reply.clone();
children.spawn(async move {
let service = service_fn(move |request: Request<Incoming>| {
let received = received.clone();
let reply = reply.clone();
async move {
received.fetch_add(1, Ordering::SeqCst);
let authority = request.headers().get(http::header::HOST).cloned().or_else(|| {
@@ -774,7 +698,6 @@ mod tests {
})
});
let body = request.into_body().collect().await?.to_bytes();
let body = reply.unwrap_or(body);
let mut trailers = http::HeaderMap::new();
trailers.insert("grpc-status", http::HeaderValue::from_static(status.unwrap_or("0")));
trailers.insert("x-proof", http::HeaderValue::from_static("unchanged-proof"));
@@ -1511,93 +1434,6 @@ mod tests {
.await
.expect("scoped discard deadline");
assert_late_packet_ownership().await;
assert_late_part_reply_refusal().await;
}
async fn assert_late_part_reply_refusal() {
use rustfs_protos::proto_gen::node_service::{Error, PreparePartTransactionRequest};
tokio::time::timeout(Duration::from_secs(10), async {
// method, success, error, gRPC status, rollback, expected successes/refusals
for (method, success, has_error, status, rollback, expected) in [
("PreparePartTransaction", false, true, None, false, (0, 1)),
("PreparePartTransaction", true, false, None, false, (1, 0)),
("PreparePartTransaction", false, false, None, false, (0, 0)),
("SettlePartTransaction", true, false, None, false, (1, 0)),
("SettlePartTransaction", false, true, None, true, (0, 1)),
("SettlePartTransaction", true, true, None, false, (0, 0)),
("SettlePartTransaction", false, true, Some("16"), false, (0, 0)),
("PreparePartTransaction", false, true, Some("16"), false, (0, 0)),
("RenameMultipartData", false, true, None, false, (0, 1)),
] {
let error = has_error.then(|| Error {
code: 1,
error_info: "retired source".to_owned(),
});
let reply = match method {
"PreparePartTransaction" => unary_frame(&PreparePartTransactionResponse { success, error }),
"SettlePartTransaction" => unary_frame(&SettlePartTransactionResponse { success, error }),
_ => unary_frame(&RenameDataResponse {
success,
error,
..Default::default()
}),
};
// An unknown field must survive typed inspection byte-for-byte.
let mut framed = reply.to_vec();
framed.extend_from_slice(&[0xfa, 0x03, 0x03, b'x', b'y', b'z']);
let payload_len = u32::try_from(framed.len() - 5).expect("small reply");
framed[1..5].copy_from_slice(&payload_len.to_be_bytes());
let reply = Bytes::from(framed);
let backend = backend_with_reply(status, Some(reply.clone()))
.await
.expect("typed reply backend");
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
.await
.expect("proxy");
let pause = proxy.pause_late(&[method], 1, "late-upload").expect("part gate");
let request = match method {
"PreparePartTransaction" => unary_frame(&PreparePartTransactionRequest {
disk: "disk".to_owned(),
dst_path: "late-upload/part.1".to_owned(),
..Default::default()
}),
"SettlePartTransaction" => unary_frame(&SettlePartTransactionRequest {
disk: "disk".to_owned(),
path: "late-upload/part.1".to_owned(),
rollback,
..Default::default()
}),
_ => Bytes::from_static(b"late-upload"),
};
let address = proxy.local_addr();
let caller = tokio::spawn(async move {
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
let response = call_body(&client, address, method, request).await.expect("retained reply");
response.into_body().collect().await.expect("typed response frames")
});
pause.wait_for_hits(1).await.expect("actual retained request");
pause.release();
let response = caller.await.expect("caller task");
let trailers = response.trailers().expect("reply trailers");
assert_eq!(trailers["x-proof"], "unchanged-proof");
assert_eq!(trailers["grpc-status"], status.unwrap_or("0"));
assert_eq!(response.to_bytes(), reply);
assert_eq!(pause.responses(), 1);
assert_eq!(
(pause.successful_mutation_replies(), pause.refused_mutations()),
expected,
"typed {method} outcome"
);
let actions = if method == "SettlePartTransaction" {
if rollback { (0, 1) } else { (1, 0) }
} else {
(0, 0)
};
assert_eq!(pause.retained_settle_actions(), actions);
}
})
.await
.expect("part reply deadline");
}
async fn assert_late_packet_ownership() {
tokio::time::timeout(Duration::from_secs(10), async {
-3
View File
@@ -12,9 +12,6 @@ Applies to `crates/ecstore/`.
- Be careful with allocations and locking in hot paths.
- Keep network and disk operations async-friendly; avoid introducing unnecessary blocking.
- Coalesce related `open`, `read`/`write`, `fsync`, and `close` filesystem steps into a single `blocking_queue` or `run_blocking_namespace_operation` closure per disk rather than chaining multiple `tokio::fs` or `spawn_blocking` dispatches.
- Pin a parent directory once (`openat2(BENEATH | NO_SYMLINKS)` on Linux) and reuse the pinned guard across sibling metadata reads (`read_optional_in_pinned`, `part.N.meta` loops) instead of re-walking ancestor components per file or re-opening when `source_parent == destination_parent`.
- After `file.metadata()`, pre-size read buffers (`Vec::with_capacity(len + 1)`) before `read_to_end` to avoid `std::io::read_to_end`'s 32-byte exponential growth (`6–7` `read()` syscalls per small metadata file).
- Benchmark-sensitive changes should include measurable rationale.
## Allocation Discipline in Hot Paths
@@ -11314,7 +11314,13 @@ mod tests {
let marker = marker
.as_deref()
.expect("production checkpoint should persist an object marker");
assert_eq!(marker, expected_marker, "production checkpoint should stop at the first page boundary");
let marker_suffix = marker
.strip_prefix(&expected_marker)
.expect("production checkpoint should stop at the first page boundary");
assert!(
marker_suffix.is_empty() || marker_suffix.starts_with("[rustfs_cache:v2,"),
"unexpected production checkpoint marker suffix: {marker_suffix}"
);
assert_eq!(version_marker.as_deref(), Some("null"));
}
+12 -4
View File
@@ -3186,9 +3186,17 @@ pub mod test_support {
use crate::store::init_local_disks_with_instance_ctx;
pub async fn isolated_store_over_temp_disks() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
let mut dirs = Vec::with_capacity(4);
let mut endpoints = Vec::with_capacity(4);
for disk_idx in 0..4 {
isolated_store_over_temp_disks_with_count(4).await
}
pub async fn isolated_store_over_temp_disks_with_count(disk_count: usize) -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
assert!(
matches!(disk_count, 4 | 16),
"isolated store fixture requires a registered logical set geometry"
);
let mut dirs = Vec::with_capacity(disk_count);
let mut endpoints = Vec::with_capacity(disk_count);
for disk_idx in 0..disk_count {
let dir = tempfile::tempdir().expect("tempdir should be created");
let mut endpoint =
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
@@ -3201,7 +3209,7 @@ pub mod test_support {
let endpoint_pools = EndpointServerPools(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
drives_per_set: disk_count,
endpoints: Endpoints::from(endpoints),
cmd_line: "metadata-sys-cache-test".to_string(),
platform: "test".to_string(),
+61 -14
View File
@@ -43,6 +43,11 @@ const EVENT_METACACHE_LISTING: &str = "metacache_listing";
pub type AgreedFn = Box<dyn Fn(MetaCacheEntry) -> Pin<Box<dyn Future<Output = Result<(), DiskError>> + Send>> + Send + 'static>;
pub type PartialFn =
Box<dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
pub(crate) type PartialResultFn = Box<
dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = disk::error::Result<()>> + Send>>
+ Send
+ 'static,
>;
type FinishedFn = Box<dyn Fn(&[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
#[derive(Clone, Default)]
@@ -55,7 +60,6 @@ impl FallbackClaimTracker {
self.claimed.lock().await.insert(disk.endpoint().to_string());
}
#[cfg(test)]
pub(crate) async fn claimed_keys(&self) -> HashSet<String> {
self.claimed.lock().await.clone()
}
@@ -313,17 +317,7 @@ pub struct ListPathRawOutcome {
pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
let rx = rx.child_token();
let _cancel_guard = rx.clone().drop_guard();
list_path_raw_inner(rx, opts, None).await.map(|_| ())
}
/// Like [`list_path_raw`], but reports how the merged stream ended.
pub(crate) async fn list_path_raw_with_outcome(
rx: CancellationToken,
opts: ListPathRawOptions,
) -> disk::error::Result<ListPathRawOutcome> {
let rx = rx.child_token();
let _cancel_guard = rx.clone().drop_guard();
list_path_raw_inner(rx, opts, None).await
list_path_raw_inner(rx, opts, None, None).await.map(|_| ())
}
pub(crate) async fn list_path_raw_with_claim_tracker(
@@ -333,13 +327,25 @@ pub(crate) async fn list_path_raw_with_claim_tracker(
) -> disk::error::Result<ListPathRawOutcome> {
let rx = rx.child_token();
let _cancel_guard = rx.clone().drop_guard();
list_path_raw_inner(rx, opts, Some(claim_tracker)).await
list_path_raw_inner(rx, opts, Some(claim_tracker), None).await
}
pub(crate) async fn list_path_raw_with_partial_result(
rx: CancellationToken,
opts: ListPathRawOptions,
claim_tracker: FallbackClaimTracker,
partial_result: PartialResultFn,
) -> disk::error::Result<ListPathRawOutcome> {
let rx = rx.child_token();
let _cancel_guard = rx.clone().drop_guard();
list_path_raw_inner(rx, opts, Some(claim_tracker), Some(partial_result)).await
}
async fn list_path_raw_inner(
rx: CancellationToken,
opts: ListPathRawOptions,
fallback_claim_tracker: Option<FallbackClaimTracker>,
partial_result: Option<PartialResultFn>,
) -> disk::error::Result<ListPathRawOutcome> {
if opts.disks.is_empty() {
return Err(DiskError::ErasureReadQuorum);
@@ -1014,7 +1020,13 @@ async fn list_path_raw_inner(
}
}
if let Some(partial_fn) = opts.partial.as_ref() {
if let Some(partial_fn) = partial_result.as_ref() {
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
result = partial_fn(MetaCacheEntries(top_entries), &errs) => result?,
}
} else if let Some(partial_fn) = opts.partial.as_ref() {
tokio::select! {
biased;
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
@@ -1296,6 +1308,7 @@ mod tests {
..Default::default()
},
None,
None,
)
.await
.expect("listing should complete");
@@ -1320,6 +1333,7 @@ mod tests {
..Default::default()
},
None,
None,
)
.await
.expect("listing should complete");
@@ -1813,6 +1827,39 @@ mod tests {
);
}
#[tokio::test]
async fn list_path_raw_propagates_partial_callback_failure() {
let first = MetaCacheEntry {
name: "bucket/object-a".to_string(),
metadata: vec![1],
..Default::default()
};
let second = MetaCacheEntry {
name: "bucket/object-b".to_string(),
metadata: vec![2],
..Default::default()
};
let err = list_path_raw_with_partial_result(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![None, None],
min_disks: 2,
test_reader_behaviors: vec![
TestReaderBehavior::Entries(vec![first]),
TestReaderBehavior::Entries(vec![second]),
],
..Default::default()
},
FallbackClaimTracker::default(),
Box::new(move |_, _| Box::pin(async { Err(DiskError::ErasureReadQuorum) })),
)
.await
.expect_err("a resolution failure after reader advancement must fail the listing");
assert_eq!(err, DiskError::ErasureReadQuorum);
}
#[tokio::test]
async fn list_path_raw_cancels_a_blocked_partial_callback() {
let cancel = CancellationToken::new();
@@ -8748,89 +8748,6 @@ mod tests {
peer.stop().await;
}
#[tokio::test]
#[serial(remote_disk_recovery_probe)]
async fn endpoint_reconnect_admits_ready_drive_while_another_format_probe_is_blocked() {
runtime_sources::ensure_test_rpc_secret();
let reference = crate::layout::format::FormatV3::new(1, 2);
let mut peers = Vec::new();
let mut endpoints = Vec::new();
for index in 0..2 {
let mut format = reference.clone();
format.erasure.this = reference.erasure.sets[0][index];
let peer = TestGrpcPeer::spawn(
Bytes::from(format.to_json().expect("format should serialize")),
Bytes::from_static(b"recovered-peer-data"),
)
.await
.expect("authenticated regression peer is required");
endpoints.push(Endpoint {
url: url::Url::parse(&format!("{}/data/rustfs{index}", peer.addr)).expect("endpoint should parse"),
is_local: false,
pool_idx: 0,
set_idx: 0,
disk_idx: i32::try_from(index).expect("fixture index fits"),
});
peers.push(peer);
}
let set = crate::set_disk::SetDisks::new(
"independent-reconnect-test".to_owned(),
Arc::new(tokio::sync::RwLock::new(vec![None; 2])),
2,
1,
0,
0,
endpoints,
reference.clone(),
Vec::new(),
)
.await;
let gate = peers[0].peer.pause_next_format_read();
let pending_set = Arc::clone(&set);
let pending = tokio::spawn(async move { pending_set.connect_disks().await });
time::timeout(Duration::from_secs(10), gate.entered.notified())
.await
.expect("the first endpoint must reach the format barrier");
let ready = time::timeout(Duration::from_secs(10), async {
loop {
if let Some(disk) = set.disks.read().await[1].clone() {
return disk;
}
tokio::task::yield_now().await;
}
})
.await;
if let Ok(disk) = &ready {
assert!(set.disks.read().await[0].is_none(), "the blocked endpoint must remain unpublished");
assert!(!pending.is_finished(), "the sweep still owns its blocked endpoint probe");
assert_eq!(
disk.read_all("bucket", "object")
.await
.expect("the ready endpoint must serve real I/O"),
Bytes::from_static(b"recovered-peer-data")
);
assert_eq!(
peers[1].peer.object_read_all_disks(),
vec![reference.erasure.sets[0][1].to_string()],
"the usable handle must carry its validated slot identity"
);
}
gate.release.cancel();
pending.await.expect("all endpoint probes must finish after release");
for disk in set.disks.read().await.iter().flatten() {
disk.close().await.expect("stop each published handle's monitoring");
}
for peer in peers {
peer.stop().await;
}
ready.expect("a blocked earlier endpoint must not delay a ready drive in the same set");
assert!(
set.disks.read().await.iter().all(Option::is_some),
"both endpoints must eventually be admitted"
);
}
#[tokio::test]
#[serial(remote_disk_recovery_probe)]
async fn missing_read_disks_preserve_concurrently_published_handles() {
@@ -41,10 +41,6 @@ use tonic::service::interceptor::InterceptedService;
use tonic::{Request, Response};
use tracing::{debug, info, warn};
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_REMOTE_LOCK: &str = "remote_lock";
fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(request: &mut Request<T>) -> std::io::Result<()> {
set_tonic_rolling_mutation_body_digest(request)
}
@@ -237,20 +233,6 @@ pub struct RemoteClient {
impl RemoteClient {
const ONLINE_CHECK_RESOURCE: &'static str = "health-lock-online";
fn record_release_ack(&self, lock_id: &LockId) {
tracing::trace!(
target: "rustfs_ecstore::object_lock_diag",
event = EVENT_NAMESPACE_LOCK_RELEASE,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
state = "acknowledged",
peer_kind = "remote",
lock_id = %lock_id,
peer = %self.addr,
"Remote namespace lock release acknowledged"
);
}
pub fn new(endpoint: String) -> Self {
Self { addr: endpoint }
}
@@ -1222,6 +1204,8 @@ impl LockClient for RemoteClient {
}
async fn release(&self, lock_id: &LockId) -> Result<bool> {
info!("remote release for {}", lock_id);
let unlock_request = Self::create_unlock_request(lock_id);
let request_string = serde_json::to_string(&unlock_request)
.map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?;
@@ -1242,9 +1226,6 @@ impl LockClient for RemoteClient {
if let Some(error_info) = resp.error_info {
return Err(LockError::internal(error_info));
}
if resp.success {
self.record_release_ack(lock_id);
}
Ok(resp.success)
}
@@ -1280,13 +1261,7 @@ impl LockClient for RemoteClient {
Ok(lock_ids
.iter()
.enumerate()
.map(|(idx, lock_id)| {
let result = resp.results.get(idx);
if result.is_some_and(|result| result.success && result.error_info.is_none()) {
self.record_release_ack(lock_id);
}
result.map(|result| result.success).unwrap_or(false)
})
.map(|(idx, _)| resp.results.get(idx).map(|result| result.success).unwrap_or(false))
.collect())
}
-11
View File
@@ -317,17 +317,6 @@ impl Sets {
&self.ctx
}
#[cfg(test)]
pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
if let Some(exit_signal) = &self.exit_signal {
let _ = exit_signal.send(());
// Wait for an in-flight reconnect pass as well as its receiver to exit.
while exit_signal.receiver_count() != 0 {
tokio::task::yield_now().await;
}
}
}
/// Keep simulated peers' metadata ownership separate while sharing disks and lock clients.
#[cfg(test)]
pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
+2 -249
View File
@@ -19,7 +19,6 @@ use tracing::Span;
use uuid::Uuid;
const EVENT_OBJECT_NAMESPACE_LOCK: &str = "object_namespace_lock";
const EVENT_OBJECT_NAMESPACE_LOCK_PEER: &str = "object_namespace_lock_peer";
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
const LOG_SUBSYSTEM_OBJECT_LOCK: &str = "object_lock";
@@ -57,7 +56,6 @@ struct AttemptContext {
requested_version_id: Option<String>,
namespace: String,
owner: String,
lock_id: Option<String>,
mode: &'static str,
timeout: Duration,
started_at: Instant,
@@ -87,7 +85,6 @@ impl ObjectLockAttempt {
requested_version_id: requested_version_id.map(str::to_owned),
namespace: lock.namespace().to_owned(),
owner: lock.owner().to_owned(),
lock_id: None,
mode,
timeout,
started_at: Instant::now(),
@@ -108,29 +105,9 @@ impl ObjectLockAttempt {
context.finished = true;
context.trace("disabled");
}
Ok(guard) => {
context.lock_id = guard.lock_id().map(ToString::to_string);
Ok(_) => {
context.acquired_at = Some(Instant::now());
context.trace("acquired");
if let NamespaceLockGuard::Standard(guard) = guard {
for (peer_index, peer_lock_id) in guard.granted_lock_ids().enumerate() {
context.parent.in_scope(|| {
tracing::trace!(
target: "rustfs_ecstore::object_lock_diag",
parent: &context.parent,
event = EVENT_OBJECT_NAMESPACE_LOCK_PEER,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_OBJECT_LOCK,
state = "granted",
lock_attempt_id = %context.id,
lock_id = context.lock_id.as_deref(),
peer_lock_id = %peer_lock_id,
peer_index,
"Object namespace peer owner granted"
);
});
}
}
}
Err(error) => {
context.finished = true;
@@ -194,7 +171,6 @@ impl AttemptContext {
mode = self.mode,
owner = %self.owner,
lock_attempt_id = %self.id,
lock_id = self.lock_id.as_deref(),
acquire_ms = duration_ms(acquired_at.duration_since(self.started_at)),
hold_ms = duration_ms(now.duration_since(acquired_at)),
timeout_ms = duration_ms(self.timeout),
@@ -259,238 +235,15 @@ pub(crate) mod tests {
}
pub(crate) fn rows(&self) -> Vec<serde_json::Value> {
self.all_rows()
.into_iter()
.filter(|row| row["fields"]["event"] == EVENT_OBJECT_NAMESPACE_LOCK)
.collect()
}
fn all_rows(&self) -> Vec<serde_json::Value> {
let bytes = self.0.lock().expect("lock diagnostic capture mutex");
let text = std::str::from_utf8(&bytes).expect("UTF-8 diagnostic events");
text.lines()
.map(|line| serde_json::from_str::<serde_json::Value>(line).expect("JSON diagnostic event"))
.filter(|row| row["fields"]["event"] == EVENT_OBJECT_NAMESPACE_LOCK)
.collect()
}
}
#[test]
#[serial_test::serial]
fn object_lock_diagnostics_join_guard_identity_to_physical_release() {
use rustfs_lock::{
DisabledLockManager, FastObjectLockManager, GlobalLockManager, LocalClient, LockClient, LockRequest, LockType,
NamespaceLock, ObjectKey,
};
temp_env::with_vars([(rustfs_config::ENV_OBJECT_LOCK_DIAG_ENABLE, Some("true"))], || {
let capture = CapturedLockEvents::default();
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("diagnostic release runtime");
tracing::subscriber::with_default(capture.subscriber(), || {
runtime.block_on(async {
let clients = (0..4)
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn LockClient>)
.collect::<Vec<_>>();
let lock = NamespaceLockWrapper::new(
NamespaceLock::with_clients_and_quorum("diagnostics".to_owned(), clients.clone(), 4),
ObjectKey::new("bucket", "object"),
"coordinator".to_owned(),
);
let request = tracing::info_span!("request", request_id = "original-multipart-request");
let mut attempt = request.in_scope(|| {
ObjectLockAttempt::start(
"abort_multipart_upload",
"bucket",
"object",
None,
&lock,
"write",
Duration::from_secs(5),
)
});
let result = lock.get_write_lock(Duration::from_secs(5)).await;
attempt.observe(&result);
let guard = result.expect("four real namespace owners");
let id = guard.lock_id().expect("distributed guard identity").clone();
let peer_ids = match &guard {
NamespaceLockGuard::Standard(guard) => {
assert_eq!(guard.granted_lock_ids().count(), 4);
guard.granted_lock_ids().cloned().collect::<std::collections::HashSet<_>>()
}
NamespaceLockGuard::Fast(_) => panic!("real distributed namespace required"),
};
let rows = capture.rows();
assert_eq!(rows.last().expect("acquired event")["fields"]["lock_id"], id.to_string());
let rows = capture.all_rows();
let mappings = rows
.iter()
.filter(|row| row["fields"]["event"] == EVENT_OBJECT_NAMESPACE_LOCK_PEER)
.collect::<Vec<_>>();
assert_eq!(mappings.len(), 4, "every granted peer must map to its logical guard");
assert!(mappings.iter().all(|row| {
row["fields"]["lock_id"] == id.to_string()
&& peer_ids
.iter()
.any(|peer_id| row["fields"]["peer_lock_id"] == peer_id.to_string())
}));
for client in &clients {
let mut owned = 0;
for peer_id in &peer_ids {
owned += usize::from(
client
.check_status(peer_id)
.await
.expect("actual peer owner status")
.is_some(),
);
}
assert_eq!(owned, 1, "each client must hold exactly one actual granted identity");
}
drop((guard, attempt));
assert!(
capture.all_rows().iter().all(|row| row["fields"]["state"] != "released"),
"enqueue is not release acknowledgement"
);
tokio::time::timeout(Duration::from_secs(5), async {
while capture
.all_rows()
.iter()
.filter(|row| {
row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "released"
})
.count()
< 4
{
tokio::task::yield_now().await;
}
})
.await
.expect("all physical release callbacks finish");
for client in &clients {
for peer_id in &peer_ids {
assert!(
client
.check_status(peer_id)
.await
.expect("released peer owner status")
.is_none()
);
assert!(!client.release(peer_id).await.expect("duplicate release is absent"));
}
}
let disabled = LocalClient::with_manager(Arc::new(GlobalLockManager::Disabled(DisabledLockManager::new())));
let disabled_request =
LockRequest::new(ObjectKey::new("bucket", "disabled"), LockType::Exclusive, "disabled-owner".to_owned());
assert!(
disabled
.acquire_lock(&disabled_request)
.await
.expect("disabled acquire bypass")
.success
);
assert!(
disabled
.release(&disabled_request.lock_id)
.await
.expect("disabled release bypass")
);
let rows = capture.all_rows();
let releases = rows
.iter()
.filter(|row| row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "released")
.collect::<Vec<_>>();
assert_eq!(
releases.len(),
4,
"duplicates and disabled guards cannot acknowledge another physical release"
);
let physical_mappings = rows
.iter()
.filter(|row| row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "mapped")
.collect::<Vec<_>>();
assert_eq!(physical_mappings.len(), 4);
assert!(releases.iter().all(|row| physical_mappings.iter().any(|mapped| {
mapped["fields"]["physical_guard_id"] == row["fields"]["physical_guard_id"]
&& mapped["fields"]["process_id"] == row["fields"]["process_id"]
&& peer_ids
.iter()
.any(|peer_id| mapped["fields"]["lock_id"] == peer_id.to_string())
})));
let dropped = capture.rows().pop().expect("guard drop event");
assert_eq!(dropped["fields"]["lock_id"], id.to_string());
assert!(
dropped["spans"]
.as_array()
.expect("retained request span")
.iter()
.any(|span| span["request_id"] == "original-multipart-request")
);
let manager = Arc::new(FastObjectLockManager::new());
let stale_client = LocalClient::with_manager(Arc::new(GlobalLockManager::Enabled(manager.clone())));
let stale_request = LockRequest::new(
ObjectKey::new("bucket", "externally-released"),
LockType::Exclusive,
"stale-owner".to_owned(),
);
assert!(
stale_client
.acquire_lock(&stale_request)
.await
.expect("real stale owner")
.success
);
assert_eq!(manager.force_unlock(&stale_request.resource), 1);
let mut successor = manager
.acquire_write_lock(stale_request.resource.clone(), "successor-owner")
.await
.expect("real successor guard");
assert!(
stale_client
.release(&stale_request.lock_id)
.await
.expect("retire stale client record")
);
assert_eq!(
manager
.get_lock_info(&stale_request.resource)
.expect("successor remains locked")
.owner
.as_ref(),
"successor-owner"
);
assert_eq!(
capture
.all_rows()
.iter()
.filter(
|row| row["fields"]["event"] == "namespace_lock_release" && row["fields"]["state"] == "released"
)
.count(),
4,
"a stale guard's failed physical release must not produce an ACK"
);
let successor_id = successor.guard_id();
assert!(successor.release(), "the actual successor release succeeds");
assert!(!successor.release(), "a duplicate explicit release cannot complete again");
drop(successor);
assert_eq!(
capture
.all_rows()
.iter()
.filter(|row| row["fields"]["event"] == "namespace_lock_release"
&& row["fields"]["state"] == "released"
&& row["fields"]["physical_guard_id"] == successor_id)
.count(),
1,
"explicit release followed by Drop records one physical completion"
);
});
});
});
}
fn namespace() -> NamespaceLockWrapper {
let manager = rustfs_lock::GlobalLockManager::Enabled(Arc::new(rustfs_lock::fast_lock::FastObjectLockManager::new()));
NamespaceLockWrapper::new(
+1 -44
View File
@@ -1235,13 +1235,7 @@ impl DiskHealthTracker {
}
pub(crate) fn metric_epoch_for_reconnect(&self) -> Self {
let health = Self::new();
// Explicit test faults stay attached to the drive across handle renewal.
#[cfg(test)]
if self.test_forced_offline.load(Ordering::Acquire) {
health.force_offline_for_test();
}
health
Self::new()
}
pub fn last_capacity_snapshot(&self) -> Option<(u64, u64, u64, u64)> {
@@ -2670,11 +2664,6 @@ impl DiskAPI for LocalDiskWrapper {
}
async fn is_online(&self) -> bool {
// A matching format does not make a handle usable while its I/O gate is faulty.
// Reconnect must renew it even if its old recovery monitor has stopped.
if self.health.is_faulty() {
return false;
}
let Ok(Some(disk_id)) = self.disk.get_disk_id().await else {
return false;
};
@@ -4706,38 +4695,6 @@ mod tests {
assert_eq!(snapshot.total_errors_availability, 1);
}
#[tokio::test]
async fn local_online_status_matches_io_health_and_disk_identity() {
let dir = tempfile::tempdir().expect("create a drive root");
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 root")).expect("local endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let meta_dir = dir.path().join(RUSTFS_META_BUCKET);
tokio::fs::create_dir_all(&meta_dir).await.expect("create metadata directory");
let mut format = FormatV3::new(1, 1);
format.erasure.this = format.erasure.sets[0][0];
tokio::fs::write(meta_dir.join(FORMAT_CONFIG_FILE), format.to_json().expect("serialize format"))
.await
.expect("persist original identity");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("open formatted drive"));
let wrapper = LocalDiskWrapper::new(disk, false);
assert!(wrapper.is_online().await, "healthy formatted drive is online");
wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Suspect);
assert!(wrapper.is_online().await, "suspect drives still admit foreground I/O");
wrapper.health.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
assert!(!wrapper.is_online().await, "a readable format cannot make a faulty handle usable");
assert!(!wrapper.health.mark_recovery_success(&endpoint, "test_probe"));
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Returning);
assert!(
!wrapper.is_online().await,
"returning drives retain the faulty I/O gate until recovery completes"
);
wrapper.reset_health_for_store_init_retry();
wrapper.set_disk_id_state(Some(Uuid::new_v4())).await;
assert!(!wrapper.is_online().await, "healthy state must not bypass a mismatched disk identity");
}
#[tokio::test]
async fn fresh_capacity_keeps_disk_identity_precheck_without_requiring_id_in_snapshot() {
let dir = tempfile::tempdir().expect("temp dir should be created");
+6 -18
View File
@@ -159,18 +159,8 @@ pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
std::fs::symlink_metadata(path)
}
pub fn create_dir_all_std(path: &Path) -> io::Result<()> {
match std::fs::create_dir(path) {
Ok(()) => Ok(()),
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => Ok(()),
Err(e) if e.kind() == io::ErrorKind::NotFound => std::fs::create_dir_all(path),
Err(e) => Err(e),
}
}
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
let path = path.as_ref().to_path_buf();
tokio::task::spawn_blocking(move || create_dir_all_std(&path)).await?
fs::create_dir_all(path.as_ref()).await
}
fn is_dir_error(e: &io::Error) -> bool {
@@ -220,7 +210,6 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
}
}
#[cfg(test)]
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
fs::create_dir(path.as_ref()).await
}
@@ -250,7 +239,7 @@ impl BucketExistenceCache {
}
}
fn check_exists(&self, path: &Path) -> Option<bool> {
fn check_exists(&self, path: &PathBuf) -> Option<bool> {
let mut cache = self.cache.lock().ok()?;
if let Some((timestamp, exists)) = cache.get(path) {
if timestamp.elapsed() < self.ttl {
@@ -267,7 +256,7 @@ impl BucketExistenceCache {
}
}
fn invalidate(&self, path: &Path) {
fn invalidate(&self, path: &PathBuf) {
if let Ok(mut cache) = self.cache.lock() {
cache.remove(path);
}
@@ -279,16 +268,15 @@ static BUCKET_EXISTENCE_CACHE: std::sync::LazyLock<BucketExistenceCache> =
/// Cached access check - reduces statx syscalls
pub async fn cached_access(path: impl AsRef<Path>) -> io::Result<()> {
let path = path.as_ref();
let path_buf = path.as_ref().to_path_buf();
if let Some(exists) = BUCKET_EXISTENCE_CACHE.check_exists(path) {
if let Some(exists) = BUCKET_EXISTENCE_CACHE.check_exists(&path_buf) {
if exists {
return Ok(());
}
return Err(io::Error::new(io::ErrorKind::NotFound, "bucket not found (cached)"));
}
let path_buf = path.to_path_buf();
let result = fs::metadata(&path_buf).await;
match &result {
@@ -304,7 +292,7 @@ pub async fn cached_access(path: impl AsRef<Path>) -> io::Result<()> {
}
pub fn invalidate_bucket_cache(path: impl AsRef<Path>) {
BUCKET_EXISTENCE_CACHE.invalidate(path.as_ref());
BUCKET_EXISTENCE_CACHE.invalidate(&path.as_ref().to_path_buf());
}
#[cfg(test)]
+108 -85
View File
@@ -115,7 +115,7 @@ use time::OffsetDateTime;
use tokio::fs::{self, File};
#[cfg(not(unix))]
use tokio::io::AsyncReadExt;
use tokio::io::{AsyncRead, AsyncSeekExt, AsyncWrite, ErrorKind, ReadBuf};
use tokio::io::{AsyncRead, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind, ReadBuf};
use tokio::sync::{Notify, RwLock, Semaphore};
use tokio::time::{Instant, Sleep, interval_at, timeout};
use tracing::{debug, error, info, warn};
@@ -301,10 +301,9 @@ fn read_all_data_std(
let size = usize::try_from(metadata.len()).map_err(|err| ReadAllError::Disk(DiskError::other(err)))?;
let mut bytes = Vec::new();
bytes
.try_reserve_exact(size.saturating_add(1))
.try_reserve_exact(size)
.map_err(|err| ReadAllError::Disk(Error::other(err)))?;
std::io::Read::read_to_end(&mut std::io::Read::take(&mut file, metadata.len()), &mut bytes)
.map_err(|err| ReadAllError::Disk(to_file_error(err).into()))?;
std::io::Read::read_to_end(&mut file, &mut bytes).map_err(|err| ReadAllError::Disk(to_file_error(err).into()))?;
let modtime = metadata.modified().ok().map(OffsetDateTime::from);
Ok((bytes, modtime))
@@ -8380,7 +8379,7 @@ impl LocalDisk {
return Err(DiskError::FileNotFound);
}
let meta_path = file_path.as_ref().join(STORAGE_FORMAT_FILE);
let meta_path = path_join(&[file_path.as_ref(), Path::new(STORAGE_FORMAT_FILE)]);
let res = {
if read_data {
@@ -8416,7 +8415,8 @@ impl LocalDisk {
}
#[hotpath::measure(impl_type = "LocalDisk")]
async fn read_metadata_with_dmtime(&self, file_path: PathBuf) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
async fn read_metadata_with_dmtime(&self, file_path: impl AsRef<Path>) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
check_path_length(file_path.as_ref().to_string_lossy().as_ref())?;
let io_permit = acquire_context_disk_io_permit().await.map_err(DiskError::Io)?;
// HP-12 item 1 (sub-change A): fold the open + fstat + bounded xl.meta
@@ -8429,7 +8429,9 @@ impl LocalDisk {
// - is_dir -> Error::FileNotFound (NOT to_file_error(EISDIR))
// - metadata failure -> to_file_error
// - parse failure -> propagated verbatim from read_xl_meta_no_data_sync (`?`)
let (data, modtime) = self.run_metadata_read(io_permit, file_path, read_metadata_only_std).await??;
let (data, modtime) = self
.run_metadata_read(io_permit, file_path.as_ref().to_path_buf(), read_metadata_only_std)
.await??;
Ok((data, modtime))
}
@@ -9460,64 +9462,101 @@ impl LocalDisk {
skip_parent
};
#[cfg(test)]
let is_owned = matches!(data, InternalBuf::Owned(_));
let buf = match data {
InternalBuf::Ref(slice) => Bytes::copy_from_slice(slice),
InternalBuf::Owned(bytes) => bytes,
};
let path = file_path.to_path_buf();
let create_parent = if let Some(parent) = path.parent()
&& parent != skip_parent
{
os::check_path_length(parent.to_string_lossy().as_ref())?;
!(!skip_parent.as_os_str().is_empty() && skip_parent.starts_with(parent))
} else {
false
};
let root = self.publication_root.clone();
let operation = move |is_cancelled: &dyn Fn() -> bool| {
if is_cancelled() {
return Err(std::io::Error::other("blocking namespace operation cancelled"));
}
if create_parent && let Some(parent) = path.parent() {
super::fs::create_dir_all_std(parent)?;
}
#[cfg(test)]
if is_owned {
run_owned_file_write_before_open(&path);
}
let (mut file, parent_handle) = if sync == SyncMode::FileAndDir {
root.create_truncate_with_parent(&path)?
} else {
(root.create_truncate(&path)?, None)
};
std::io::Write::write_all(&mut file, buf.as_ref())?;
if sync != SyncMode::None {
file.sync_data()?;
// FileOnly callers rename the file away, so the tmp
// directory entry never needs to become durable.
if sync == SyncMode::FileAndDir
&& let Some(parent) = path.parent()
{
os::fsync_directory_handle_std(parent, parent_handle.as_ref())?;
match data {
InternalBuf::Ref(buf) => {
let mut f = self.open_file(file_path, O_CREATE | O_WRONLY | O_TRUNC, skip_parent).await?;
f.write_all(buf).await.map_err(to_file_error)?;
if sync != SyncMode::None {
f.sync_data().await.map_err(to_file_error)?;
// Persist the directory entry too, so a freshly created file
// survives power loss along with its contents. Skipped for
// FileOnly: the caller renames the file away immediately.
if sync == SyncMode::FileAndDir
&& let Some(parent) = file_path.parent()
{
os::fsync_dir(parent).await.map_err(to_file_error)?;
}
}
}
Ok(())
};
let result = if sync == SyncMode::None {
self.blocking_queue.run_cancellable(operation).await
} else {
self.blocking_queue.run_fsync_cancellable(operation).await
};
result.map_err(to_file_error)?;
InternalBuf::Owned(buf) => {
let path = file_path.to_path_buf();
if let Some(parent) = path.parent()
&& parent != skip_parent
{
os::make_dir_all(parent, skip_parent).await?;
}
let root = self.publication_root.clone();
let operation = move |is_cancelled: &dyn Fn() -> bool| {
if is_cancelled() {
return Err(std::io::Error::other("blocking namespace operation cancelled"));
}
#[cfg(test)]
run_owned_file_write_before_open(&path);
let (mut file, parent_handle) = if sync == SyncMode::FileAndDir {
root.create_truncate_with_parent(&path)?
} else {
(root.create_truncate(&path)?, None)
};
std::io::Write::write_all(&mut file, buf.as_ref())?;
if sync != SyncMode::None {
file.sync_data()?;
// FileOnly callers rename the file away, so the tmp
// directory entry never needs to become durable.
if sync == SyncMode::FileAndDir
&& let Some(parent) = path.parent()
{
os::fsync_directory_handle_std(parent, parent_handle.as_ref())?;
}
}
Ok(())
};
let result = if sync == SyncMode::None {
self.blocking_queue.run_cancellable(operation).await
} else {
self.blocking_queue.run_fsync_cancellable(operation).await
};
result.map_err(to_file_error)?;
}
}
Ok(())
}
async fn open_file(&self, path: impl AsRef<Path>, mode: usize, skip_parent: impl AsRef<Path>) -> Result<File> {
let mut skip_parent = skip_parent.as_ref();
if skip_parent.as_os_str().is_empty() {
skip_parent = self.io_root();
}
if let Some(parent) = path.as_ref().parent()
&& parent != skip_parent
{
os::make_dir_all(parent, skip_parent).await?;
}
let f = if mode == O_CREATE | O_WRONLY | O_TRUNC {
let path = path.as_ref().to_path_buf();
let root = self.publication_root.clone();
File::from_std(
self.blocking_queue
.run_cancellable(move |is_cancelled| {
if is_cancelled() {
return Err(std::io::Error::other("blocking namespace operation cancelled"));
}
root.create_truncate(&path)
})
.await
.map_err(to_file_error)?,
)
} else {
super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?
};
Ok(f)
}
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
async fn open_file_read_only(&self, path: impl AsRef<Path>) -> Result<File> {
let f = super::fs::open_file(path.as_ref(), O_RDONLY).await.map_err(to_file_error)?;
@@ -10477,11 +10516,10 @@ async fn read_file_all(path: impl AsRef<Path>, root: &os::PublicationRoot) -> Re
let root = root.clone();
tokio::task::spawn_blocking(move || {
let result = (|| {
let file = root.open_readonly(&path)?;
let mut file = root.open_readonly(&path)?;
let metadata = file.metadata()?;
let size = usize::try_from(metadata.len()).map_err(std::io::Error::other)?;
let mut data = Vec::with_capacity(size.saturating_add(1));
std::io::Read::read_to_end(&mut std::io::Read::take(file, metadata.len()), &mut data)?;
let mut data = Vec::new();
std::io::Read::read_to_end(&mut file, &mut data)?;
Ok::<_, std::io::Error>((Bytes::from(data), metadata))
})();
result
@@ -10537,21 +10575,7 @@ fn local_disk_object_path(root: &Path, bucket: &str, key: &str) -> Result<PathBu
Ok(path)
}
#[cfg(not(windows))]
#[inline]
fn is_clean_relative_segment(s: &str) -> bool {
let b = s.as_bytes();
!matches!(b.first(), None | Some(b'/' | b'.')) && !b.ends_with(b"/") && !b.windows(2).any(|w| matches!(w, b"//" | b"/."))
}
fn build_local_disk_object_path(root: &Path, bucket: &str, key: &str) -> (PathBuf, PathBuf) {
#[cfg(not(windows))]
if is_clean_relative_segment(bucket) && is_clean_relative_segment(key) {
let bucket_path = root.join(bucket);
let path = bucket_path.join(key);
return (bucket_path, path);
}
let cache_key = if key.is_empty() {
bucket.to_string()
} else {
@@ -10716,10 +10740,9 @@ fn reject_local_disk_symlink_components(root: &Path, path: &Path) -> Result<()>
fn normalize_path_cow(path: &Path) -> std::borrow::Cow<'_, Path> {
// Windows prefixes retain the existing PathBuf reconstruction semantics.
#[cfg(not(windows))]
if !path.as_os_str().as_encoded_bytes().contains(&b'.')
|| !path
.components()
.any(|c| matches!(c, std::path::Component::ParentDir | std::path::Component::CurDir))
if !path
.components()
.any(|c| matches!(c, std::path::Component::ParentDir | std::path::Component::CurDir))
{
return std::borrow::Cow::Borrowed(path);
}
@@ -12439,12 +12462,12 @@ impl DiskAPI for LocalDisk {
let src_volume_dir = self.io_get_bucket_path(src_volume)?;
let dst_volume_dir = self.io_get_bucket_path(dst_volume)?;
if !skip_access_checks(src_volume) {
cached_access(&src_volume_dir)
access(&src_volume_dir)
.await
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
}
if !skip_access_checks(dst_volume) {
cached_access(&dst_volume_dir)
access(&dst_volume_dir)
.await
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
}
@@ -12523,7 +12546,7 @@ impl DiskAPI for LocalDisk {
if !origvolume.is_empty() {
let origvolume_dir = self.io_get_bucket_path(origvolume)?;
if !skip_access_checks(origvolume) {
cached_access(origvolume_dir)
access(origvolume_dir)
.await
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
}
@@ -13153,7 +13176,7 @@ impl DiskAPI for LocalDisk {
if !org_volume.is_empty() {
let org_volume_path = self.io_get_bucket_path(org_volume)?;
if !skip_access_checks(org_volume) {
cached_access(&org_volume_path)
access(&org_volume_path)
.await
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
}
@@ -6,7 +6,6 @@ use super::{
LocalDisk,
multipart_source_state::{
LoadedSource, SourceHead, SourceState, SourceStore, decode, encode, load_source, raw_sha, read_optional,
read_optional_in_pinned,
},
};
use crate::disk::{
@@ -128,6 +127,20 @@ pub(super) fn validate_retired_head(
}
impl SourceStore {
pub async fn root_missing(&self) -> Result<bool> {
let source = self.source.clone();
let base = self.base.clone();
let root = self.publication_root.clone();
os::run_blocking_namespace_operation(self.lease.clone(), move || {
match os::pin_multipart_source_directory(&source, &base, &root) {
Ok(_) => Ok(false),
Err(error) if error.kind() == ErrorKind::NotFound => Ok(true),
Err(error) => Err(error),
}
})
.await
.map_err(DiskError::from)
}
fn verify_abort_cohort(&self, certificate: &MultipartAbortCertificate) -> Result<()> {
certificate.verify_with_authority(&self.authority)?;
if certificate.disk_ids != self.disk_ids
@@ -252,7 +265,7 @@ impl SourceStore {
let certificate = self.read_abort_certificate(&loaded).await?;
Ok(MultipartSourceInspection {
disk_id: self.disk_id,
root_missing: loaded.root_missing,
root_missing: self.root_missing().await?,
abort_round: loaded
.state
.as_ref()
@@ -607,17 +620,12 @@ pub(super) struct OrdinaryMultipartDeleteCheck {
}
impl OrdinaryMultipartDeleteCheck {
pub(super) fn validate(&self) -> Result<()> {
let pinned = match os::pin_multipart_source_directory(&self.source, &self.base, &self.root) {
Ok(pinned) => pinned,
Err(error) if error.kind() == ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error.into()),
};
if let Some(raw) = read_optional_in_pinned(&pinned, &self.source, STATE, &self.base, &self.root)? {
if let Some(raw) = read_optional(&self.source.join(STATE), &self.base, &self.root)? {
let state: SourceState = decode(&raw)?;
if state.abort.is_some() || state.phase == MultipartSourcePhase::Aborting {
return Err(DiskError::ConditionalDeleteConflict);
}
} else if os::multipart_source_has_native_controls_in_pinned(&self.source, &pinned)? {
} else if os::multipart_source_has_native_controls(&self.source, &self.base, &self.root)? {
return Err(DiskError::ConditionalDeleteConflict);
}
Ok(())
@@ -170,7 +170,7 @@ impl LocalDisk {
recovery: loaded.state.as_ref().and_then(|state| state.recovery.clone()),
negative_certificate,
ready_certificate,
root_missing: loaded.root_missing,
root_missing: store.root_missing().await?,
abort_round: loaded
.state
.as_ref()
@@ -106,7 +106,6 @@ impl SourceHead {
}
pub(super) struct LoadedSource {
pub root_missing: bool,
pub state: Option<SourceState>,
pub head: Option<SourceHead>,
pub original: Option<Bytes>,
@@ -225,31 +224,11 @@ pub(super) fn read_optional(path: &Path, base: &Path, root: &os::PublicationRoot
Err(error) => Err(error.into()),
}
}
pub(super) fn read_optional_in_pinned(
pinned: &os::ExistingBaseDirectoryGuard,
parent: &Path,
name: &str,
base: &Path,
root: &os::PublicationRoot,
) -> Result<Option<Bytes>> {
match os::read_multipart_source_metadata_in_pinned(pinned, parent, std::ffi::OsStr::new(name), base, root) {
Ok(bytes) => Ok(Some(Bytes::from(bytes))),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(error) => Err(error.into()),
}
}
/// Canonical source mutations are admitted under SourceRW at the syscall,
/// not on the earlier async caller that may have timed out.
pub(super) fn validate_source_mutation(source: &Path, base: &Path, root: &os::PublicationRoot, initial: bool) -> Result<()> {
let pinned = match os::pin_multipart_source_directory(source, base, root) {
Ok(pinned) => pinned,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
return if initial { Ok(()) } else { Err(DiskError::FileNotFound) };
}
Err(error) => return Err(error.into()),
};
let state = read_optional_in_pinned(&pinned, source, STATE, base, root)?;
let head = read_optional_in_pinned(&pinned, source, STORAGE_FORMAT_FILE, base, root)?;
let state = read_optional(&source.join(STATE), base, root)?;
let head = read_optional(&source.join(STORAGE_FORMAT_FILE), base, root)?;
let Some(head) = head else {
return if initial && state.is_none() {
Ok(())
@@ -369,9 +348,6 @@ impl SourceStore {
.map_err(DiskError::from)?
}
pub async fn persist(&self, state: &SourceState) -> Result<()> {
self.persist_with_source_sync(state, false).await
}
async fn persist_with_source_sync(&self, state: &SourceState, sync_source_before: bool) -> Result<()> {
state.validate(self.disk_id, &self.path)?;
let raw = encode(state)?;
let source = self.source.clone();
@@ -389,9 +365,6 @@ impl SourceStore {
});
os::run_blocking_namespace_operation(self.lease.clone(), move || {
let _pin = os::pin_multipart_source_directory(&source, &base, &root)?;
if sync_source_before {
os::fsync_dir_std(&source)?;
}
if let Some(path) = terminal {
os::replace_multipart_source_metadata(&path, &raw, &base, &root)?;
}
@@ -400,7 +373,6 @@ impl SourceStore {
.await
.map_err(DiskError::from)
}
#[cfg(test)]
pub async fn original(&self, state: &SourceState) -> Result<Bytes> {
let path = original_path(&self.source, state.backup_owner);
let base = self.base.clone();
@@ -518,7 +490,7 @@ impl SourceStore {
if encode(actual)? != encode(state)? {
return Err(DiskError::ConditionalDeleteConflict);
}
let raw = loaded.original.as_ref().ok_or(DiskError::FileCorrupt)?;
let raw = self.original(state).await?;
let allowed = match loaded.head.as_ref() {
None => {
state.phase == MultipartSourcePhase::Recovering
@@ -579,7 +551,7 @@ impl SourceStore {
if !allowed {
return Err(DiskError::ConditionalDeleteConflict);
}
let mut info = file_info(raw, RUSTFS_META_MULTIPART_BUCKET, &self.path, state.lineage.original_version_id)?;
let mut info = file_info(&raw, RUSTFS_META_MULTIPART_BUCKET, &self.path, state.lineage.original_version_id)?;
strip_native_controls(&mut info.metadata);
mark_native_source(&mut info);
set_source_status(
@@ -800,32 +772,13 @@ pub(super) fn load_source(
physical_slot: usize,
disk_ids: &[Uuid],
) -> Result<LoadedSource> {
let pinned = match os::pin_multipart_source_directory(source, base, root) {
Ok(pinned) => pinned,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
return Ok(LoadedSource {
root_missing: true,
state: None,
head: None,
original: None,
info: None,
snapshot: None,
head_matches_state: false,
head_matches_activation: false,
});
}
Err(error) => return Err(error.into()),
};
let state_raw = read_optional_in_pinned(&pinned, source, STATE, base, root)?;
let state_raw = read_optional(&source.join(STATE), base, root)?;
let state: Option<SourceState> = state_raw.as_deref().map(decode).transpose()?;
if state.is_none() && os::multipart_source_has_native_controls_in_pinned(source, &pinned)? {
if state.is_none() && os::multipart_source_has_native_controls(source, base, root)? {
return Err(DiskError::ConditionalDeleteConflict);
}
let head = if let Some(state) = state
.as_ref()
.filter(|state| state.prepared_length.is_some_and(|len| len > 64 * 1024))
{
let head_path = source.join(STORAGE_FORMAT_FILE);
let head_path = source.join(STORAGE_FORMAT_FILE);
let head = if let Some(state) = state.as_ref() {
let maximum = state
.prepared_length
.unwrap_or(0)
@@ -836,8 +789,7 @@ pub(super) fn load_source(
let raw = if is_proxy {
None
} else {
let raw = read_optional_in_pinned(&pinned, source, STORAGE_FORMAT_FILE, base, root)?
.ok_or(DiskError::FileCorrupt)?;
let raw = read_optional(&head_path, base, root)?.ok_or(DiskError::FileCorrupt)?;
if raw_sha(&raw) != sha256 {
return Err(DiskError::FileCorrupt);
}
@@ -849,17 +801,10 @@ pub(super) fn load_source(
Err(error) => return Err(error.into()),
}
} else {
read_optional_in_pinned(&pinned, source, STORAGE_FORMAT_FILE, base, root)?.map(|raw| {
let sha256 = raw_sha(&raw);
let length = raw.len() as u64;
let is_proxy = state
.as_ref()
.is_some_and(|state| Some(sha256) == state.prepared_sha256 && Some(length) == state.prepared_length);
SourceHead {
sha256,
length,
raw: (!is_proxy).then_some(raw),
}
read_optional(&head_path, base, root)?.map(|raw| SourceHead {
sha256: raw_sha(&raw),
length: raw.len() as u64,
raw: Some(raw),
})
};
let mut original = None;
@@ -1005,7 +950,6 @@ pub(super) fn load_source(
head_matches_state = true;
}
Ok(LoadedSource {
root_missing: false,
state,
head,
original,
@@ -1643,8 +1587,10 @@ impl NativeMultipartPublication {
}
}
self.store.sync_target_with_lease(&self.publication, target_lease).await?;
let source = self.store.source.clone();
os::run_blocking_namespace_operation(self.store.lease.clone(), move || os::fsync_dir_std(&source)).await?;
state.advance(MultipartSourcePhase::ConsumedUnsealed)?;
self.store.persist_with_source_sync(&state, true).await?;
self.store.persist(&state).await?;
*self.state.lock() = state;
let loaded = self.store.load().await?;
let snapshot = loaded.snapshot.ok_or(DiskError::FileCorrupt)?;
@@ -411,23 +411,15 @@ impl LocalDisk {
}
let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
if incoming.data.is_none() && !incoming.is_remote() {
let source_data_dir = source_object.join(source_dir.to_string());
let pinned_source_data = (!incoming.parts.is_empty())
.then(|| os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root))
.transpose()?;
let mut names = Vec::with_capacity(incoming.parts.len().saturating_mul(3));
for part in &incoming.parts {
if cancelled() {
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
}
let meta_name = format!("part.{}.meta", part.number);
let bytes = os::read_multipart_source_metadata_in_pinned(
pinned_source_data.as_ref().expect("pinned source data directory"),
&source_data_dir,
std::ffi::OsStr::new(&meta_name),
&base,
&publication_root,
)?;
let path = source_object
.join(source_dir.to_string())
.join(format!("part.{}.meta", part.number));
let bytes = os::read_multipart_source_metadata(&path, &base, &publication_root)?;
let actual = rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?;
let normalized = i64::try_from(actual.size)
.map_err(std::io::Error::other)?
@@ -449,13 +441,13 @@ impl LocalDisk {
// Preserve the native completed-MPU directory format. This
// immutable, validated generation record is distinct from
// the upload's mutable top-level xl.meta.
names.push(meta_name);
names.push(format!("part.{}.meta", part.number));
if let Some(integrity) = &part.integrity {
names.push(integrity.file_name());
}
}
os::hardlink_multipart_source_files(
&source_data_dir,
&source_object.join(source_dir.to_string()),
&private_object.join(final_dir.to_string()),
&names,
&base,
+64 -135
View File
@@ -73,10 +73,6 @@ pub fn check_path_length(path_name: &str) -> Result<()> {
return Err(DiskError::FileAccessDenied);
}
if path_name.len() <= 255 {
return Ok(());
}
// Check each path segment length is > 255 on all Unix
// platforms, look for this value as NAME_MAX in
// /usr/include/linux/limits.h
@@ -3372,29 +3368,6 @@ fn open_existing_directory_below_base_std(
use rustix::fs::{Mode, OFlags, openat};
let relative = existing_base_directory_suffix(directory, base)?;
#[cfg(target_os = "linux")]
{
use rustix::fs::{ResolveFlags, openat2};
use std::os::fd::AsFd;
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
let owned_base;
let base_fd = if base == publication_root.io_path.as_path() || base == publication_root.path.as_path() {
publication_root.directory.as_fd()
} else {
owned_base = std::os::fd::OwnedFd::from(publication_root.open_directory(base)?);
owned_base.as_fd()
};
let target = if relative.as_os_str().is_empty() {
Path::new(".")
} else {
relative
};
match openat2(base_fd, target, flags, Mode::empty(), ResolveFlags::BENEATH | ResolveFlags::NO_SYMLINKS) {
Ok(leaf) => return Ok(vec![leaf]),
Err(rustix::io::Errno::NOSYS) => {}
Err(err) => return Err(err.into()),
}
}
#[cfg(target_os = "linux")]
let base = publication_root.open_directory(base)?.into();
#[cfg(not(target_os = "linux"))]
let base = {
@@ -4948,16 +4921,12 @@ impl LinuxCommitDirectories {
use std::os::unix::fs::MetadataExt;
// A pinned directory may have been unlinked or replaced. Do not report a
// successful publication into a detached object tree.
let current = self.root.open_directory(&self.source_path)?.metadata()?;
if (current.dev(), current.ino()) != (self.source.device, self.source.inode) {
return Err(io::Error::new(io::ErrorKind::InvalidData, "commit directory identity changed"));
}
let destination = self.prepare_destination()?;
if self.source_path != self.destination_path
|| (self.source.device, self.source.inode) != (destination.device, destination.inode)
{
let current = self.root.open_directory(&self.destination_path)?.metadata()?;
if (current.dev(), current.ino()) != (destination.device, destination.inode) {
for (path, directory) in [
(&self.source_path, &self.source),
(&self.destination_path, self.prepare_destination()?),
] {
let current = self.root.open_directory(path)?.metadata()?;
if (current.dev(), current.ino()) != (directory.device, directory.inode) {
return Err(io::Error::new(io::ErrorKind::InvalidData, "commit directory identity changed"));
}
}
@@ -5250,20 +5219,12 @@ pub(crate) fn prepare_rename_commit_guard(
existing_base_directory_suffix(destination_parent, destination_base)?;
let source = LinuxCommitDirectory::new(publication_root.open_directory(source_parent)?)?;
let destination = std::sync::OnceLock::new();
if source_parent == destination_parent {
let _ = destination.set(LinuxCommitDirectory {
file: Arc::clone(&source.file),
device: source.device,
inode: source.inode,
});
} else {
match publication_root.open_directory(destination_parent) {
Ok(directory) => {
let _ = destination.set(LinuxCommitDirectory::new(directory)?);
}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => return Err(err),
match publication_root.open_directory(destination_parent) {
Ok(directory) => {
let _ = destination.set(LinuxCommitDirectory::new(directory)?);
}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => return Err(err),
}
Ok(RenameCommitGuard {
directories: Arc::new(LinuxCommitDirectories {
@@ -6123,20 +6084,12 @@ fn existing_base_directory_suffix<'a>(dir_path: &'a Path, base_dir: &Path) -> io
/// Missing state never downgrades surviving native control records to legacy.
/// Inspect only this pinned upload root; parts and proof trees are not traversed.
#[cfg(test)]
pub(crate) fn multipart_source_has_native_controls(source: &Path, base: &Path, root: &PublicationRoot) -> io::Result<bool> {
let pin = match pin_multipart_source_directory(source, base, root) {
let _pin = match pin_multipart_source_directory(source, base, root) {
Ok(pin) => pin,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false),
Err(error) => return Err(error),
};
multipart_source_has_native_controls_in_pinned(source, &pin)
}
pub(crate) fn multipart_source_has_native_controls_in_pinned(
source: &Path,
_pin: &ExistingBaseDirectoryGuard,
) -> io::Result<bool> {
let mut entries = std::fs::read_dir(source)?;
for _ in 0..READ_DIR_PROBE_RAW_LIMIT {
let Some(entry) = entries.next() else {
@@ -6440,10 +6393,15 @@ pub(crate) fn sync_multipart_owned_record_parents(
if !directory.starts_with(source) {
return Err(io::Error::new(io::ErrorKind::InvalidInput, "native record barrier escaped source"));
}
let mut pinned = pin_multipart_source_directory(directory, base, publication_root)?;
let pinned = pin_multipart_source_directory(directory, base, publication_root)?;
#[cfg(unix)]
{
let opened = std::fs::File::from(pinned.pop().ok_or_else(|| io::Error::other("empty native directory guard"))?);
let opened = std::fs::File::from(
pinned
.last()
.ok_or_else(|| io::Error::other("empty native directory guard"))?
.try_clone()?,
);
fsync_directory_handle_std(directory, Some(&opened))?;
}
#[cfg(not(unix))]
@@ -6555,16 +6513,6 @@ pub(crate) fn read_multipart_source_metadata(
read_multipart_metadata(path, base, publication_root, Some(16 * 1024 * 1024))
}
pub(crate) fn read_multipart_source_metadata_in_pinned(
pinned: &ExistingBaseDirectoryGuard,
parent: &Path,
name: &std::ffi::OsStr,
base: &Path,
publication_root: &PublicationRoot,
) -> io::Result<Vec<u8>> {
read_multipart_metadata_in_pinned(pinned, parent, name, base, publication_root, Some(16 * 1024 * 1024))
}
/// Target FI histories have no source-control size limit. Keep the same
/// pinned-parent and no-follow checks when inspecting every referenced DD.
pub(crate) fn read_multipart_reclaim_target_metadata(
@@ -6575,71 +6523,22 @@ pub(crate) fn read_multipart_reclaim_target_metadata(
read_multipart_metadata(path, base, publication_root, None)
}
fn read_multipart_metadata_bytes(
mut reader: impl std::io::Read,
size_hint: Option<u64>,
maximum: Option<u64>,
) -> io::Result<Vec<u8>> {
if let (Some(len), Some(maximum)) = (size_hint, maximum)
&& len > maximum
{
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart metadata exceeds its size limit"));
}
let cap = size_hint
.and_then(|len| usize::try_from(len).ok())
.map_or(0, |len| len.saturating_add(1));
let mut bytes = Vec::with_capacity(cap);
match (size_hint, maximum) {
(Some(len), _) => {
std::io::Read::read_to_end(&mut reader.take(len), &mut bytes)?;
}
(None, Some(maximum)) => {
fn read_multipart_metadata_bytes(mut reader: impl std::io::Read, maximum: Option<u64>) -> io::Result<Vec<u8>> {
let mut bytes = Vec::new();
match maximum {
Some(maximum) => {
std::io::Read::read_to_end(&mut reader.take(maximum.saturating_add(1)), &mut bytes)?;
if bytes.len() as u64 > maximum {
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart metadata exceeds its size limit"));
}
}
(None, None) => {
None => {
std::io::Read::read_to_end(&mut reader, &mut bytes)?;
}
}
Ok(bytes)
}
fn read_multipart_metadata_in_pinned(
pinned: &ExistingBaseDirectoryGuard,
parent: &Path,
name: &std::ffi::OsStr,
base: &Path,
publication_root: &PublicationRoot,
maximum: Option<u64>,
) -> io::Result<Vec<u8>> {
#[cfg(unix)]
{
use rustix::fs::{Mode, OFlags, openat};
let _ = (parent, base, publication_root);
let directory = pinned
.last()
.ok_or_else(|| io::Error::other("missing multipart source directory"))?;
let mut file = std::fs::File::from(openat(
directory,
name,
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
Mode::empty(),
)?);
let metadata = file.metadata()?;
if !metadata.is_file() {
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart metadata is not a regular file"));
}
read_multipart_metadata_bytes(&mut file, Some(metadata.len()), maximum)
}
#[cfg(not(unix))]
{
let _ = pinned;
read_multipart_metadata(&parent.join(name), base, publication_root, maximum)
}
}
fn read_multipart_metadata(
path: &Path,
base: &Path,
@@ -6648,14 +6547,27 @@ fn read_multipart_metadata(
) -> io::Result<Vec<u8>> {
#[cfg(unix)]
{
use rustix::fs::{Mode, OFlags, openat};
let parent = path
.parent()
.ok_or_else(|| io::Error::other("multipart metadata has no parent"))?;
let guards = open_existing_directory_below_base_std(parent, base, publication_root)?;
let directory = guards
.last()
.ok_or_else(|| io::Error::other("missing multipart source directory"))?;
let name = path
.file_name()
.ok_or_else(|| io::Error::other("multipart metadata has no name"))?;
let guards = open_existing_directory_below_base_std(parent, base, publication_root)?;
read_multipart_metadata_in_pinned(&guards, parent, name, base, publication_root, maximum)
let mut file = std::fs::File::from(openat(
directory,
name,
OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
Mode::empty(),
)?);
if !file.metadata()?.is_file() {
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart metadata is not a regular file"));
}
read_multipart_metadata_bytes(&mut file, maximum)
}
#[cfg(windows)]
{
@@ -6686,7 +6598,7 @@ fn read_multipart_metadata(
if windows_file_attribute_tag(&file)?.FileAttributes & (FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT) != 0 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "multipart metadata is not a regular file"));
}
read_multipart_metadata_bytes(file.as_file_mut(), None, maximum)
read_multipart_metadata_bytes(file.as_file_mut(), maximum)
}
#[cfg(not(any(unix, windows)))]
{
@@ -7322,16 +7234,33 @@ pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Pat
/// This function will not create directories under base_dir.
#[tracing::instrument(level = "debug", skip_all)]
pub async fn os_mkdir_all(dir_path: impl AsRef<Path>, base_dir: impl AsRef<Path>) -> io::Result<()> {
let dir_path = dir_path.as_ref();
let base_dir = base_dir.as_ref();
if !base_dir.as_os_str().is_empty() && base_dir.starts_with(dir_path) {
if !base_dir.as_ref().to_string_lossy().is_empty() && base_dir.as_ref().starts_with(dir_path.as_ref()) {
return Ok(());
}
if let Err(err) = super::fs::make_dir_all(dir_path).await
&& err.kind() != io::ErrorKind::AlreadyExists
{
return Err(err);
if let Err(e) = super::fs::mkdir(dir_path.as_ref()).await {
if e.kind() == io::ErrorKind::AlreadyExists {
return Ok(());
}
if e.kind() != io::ErrorKind::NotFound {
return Err(e);
}
if let Some(parent) = dir_path.as_ref().parent() {
// Fall back to creating the missing parent chain only when the direct mkdir proves it is required.
if let Err(parent_err) = super::fs::make_dir_all(parent).await
&& parent_err.kind() != io::ErrorKind::AlreadyExists
{
return Err(parent_err);
}
}
if let Err(retry_err) = super::fs::mkdir(dir_path.as_ref()).await
&& retry_err.kind() != io::ErrorKind::AlreadyExists
{
return Err(retry_err);
}
}
Ok(())
@@ -603,6 +603,10 @@ pub(in crate::set_disk) async fn submit_read_repair_heal_with_submitter(
request.object_version_id = version_id.filter(|value| !value.is_empty()).map(str::to_string);
// A degraded read's surviving minority is not a recreate instruction; the
// locked storage read decides whether the target is absent (backlog#2900).
// Reader failures include checksum damage that metadata-only healing cannot detect.
if matches!(reason, "missing_shards" | "decode_error") {
request.scan_mode = Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep);
}
let request_id = request.id.clone();
let bucket = bucket.to_string();
@@ -5232,10 +5236,14 @@ impl SetDisks {
let disks = self.get_disks_internal().await;
let transaction_path = part_transaction_path(dst_object);
let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
let rollback_path = format!("{transaction_path}/{PART_TRANSACTION_ROLLBACK}");
let current_meta_path = format!("{dst_object}.meta");
let reads = disks.iter().map(|disk| {
let disk = disk.clone();
let transaction_meta_path = transaction_meta_path.clone();
let rollback_path = rollback_path.clone();
let current_meta_path = current_meta_path.clone();
async move {
let Some(disk) = disk else {
return PartTransactionObservation {
@@ -5245,20 +5253,44 @@ impl SetDisks {
err: Some(DiskError::DiskNotFound),
};
};
let (transaction_meta, err) = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &transaction_meta_path).await {
Ok(meta) => (Some(meta), None),
Err(DiskError::FileNotFound) => (None, None),
Err(err) => (None, Some(err)),
let transaction_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &transaction_meta_path).await {
Ok(meta) => Some(meta),
Err(DiskError::FileNotFound) => None,
Err(err) => {
return PartTransactionObservation {
transaction_meta: None,
current_meta: None,
rollback: false,
err: Some(err),
};
}
};
let rollback = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &rollback_path).await {
Ok(_) => true,
Err(DiskError::FileNotFound) => false,
Err(err) => {
return PartTransactionObservation {
transaction_meta,
current_meta: None,
rollback: false,
err: Some(err),
};
}
};
let current_meta = match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &current_meta_path).await {
Ok(meta) => Some(meta),
Err(DiskError::FileNotFound | DiskError::DiskNotFound) => None,
Err(_) => None,
};
PartTransactionObservation {
transaction_meta,
current_meta: None,
rollback: false,
err,
current_meta,
rollback,
err: None,
}
}
});
let mut observations = join_all(reads).await;
let observations = join_all(reads).await;
let read_errs = observations
.iter()
.map(|observation| observation.err.clone())
@@ -5275,42 +5307,6 @@ impl SetDisks {
return Ok(false);
}
let rollback_path = format!("{transaction_path}/{PART_TRANSACTION_ROLLBACK}");
let current_meta_path = format!("{dst_object}.meta");
let details = disks.iter().zip(observations.iter_mut()).map(|(disk, observation)| {
let disk = disk.clone();
let rollback_path = rollback_path.clone();
let current_meta_path = current_meta_path.clone();
async move {
if observation.err.is_some() {
return;
}
let Some(disk) = disk else {
observation.err = Some(DiskError::DiskNotFound);
return;
};
match disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &rollback_path).await {
Ok(_) => observation.rollback = true,
Err(DiskError::FileNotFound) => {}
Err(err) => {
observation.err = Some(err);
return;
}
}
if let Ok(meta) = disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &current_meta_path).await {
observation.current_meta = Some(meta);
}
}
});
join_all(details).await;
let read_errs = observations
.iter()
.map(|observation| observation.err.clone())
.collect::<Vec<_>>();
if let Some(err) = reduce_write_quorum_errs(&read_errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
return Err(err);
}
let mut current_counts: HashMap<Bytes, usize> = HashMap::new();
let mut transaction_meta_values = HashSet::new();
for observation in observations.iter().filter(|observation| observation.err.is_none()) {
@@ -13588,6 +13584,61 @@ mod tests {
release_read_repair_heal_reservation(&key).await;
}
#[tokio::test]
async fn read_repair_reader_failure_requests_deep_scan_for_the_exact_version() {
fn captured() -> &'static std::sync::Mutex<Option<rustfs_heal_contracts::heal_channel::HealChannelRequest>> {
static REQUEST: OnceLock<std::sync::Mutex<Option<rustfs_heal_contracts::heal_channel::HealChannelRequest>>> =
OnceLock::new();
REQUEST.get_or_init(|| std::sync::Mutex::new(None))
}
fn capture(request: rustfs_heal_contracts::heal_channel::HealChannelRequest) -> ReadRepairAdmissionFuture {
*captured().lock().expect("read-repair request capture must be available") = Some(request);
Box::pin(async { ReadRepairAdmissionOutcome::Response(HealAdmissionResult::Accepted) })
}
for (reason, expected_scan) in [
("missing_shards", Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep)),
("decode_error", Some(rustfs_heal_contracts::heal_channel::HealScanMode::Deep)),
("metadata_read_error", None),
] {
let object = format!("reader-failure-{}", Uuid::new_v4());
let version = Uuid::new_v4().to_string();
submit_read_repair_heal_with_submitter(
ReadRepairHealSubmission {
bucket: "reader-failure",
object: &object,
version_id: Some(&version),
pool_index: 1,
set_index: 2,
part_number: Some(1),
reason,
mrf_intent: None,
},
capture,
)
.await;
let request = tokio::time::timeout(Duration::from_secs(5), async {
loop {
if let Some(request) = captured().lock().expect("captured request must be readable").take() {
break request;
}
tokio::task::yield_now().await;
}
})
.await
.expect("the actual asynchronous admission must be captured");
assert_eq!(request.scan_mode, expected_scan, "reader damage needs payload verification");
assert_eq!(request.source, HealRequestSource::ReadRepair);
assert_eq!(request.bucket, "reader-failure");
assert_eq!(request.object_prefix.as_deref(), Some(object.as_str()));
assert_eq!(request.object_version_id.as_deref(), Some(version.as_str()));
assert_eq!((request.pool_index, request.set_index), (Some(1), Some(2)));
assert_eq!(request.recreate_missing, None, "storage evidence decides absence");
let key = ReadRepairHealCacheKey::new("reader-failure", &object, Some(&version), 1, 2);
release_read_repair_heal_reservation(&key).await;
}
}
// ========================================================================
// backlog#898 — groups A/C: pure old-data-dir cleanup classification.
// ========================================================================
@@ -36,12 +36,13 @@ use crate::disk::{
self, BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, Disk, DiskAPI,
};
use crate::set_disk::{
DiskError, DiskStore, FileInfo, HashAlgorithm, ReadOptions, SetDisks, can_try_inline_data_shards_direct,
codec_streaming_rollout_applies, collect_inline_data_shard_fileinfos_from_observations, file_info_is_valid_for_metadata,
get_metadata_slowtail_fault_request, inline_erasure_shard_file_offset, inline_erasure_shard_size,
is_get_metadata_data_read_early_stop_enabled, is_get_metadata_early_stop_bounded_fanout_enabled,
is_get_metadata_early_stop_enabled, is_get_metadata_non_inline_data_read_early_stop_enabled, is_version_early_stop_enabled,
object_fits_single_block,
DiskError, DiskStore, FileInfo, HashAlgorithm, ReadOptions, SetDisks, build_inline_bitrot_readers_from_refs,
can_try_inline_data_shards_direct, codec_streaming_rollout_applies, coding,
collect_inline_data_shard_fileinfos_from_observations, file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request,
inline_erasure_shard_file_offset, inline_erasure_shard_size, is_get_metadata_data_read_early_stop_enabled,
is_get_metadata_early_stop_bounded_fanout_enabled, is_get_metadata_early_stop_enabled,
is_get_metadata_non_inline_data_read_early_stop_enabled, is_version_early_stop_enabled, object_fits_single_block,
try_read_inline_data_shards_direct,
};
use futures::future::join_all;
use metrics::counter;
@@ -124,7 +125,6 @@ fn metadata_metrics_path(bucket: &str) -> &'static str {
}
}
#[cfg(test)]
pub(super) fn metadata_distribution_key(bucket: &str, object: &str) -> String {
[bucket, object].join("/")
}
@@ -449,16 +449,36 @@ pub(in crate::set_disk) fn bounded_metadata_fanout_order(
total_disks: usize,
default_parity_count: usize,
) -> Vec<usize> {
let fallback_order = || (0..total_disks).collect::<Vec<_>>();
if default_parity_count == 0 || default_parity_count >= total_disks {
return (0..total_disks).collect();
return fallback_order();
}
let data_blocks = total_disks - default_parity_count;
let start = FileInfo::distribution_start(bucket, object, total_disks);
let offset = (start + 1) % total_disks;
let distribution_key = metadata_distribution_key(bucket, object);
let distribution = FileInfo::new(&distribution_key, data_blocks, default_parity_count)
.erasure
.distribution;
if distribution.len() != total_disks {
return fallback_order();
}
let mut order = Vec::with_capacity(total_disks);
order.extend((0..data_blocks).map(|i| (total_disks + i - offset) % total_disks));
order.extend((0..total_disks).filter(|&disk_index| (start + disk_index + 1) % total_disks >= data_blocks));
for block_index in 1..=data_blocks {
let Some(disk_index) = distribution
.iter()
.position(|distributed_block| *distributed_block == block_index)
else {
return fallback_order();
};
order.push(disk_index);
}
order.extend(
distribution
.iter()
.enumerate()
.filter_map(|(disk_index, block_index)| (*block_index > data_blocks).then_some(disk_index)),
);
order
}
@@ -577,9 +597,6 @@ impl MetadataFanoutDiagnostics {
}
pub(in crate::set_disk) fn record(&self, path: &'static str) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
rustfs_io_metrics::record_get_object_metadata_fanout_duration(path, self.fanout_duration.as_secs_f64());
if let Some(latency) = self.first_response_latency() {
rustfs_io_metrics::record_get_object_first_metadata_response_latency(path, latency.as_secs_f64());
@@ -603,9 +620,6 @@ impl MetadataFanoutDiagnostics {
}
pub(in crate::set_disk) fn record_quorum_candidate_latency(&self, path: &'static str, read_quorum: usize) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
if let Some(latency) = self.quorum_candidate_latency(read_quorum) {
rustfs_io_metrics::record_get_object_metadata_quorum_result(path, "reached");
rustfs_io_metrics::record_get_object_quorum_reached_latency(path, latency.as_secs_f64());
@@ -615,14 +629,15 @@ impl MetadataFanoutDiagnostics {
#[cfg(test)]
pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
_bucket: &str,
_object: &str,
bucket: &str,
object: &str,
candidate: &FileInfo,
parts_metadata: &[FileInfo],
disks: &[Option<DiskStore>],
) -> Option<&'static str> {
verify_inline_body_from_observations(
_object,
bucket,
object,
candidate,
parts_metadata
.iter()
@@ -630,11 +645,13 @@ pub(in crate::set_disk) async fn data_read_early_stop_inline_body_miss_reason(
.map(|(index, metadata)| (index, Some(metadata))),
disks,
)
.await
.err()
}
fn verify_inline_body_from_observations<'a>(
_object: &str,
async fn verify_inline_body_from_observations<'a>(
bucket: &str,
object: &str,
candidate: &FileInfo,
observations: impl IntoIterator<Item = (usize, Option<&'a FileInfo>)>,
disks: &[Option<DiskStore>],
@@ -643,18 +660,21 @@ fn verify_inline_body_from_observations<'a>(
return Err(reason);
}
let data_shards = candidate.erasure.data_blocks;
if data_shards == 0
|| candidate.erasure.block_size == 0
|| (candidate.erasure.parity_blocks > 0
&& candidate.uses_legacy_checksum
&& !reed_solomon_simd::ReedSolomonEncoder::supports(data_shards, candidate.erasure.parity_blocks))
{
let Ok(erasure) = coding::Erasure::try_new_with_options(
candidate.erasure.data_blocks,
candidate.erasure.parity_blocks,
candidate.erasure.block_size,
candidate.uses_legacy_checksum,
) else {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_GEOMETRY);
}
let data_files = collect_inline_data_shard_fileinfos_from_observations(observations, candidate, data_shards, |index| {
disks.get(index).is_some_and(Option::is_some)
})?;
};
let data_files =
match collect_inline_data_shard_fileinfos_from_observations(observations, candidate, erasure.data_shards, |index| {
disks.get(index).is_some_and(Option::is_some)
}) {
Ok(data_files) => data_files,
Err(reason) => return Err(reason),
};
let Some(part) = candidate.parts.first() else {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_PART_SHAPE);
@@ -673,51 +693,22 @@ fn verify_inline_body_from_observations<'a>(
object_size,
object_size,
candidate.erasure.block_size,
data_shards,
erasure.data_shards,
candidate.uses_legacy_checksum,
);
let shard_size = inline_erasure_shard_size(candidate.erasure.block_size, data_shards, candidate.uses_legacy_checksum);
if object_size == 0 || read_length == 0 || read_length > shard_size || data_files.len() < data_shards {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
}
let shards_needed = object_size.div_ceil(read_length);
if shards_needed > data_shards {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
}
let Some(encoded_capacity) = read_length.checked_mul(shards_needed) else {
let shard_size = inline_erasure_shard_size(candidate.erasure.block_size, erasure.data_shards, candidate.uses_legacy_checksum);
let Ok(mut readers) =
build_inline_bitrot_readers_from_refs(&data_files, bucket, object, read_length, shard_size, &checksum_algo, false).await
else {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
};
#[cfg(test)]
{
for file in &data_files {
if file.data.is_some() {
crate::set_disk::disk_call_counters::record_shard_open(_object, true);
}
}
crate::set_disk::disk_call_counters::record(_object, crate::set_disk::disk_call_counters::KIND_BODY_DECODE, 0);
}
let hash_size = checksum_algo.size();
let need = hash_size + read_length;
let mut body = Vec::with_capacity(encoded_capacity);
for file in data_files.iter().take(shards_needed) {
let Some(raw) = file.data.as_deref() else {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
};
let Some(block) = raw.get(..need) else {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
};
let (hash, data) = block.split_at(hash_size);
if hash_size > 0 && checksum_algo.hash_encode(data).as_ref() != hash {
return Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY);
}
body.extend_from_slice(data);
if body.len() >= object_size {
body.truncate(object_size);
return Ok(bytes::Bytes::from(body));
}
#[cfg(test)]
crate::set_disk::disk_call_counters::record(object, crate::set_disk::disk_call_counters::KIND_BODY_DECODE, 0);
match try_read_inline_data_shards_direct(&mut readers, erasure.data_shards, read_length, object_size).await {
Some(body) if body.len() == object_size => Ok(body),
_ => Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY),
}
Err(GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_BODY_VERIFY)
}
fn data_read_early_stop_inline_candidate_miss_reason(candidate: &FileInfo) -> Option<&'static str> {
@@ -795,18 +786,16 @@ fn data_read_inline_missing_shards_are_pending(
fanout_order: &[usize],
scheduled_fanout_len: usize,
) -> bool {
let data_shards = candidate.erasure.data_blocks;
if data_shards == 0
|| candidate.erasure.block_size == 0
|| !candidate.has_valid_erasure_geometry()
|| (candidate.erasure.parity_blocks > 0
&& candidate.uses_legacy_checksum
&& !reed_solomon_simd::ReedSolomonEncoder::supports(data_shards, candidate.erasure.parity_blocks))
{
let Ok(erasure) = coding::Erasure::try_new_with_options(
candidate.erasure.data_blocks,
candidate.erasure.parity_blocks,
candidate.erasure.block_size,
candidate.uses_legacy_checksum,
) else {
return false;
}
};
let distribution = &candidate.erasure.distribution;
let mut data_shards_seen_or_pending = vec![false; data_shards];
let mut data_shards_seen_or_pending = vec![false; erasure.data_shards];
let mut missing_pending_data_shards = 0usize;
for slot in slots {
@@ -814,7 +803,7 @@ fn data_read_inline_missing_shards_are_pending(
let Some(&block_index) = distribution.get(disk_index) else {
return false;
};
if block_index == 0 || block_index > data_shards {
if block_index == 0 || block_index > erasure.data_shards {
continue;
}
if !disks.get(disk_index).is_some_and(Option::is_some) {
@@ -1181,8 +1170,7 @@ impl SetDisks {
default_parity_count: usize,
allow_coalescing: bool,
) -> disk::error::Result<MetadataReadResult> {
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
let fanout_start = stage_metrics_enabled.then(Instant::now);
let fanout_start = Instant::now();
let mut slots = MetadataReadResult::pending(disks.len());
let mut task_ids = vec![None; disks.len()];
let mut observations = Vec::with_capacity(disks.len());
@@ -1232,7 +1220,7 @@ impl SetDisks {
let diagnostic_context = diagnostic_context;
let task = join_set.spawn(async move {
let read = async move {
let response_start = stage_metrics_enabled.then(Instant::now);
let response_start = Instant::now();
let result = if let Some(disk) = disk {
#[cfg(test)]
let _fanout_task_guard = rename_fanout_barrier::task_guard(&object);
@@ -1255,7 +1243,7 @@ impl SetDisks {
} else {
Err(DiskError::DiskNotFound)
};
(index, result, response_start.map_or(Duration::ZERO, |s| s.elapsed()))
(index, result, response_start.elapsed())
};
#[cfg(feature = "diagnostic-read-version-caller")]
return crate::cluster::rpc::runtime_sources::with_read_version_diagnostic_context(
@@ -1360,10 +1348,7 @@ impl SetDisks {
},
Err(join_error) => {
let err = DiskError::Unexpected;
observations.push(MetadataFanoutObservation::from_error(
&err,
fanout_start.map_or(Duration::ZERO, |s| s.elapsed()),
));
observations.push(MetadataFanoutObservation::from_error(&err, fanout_start.elapsed()));
if let Some(index) = task_ids.iter().position(|id| *id == Some(join_error.id())) {
slots[index].result = Some(Err(err));
accumulator.observe(&slots[index]);
@@ -1399,11 +1384,14 @@ impl SetDisks {
accumulator.candidate_has_read_reserve()
}
Some(candidate) => match verify_inline_body_from_observations(
bucket.as_ref(),
object.as_ref(),
candidate,
slots.iter().map(|slot| (slot.disk_index, slot.file_info())),
disks,
) {
)
.await
{
Ok(body) => {
verified_inline_body = Some(body);
true
@@ -1461,8 +1449,7 @@ impl SetDisks {
scheduled_count.saturating_sub(cancelled_count),
cancelled_count,
);
let diagnostics =
MetadataFanoutDiagnostics::new(fanout_start.map_or(Duration::ZERO, |s| s.elapsed()), observations);
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
return Ok(MetadataReadResult {
slots,
diagnostics,
@@ -1545,7 +1532,7 @@ impl SetDisks {
rustfs_io_metrics::record_get_object_metadata_early_stop_miss(metrics_path, final_miss_reason);
rustfs_io_metrics::record_get_object_metadata_early_stop_saved_responses(metrics_path, 0);
rustfs_io_metrics::record_get_object_metadata_fanout_lifecycle(metrics_path, scheduled_count, scheduled_count, 0);
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.map_or(Duration::ZERO, |s| s.elapsed()), observations);
let diagnostics = MetadataFanoutDiagnostics::new(fanout_start.elapsed(), observations);
Ok(MetadataReadResult {
slots,
diagnostics,
-75
View File
@@ -275,22 +275,6 @@ impl SetDisks {
pub(super) fn common_parity(parities: &[i32], default_parity_count: i32) -> i32 {
let n = parities.len() as i32;
let mut valid_parities = parities.iter().copied().filter(|&p| p != -1);
let Some(first_parity) = valid_parities.next() else {
return -1;
};
let mut same_count = 1i32;
if valid_parities.all(|p| {
same_count += 1;
p == first_parity
}) {
let read_quorum = if default_parity_count > 0 && first_parity == 0 {
n / 2 + 1
} else {
n - first_parity
};
return if same_count >= read_quorum { first_parity } else { -1 };
}
let mut occ_map: HashMap<i32, i32> = HashMap::new();
for &p in parities {
@@ -349,19 +333,6 @@ impl SetDisks {
}
pub(super) fn common_time_and_occurrence(times: &[Option<OffsetDateTime>]) -> (Option<OffsetDateTime>, usize) {
let mut valid_times = times.iter().flatten().copied();
let Some(first_time) = valid_times.next() else {
return (None, 0);
};
let mut count = 1usize;
if valid_times.all(|t| {
count += 1;
t == first_time
}) && first_time.unix_timestamp_nanos() != 0
{
return (Some(first_time), count);
}
let mut time_occurrence_map = HashMap::new();
// Ignore the uuid sentinel and count the rest.
@@ -628,25 +599,6 @@ impl SetDisks {
return write_quorum;
}
if let Some((first, _)) = parts_metadata
.iter()
.zip(errs.iter())
.find(|(meta, err)| err.is_none() && file_info_is_valid_for_metadata(meta))
{
let same_count = parts_metadata
.iter()
.zip(errs.iter())
.filter(|(meta, err)| {
err.is_none()
&& file_info_is_valid_for_metadata(meta)
&& super::core::metadata_quorum::metadata_early_stop_candidate_matches(first, meta)
})
.count();
if same_count >= write_quorum {
return write_quorum;
}
}
let mut identity_counts = HashMap::with_capacity(usable_metadata);
for (meta, err) in parts_metadata.iter().zip(errs.iter()) {
if err.is_some() || !file_info_is_valid_for_metadata(meta) {
@@ -1030,33 +982,6 @@ impl SetDisks {
return Err(DiskError::ErasureReadQuorum);
}
let matches_time_or_etag = |meta: &FileInfo| {
mod_time == &meta.mod_time
|| (mod_time.is_none() && etag.as_deref().is_some_and(|e| meta.get_etag().as_deref() == Some(e)))
};
if let Some(first) = metas
.iter()
.find(|meta| file_info_is_valid_for_metadata(meta) && matches_time_or_etag(meta))
{
let mut total_valid = 0usize;
let mut same_count = 0usize;
for meta in metas {
if !file_info_is_valid_for_metadata(meta) || !matches_time_or_etag(meta) {
continue;
}
total_valid += 1;
if super::core::metadata_quorum::metadata_early_stop_candidate_matches(first, meta) {
same_count += 1;
}
}
if same_count >= quorum && same_count > total_valid - same_count {
let mut fi = first.clone();
fi.is_latest = fi.successor_mod_time.is_none();
return Ok(fi);
}
}
let mut meta_hashes = vec![None; metas.len()];
for (i, meta) in metas.iter().enumerate() {
+150 -107
View File
@@ -129,7 +129,7 @@ use rustfs_filemeta::{
};
use rustfs_heal_contracts::heal_channel::{
DriveState, HealAdmissionResult, HealChannelPriority, HealItemType, HealOpts, HealRequestSource, HealScanMode,
send_heal_request_with_admission,
send_heal_replacement_disk, send_heal_request_with_admission,
};
use rustfs_io_metrics::{
record_object_lock_diag_acquire_duration, record_object_lock_diag_enabled, record_object_lock_diag_hold_duration,
@@ -160,7 +160,7 @@ use rustfs_utils::http::{
use rustfs_utils::{
HashAlgorithm,
crypto::hex,
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, object_name_disk_identity, path_join_buf},
path::{SLASH_SEPARATOR, encode_dir_object, encode_dir_object_ref, has_suffix, object_name_disk_identity, path_join_buf},
};
use sha2::Sha256;
use std::borrow::Cow;
@@ -889,6 +889,8 @@ pub(crate) use ops::heal::AbsenceProofRequest;
#[cfg(test)]
pub(crate) use ops::heal::DanglingDeleteFailure;
pub(crate) use ops::heal::HealedObjectAbsence;
#[cfg(test)]
pub(crate) use ops::heal::ReadRepairPauseScope;
#[cfg(test)]
pub(crate) use ops::hermetic_set_disks_isolated;
@@ -934,20 +936,12 @@ pub(in crate::set_disk) struct GetObjectFileInfo {
}
struct OwnedGetObjectFileInfo {
payload: OwnedGetObjectFileInfoPayload,
fi: FileInfo,
parts_metadata: Vec<FileInfo>,
online_disks: Vec<Option<DiskStore>>,
verified_inline_body: Option<Bytes>,
}
#[allow(clippy::large_enum_variant)]
enum OwnedGetObjectFileInfoPayload {
Direct {
fi: FileInfo,
parts_metadata: Vec<FileInfo>,
online_disks: Vec<Option<DiskStore>>,
},
Published(Arc<GetObjectMetadataCacheEntry>),
}
enum LateMetadataFanout {
CapturedDisks(Vec<Option<DiskStore>>),
RefreshCurrentSet,
@@ -959,11 +953,9 @@ impl GetObjectFileInfo {
fn owned(fi: FileInfo, parts_metadata: Vec<FileInfo>, online_disks: Vec<Option<DiskStore>>) -> Self {
Self {
owned: Some(OwnedGetObjectFileInfo {
payload: OwnedGetObjectFileInfoPayload::Direct {
fi,
parts_metadata,
online_disks,
},
fi,
parts_metadata,
online_disks,
verified_inline_body: None,
}),
shared: None,
@@ -972,21 +964,15 @@ impl GetObjectFileInfo {
}
}
fn owned_published(entry: Arc<GetObjectMetadataCacheEntry>) -> Self {
let condition_stopped = entry.capability == GetObjectMetadataCacheCapability::ConditionOnly;
Self {
owned: Some(OwnedGetObjectFileInfo {
payload: OwnedGetObjectFileInfoPayload::Published(entry),
verified_inline_body: None,
}),
shared: None,
condition_stopped,
late_metadata_fanout: None,
}
}
fn set_late_metadata_fanout_disks(&mut self, late_metadata_fanout_disks: Vec<Option<DiskStore>>) {
self.late_metadata_fanout = Some(LateMetadataFanout::CapturedDisks(late_metadata_fanout_disks));
fn owned_with_late_metadata_fanout(
fi: FileInfo,
parts_metadata: Vec<FileInfo>,
online_disks: Vec<Option<DiskStore>>,
late_metadata_fanout_disks: Vec<Option<DiskStore>>,
) -> Self {
let mut snapshot = Self::owned(fi, parts_metadata, online_disks);
snapshot.late_metadata_fanout = Some(LateMetadataFanout::CapturedDisks(late_metadata_fanout_disks));
snapshot
}
fn shared(entry: Arc<GetObjectMetadataCacheEntry>) -> Self {
@@ -1005,31 +991,28 @@ impl GetObjectFileInfo {
snapshot
}
fn as_parts(&self) -> (&FileInfo, &[FileInfo], &[Option<DiskStore>]) {
fn fi(&self) -> &FileInfo {
match (&self.owned, &self.shared) {
(Some(snapshot), None) => match &snapshot.payload {
OwnedGetObjectFileInfoPayload::Direct {
fi,
parts_metadata,
online_disks,
} => (fi, parts_metadata, online_disks),
OwnedGetObjectFileInfoPayload::Published(entry) => (&entry.fi, &entry.parts_metadata, &entry.online_disks),
},
(None, Some(entry)) => (&entry.fi, &entry.parts_metadata, &entry.online_disks),
(Some(snapshot), None) => &snapshot.fi,
(None, Some(entry)) => &entry.fi,
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
}
}
fn fi(&self) -> &FileInfo {
self.as_parts().0
}
fn parts_metadata(&self) -> &[FileInfo] {
self.as_parts().1
match (&self.owned, &self.shared) {
(Some(snapshot), None) => &snapshot.parts_metadata,
(None, Some(entry)) => &entry.parts_metadata,
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
}
}
fn online_disks(&self) -> &[Option<DiskStore>] {
self.as_parts().2
match (&self.owned, &self.shared) {
(Some(snapshot), None) => &snapshot.online_disks,
(None, Some(entry)) => &entry.online_disks,
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
}
}
fn has_late_metadata_fanout(&self) -> bool {
@@ -1043,28 +1026,17 @@ impl GetObjectFileInfo {
}
fn take_verified_inline_body(&mut self) -> Option<Bytes> {
match (&mut self.owned, &self.shared) {
(Some(owned), None) => owned.verified_inline_body.take().or_else(|| match &owned.payload {
OwnedGetObjectFileInfoPayload::Direct { .. } => None,
OwnedGetObjectFileInfoPayload::Published(entry) => entry.verified_inline_body.clone(),
}),
(None, Some(entry)) => entry.verified_inline_body.clone(),
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
}
}
fn into_payload(self) -> (OwnedGetObjectFileInfoPayload, Option<LateMetadataFanout>) {
match (self.owned, self.shared) {
(Some(snapshot), None) => (snapshot.payload, self.late_metadata_fanout),
(None, Some(entry)) => (OwnedGetObjectFileInfoPayload::Published(entry), self.late_metadata_fanout),
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
}
self.owned.as_mut().and_then(|owned| owned.verified_inline_body.take())
}
fn into_object_info(self, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
match self.into_payload().0 {
OwnedGetObjectFileInfoPayload::Direct { fi, .. } => ObjectInfo::from_owned_file_info(fi, bucket, object, versioned),
OwnedGetObjectFileInfoPayload::Published(entry) => ObjectInfo::from_file_info(&entry.fi, bucket, object, versioned),
match (self.owned, self.shared) {
(Some(snapshot), None) => ObjectInfo::from_owned_file_info(snapshot.fi, bucket, object, versioned),
(None, Some(entry)) => match Arc::try_unwrap(entry) {
Ok(entry) => ObjectInfo::from_owned_file_info(entry.fi, bucket, object, versioned),
Err(entry) => ObjectInfo::from_file_info(&entry.fi, bucket, object, versioned),
},
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
}
}
@@ -1074,17 +1046,24 @@ impl GetObjectFileInfo {
}
fn into_owned_with_late_metadata_fanout(self) -> OwnedGetObjectFileInfoParts {
let (payload, late_metadata_fanout) = self.into_payload();
match payload {
OwnedGetObjectFileInfoPayload::Direct {
fi,
parts_metadata,
online_disks,
} => (fi, parts_metadata, online_disks, late_metadata_fanout),
OwnedGetObjectFileInfoPayload::Published(entry) => {
let entry = Arc::unwrap_or_clone(entry);
(entry.fi, entry.parts_metadata, entry.online_disks, late_metadata_fanout)
}
let Self {
owned,
shared,
late_metadata_fanout,
..
} = self;
match (owned, shared) {
(Some(snapshot), None) => (snapshot.fi, snapshot.parts_metadata, snapshot.online_disks, late_metadata_fanout),
(None, Some(entry)) => match Arc::try_unwrap(entry) {
Ok(entry) => (entry.fi, entry.parts_metadata, entry.online_disks, late_metadata_fanout),
Err(entry) => (
entry.fi.clone(),
entry.parts_metadata.clone(),
entry.online_disks.clone(),
late_metadata_fanout,
),
},
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
}
}
@@ -1469,7 +1448,6 @@ mod prepared_get_object_metadata_tests {
fi,
online_disks: vec![None],
read_quorum: 0,
verified_inline_body: None,
});
let snapshot = GetObjectFileInfo::shared(Arc::clone(&cached));
@@ -2786,9 +2764,7 @@ impl SetDisks {
object: &str,
opts: &ObjectOptions,
) -> Result<PreparedGetObjectMetadata> {
let snapshot = self
.get_object_fileinfo_for_object_read(bucket, object, opts, true, false)
.await?;
let snapshot = self.get_object_fileinfo(bucket, object, opts, true, false).await?;
let object_info = build_get_object_info(snapshot.fi(), bucket, object, opts.versioned || opts.version_suspended);
Ok(PreparedGetObjectMetadata {
snapshot,
@@ -4927,7 +4903,7 @@ enum GetObjectMetadataCacheCapability {
ConditionOnly,
}
#[derive(Clone, Debug)]
#[derive(Debug)]
struct GetObjectMetadataCacheEntry {
#[allow(dead_code)] // Kept for debugging; moka handles TTL internally
created_at: Instant,
@@ -4936,7 +4912,6 @@ struct GetObjectMetadataCacheEntry {
online_disks: Vec<Option<DiskStore>>,
read_quorum: usize,
capability: GetObjectMetadataCacheCapability,
verified_inline_body: Option<Bytes>,
}
#[derive(Clone, Debug)]
@@ -5247,6 +5222,40 @@ impl SetDisks {
.with_attempt(attempt))
}
pub(crate) async fn read_listing_metadata_after_namespace_barrier(
&self,
disks: &[Option<DiskStore>],
bucket: &str,
object: &str,
) -> Result<(MetaCacheEntries, usize)> {
// This is only called after the lock-free LIST fast path observes an
// unresolved metadata generation. Waiting on the ordinary object read
// lock establishes a publication boundary with concurrent overwrites.
let stored_object = encode_dir_object_ref(object);
let _guard = self
.acquire_read_lock_diag("list_object_reconcile", bucket, stored_object.as_ref())
.await?;
let (raw_entries, errors) = Self::read_all_raw_file_info(disks, bucket, stored_object.as_ref(), false).await;
let confirmed_absent = errors
.iter()
.flatten()
.filter(|err| DiskError::is_err_object_not_found(err) || DiskError::is_err_version_not_found(err))
.count();
let entries = raw_entries
.into_iter()
.map(|raw| {
raw.filter(|raw| !raw.buf.is_empty()).map(|raw| MetaCacheEntry {
name: object.to_owned(),
metadata: raw.buf,
cached: None,
reusable: false,
})
})
.collect();
Ok((MetaCacheEntries(entries), confirmed_absent))
}
async fn acquire_write_lock_diag(&self, op: &'static str, bucket: &str, object: &str) -> Result<ObjectLockDiagGuard> {
crate::hp_guard!("SetDisks::acquire_write_lock");
let diag_enabled = is_object_lock_diag_enabled();
@@ -8807,31 +8816,17 @@ mod tests {
metrics::with_local_recorder(&recorder, || {
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
runtime.block_on(async {
let healthy: Arc<dyn LockClient> =
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
let failing: Arc<dyn LockClient> = Arc::new(FailingClient);
let ctx = Arc::new(InstanceContext::new());
ctx.update_erasure_type(SetupType::DistErasure).await;
let set = make_test_set_disks_with_ctx(Vec::new(), ctx).await;
// A real acquire can time out before the failing client is scheduled.
let err = resolve_put_object_commit_lock_acquire_result(
&set,
"put_object_commit",
"bucket",
"object",
Err(LockError::QuorumNotReached {
required: 2,
achieved: 1,
}),
)
.expect_err("a quorum failure must remain a lock error");
let set = make_test_set_disks_with_ctx(vec![healthy, failing], ctx).await;
assert!(
matches!(
err,
StorageError::NamespaceLockQuorumUnavailable {
required: 2,
achieved: 1,
..
}
),
"expected the typed quorum error, got: {err:?}"
set.acquire_write_lock_diag("put_object_commit", "bucket", "object")
.await
.is_err(),
"one healthy locker must not satisfy the PUT commit write quorum"
);
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
});
@@ -16443,6 +16438,54 @@ mod tests {
assert_eq!(err, StorageError::PreconditionFailed);
}
#[tokio::test]
async fn listing_metadata_reconcile_encodes_directory_object_and_waits_for_writer() {
let set_disks = make_local_bucket_test_set_disks().await;
let bucket = "bucket-list-reconcile-lock";
let object = "directory/";
let stored_object = encode_dir_object(object);
set_disks
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let mut reader = PutObjReader::from_vec(b"directory marker".to_vec());
set_disks
.put_object(bucket, &stored_object, &mut reader, &ObjectOptions::default())
.await
.expect("directory object should be written under its encoded key");
let disks = set_disks.disks.read().await.clone();
let namespace_lock = set_disks
.new_ns_lock(bucket, &stored_object)
.await
.expect("namespace lock should be created");
let writer_guard = namespace_lock
.get_write_lock(std::time::Duration::from_secs(30))
.await
.expect("writer lock should be acquired");
let read = set_disks.read_listing_metadata_after_namespace_barrier(&disks, bucket, object);
tokio::pin!(read);
assert!(
tokio::time::timeout(std::time::Duration::from_millis(50), &mut read)
.await
.is_err(),
"LIST reconciliation must wait for the publishing writer"
);
drop(writer_guard);
let (entries, confirmed_absent) = tokio::time::timeout(std::time::Duration::from_secs(30), read)
.await
.expect("LIST reconciliation should resume after writer release")
.expect("LIST reconciliation should read after the namespace barrier");
assert_eq!(entries.0.len(), disks.len());
assert!(
entries.0.iter().all(|entry| entry
.as_ref()
.is_some_and(|entry| entry.name == object && !entry.metadata.is_empty())),
"LIST reconciliation must read encoded metadata and retain the logical key"
);
assert_eq!(confirmed_absent, 0);
}
#[tokio::test]
async fn repeated_body_write_keeps_etag_but_changes_data_dir_generation() {
let set_disks = make_local_bucket_test_set_disks().await;
File diff suppressed because it is too large Load Diff
+47 -362
View File
@@ -21,22 +21,17 @@
use super::super::{
Arc, DiskError, DiskInfo, DiskInfoOptions, DiskOption, DiskStore, Endpoint, Error, FormatV3, HealChannelPriority, LockResult,
NamespaceLock, NamespaceLockWrapper, ObjectKey, Result, SetDisks, StorageError, debug, disk, load_format_erasure, warn,
NamespaceLock, NamespaceLockWrapper, ObjectKey, Result, SetDisks, StorageError, debug, disk, info, load_format_erasure,
send_heal_replacement_disk, warn,
};
use crate::disk::health_state::DriveMembershipSnapshot;
use crate::disk::{DiskAPI, new_disk};
use crate::layout::disks_layout::MAX_ERASURE_SET_DRIVE_COUNT;
use crate::runtime::sources as runtime_sources;
use futures::{StreamExt, stream, stream::FuturesUnordered};
use rand::prelude::SliceRandom;
use rustfs_heal_contracts::heal_channel::{create_auto_replacement_disks_request, send_heal_request};
use rustfs_utils::path::object_name_disk_identity;
#[cfg(test)]
use uuid::Uuid;
// Bound batching when another reconnect is still pending.
const REPLACEMENT_BATCH_WAIT: std::time::Duration = std::time::Duration::from_secs(1);
#[async_trait::async_trait]
impl crate::storage_api_contracts::namespace::NamespaceLocking for SetDisks {
type Error = Error;
@@ -331,83 +326,38 @@ impl SetDisks {
}
async fn connect_disks_with_active_monitoring(&self, active_monitoring: bool) {
let disks = self.get_disks_internal().await;
// Reconnect drives concurrently without holding the inventory lock.
let mut reconnects = stream::iter(disks.into_iter().enumerate())
.map(|(i, opdisk)| async move {
if let Some(disk) = opdisk {
if !active_monitoring
&& disk.get_disk_location().set_idx.is_some()
&& let disk::Disk::Local(local) = disk.as_ref()
&& local.can_skip_background_probe().await
{
return None;
}
if disk.is_online().await && disk.get_disk_location().set_idx.is_some() {
return None;
}
let rl = self.disks.read().await;
let _ = disk.close().await;
let disks = rl.clone();
// Explicitly release the lock
drop(rl);
for (i, opdisk) in disks.iter().enumerate() {
if let Some(disk) = opdisk {
if !active_monitoring
&& disk.get_disk_location().set_idx.is_some()
&& let disk::Disk::Local(local) = disk.as_ref()
&& local.can_skip_background_probe().await
{
continue;
}
if disk.is_online().await && disk.get_disk_location().set_idx.is_some() {
info!("Disk {:?} is online", disk.to_string());
continue;
}
let endpoint = self.set_endpoints.get(i)?;
self.reconnect_disk(endpoint).await.then(|| endpoint.to_string())
})
.buffer_unordered(MAX_ERASURE_SET_DRIVE_COUNT);
let mut submissions = FuturesUnordered::new();
let mut batch = Vec::new();
let mut deadline = None;
let mut reconnects_done = false;
loop {
if reconnects_done && submissions.is_empty() {
break;
let _ = disk.close().await;
}
tokio::select! {
result = reconnects.next(), if !reconnects_done => match result {
Some(Some(endpoint)) => {
batch.push(endpoint);
deadline.get_or_insert_with(|| tokio::time::Instant::now() + REPLACEMENT_BATCH_WAIT);
}
Some(None) => {}
None => {
reconnects_done = true;
if !batch.is_empty() {
submissions.push(self.submit_replacement_heal(std::mem::take(&mut batch)));
}
deadline = None;
}
},
_ = tokio::time::sleep_until(deadline.unwrap_or_else(tokio::time::Instant::now)), if deadline.is_some() => {
submissions.push(self.submit_replacement_heal(std::mem::take(&mut batch)));
deadline = None;
}
_ = submissions.next(), if !submissions.is_empty() => {}
}
}
}
async fn submit_replacement_heal(&self, replacements: Vec<String>) {
if replacements.is_empty() {
return;
if let Some(endpoint) = self.set_endpoints.get(i) {
info!("will renew disk, opdisk: {:?}", opdisk);
self.renew_disk(endpoint).await;
}
}
// Queue admission deduplicates by set, so publish ready targets together.
let request = create_auto_replacement_disks_request(
self.pool_index,
self.set_index,
replacements,
Some(HealChannelPriority::Normal),
);
let _ = send_heal_request(request).await;
}
pub async fn renew_disk(&self, ep: &Endpoint) {
if self.reconnect_disk(ep).await {
self.submit_replacement_heal(vec![ep.to_string()]).await;
}
}
/// Return whether this reconnect attached a verified unformatted target.
async fn reconnect_disk(&self, ep: &Endpoint) -> bool {
debug!("renew_disk: start {:?}", ep);
let previous_disk = {
@@ -427,17 +377,28 @@ impl SetDisks {
Err(e) => {
warn!("renew_disk: connect_endpoint err {:?}", &e);
if !matches!(e, DiskError::UnformattedDisk | DiskError::Io(_)) {
return false;
return;
}
let attached = match self.attach_unformatted_replacement_disk(ep).await {
Ok(attached) => attached,
Err(err) => {
warn!(endpoint = %ep, error = ?err, "renew_disk: unformatted replacement probe failed");
return false;
return;
}
};
return attached;
if !attached {
return;
}
info!("renew_disk attached unformatted replacement and will trigger heal_disk, {:?}", ep);
let (Ok(pool_index), Ok(set_index)) = (usize::try_from(ep.pool_idx), usize::try_from(ep.set_idx)) else {
warn!("renew_disk: replacement target has invalid pool or set index, {:?}", ep);
return;
};
let _ =
send_heal_replacement_disk(pool_index, set_index, ep.to_string(), Some(HealChannelPriority::Normal)).await;
return;
}
};
@@ -445,7 +406,7 @@ impl SetDisks {
Ok(res) => res,
Err(e) => {
warn!("renew_disk: find_disk_index err {:?}", e);
return false;
return;
}
};
@@ -463,7 +424,7 @@ impl SetDisks {
expected_set_index = self.set_index,
"renew_disk rejected a drive whose endpoint and format do not identify the same topology slot"
);
return false;
return;
}
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
@@ -483,7 +444,7 @@ impl SetDisks {
new_disk.enable_health_check();
*slot = Some(new_disk);
}
return false;
return;
}
new_disk.enable_health_check();
@@ -505,7 +466,6 @@ impl SetDisks {
let mut disk_lock = self.disks.write().await;
disk_lock[disk_idx] = Some(new_disk);
false
}
/// Attach a replacement target only after proving that the exact local slot
@@ -767,6 +727,10 @@ mod tests {
let format = FormatV3::new(1, 1);
let (_dir, endpoint, disk) = make_formatted_local_disk(0, &format).await;
assert!(disk.is_online().await, "initialize the expected disk identity");
let disk::Disk::Local(wrapper) = disk.as_ref() else {
panic!("fixture must use a local disk");
};
let local = wrapper.get_disk();
let set = SetDisks::new(
"quiet-monitor".to_owned(),
Arc::new(RwLock::new(vec![Some(disk.clone())])),
@@ -785,136 +749,13 @@ mod tests {
(false, RuntimeDriveHealthState::Offline),
(false, RuntimeDriveHealthState::Returning),
] {
let disk = set.get_disks_internal().await[0].clone().expect("current drive");
let disk::Disk::Local(wrapper) = disk.as_ref() else {
panic!("fixture must use a local disk");
};
let local = wrapper.get_disk();
disk.force_runtime_state_for_test(state);
local.format_info.write().await.last_check = None;
set.connect_disks_with_active_monitoring(active).await;
let current = set.get_disks_internal().await[0].clone().expect("probed or renewed drive");
let disk::Disk::Local(wrapper) = current.as_ref() else {
panic!("fixture must use a local disk");
};
assert!(
wrapper.get_disk().format_info.read().await.last_check.is_some(),
local.format_info.read().await.last_check.is_some(),
"probe required for active={active}, state={state:?}"
);
current
.disk_info(&DiskInfoOptions::default())
.await
.expect("the probed or renewed drive must admit real I/O");
}
}
#[tokio::test]
async fn partial_reconnect_restores_ec4_io_without_all_disks_returning() {
use crate::object_api::{ObjectOptions, PutObjReader};
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
use crate::storage_api_contracts::object::ObjectIO;
use disk::health_state::RuntimeDriveHealthState;
use http::HeaderMap;
use tokio::io::AsyncReadExt;
for active_monitoring in [true, false] {
let format = FormatV3::new(1, 16);
let mut dirs = Vec::new();
let mut endpoints = Vec::new();
let mut disks = Vec::new();
for index in 0..16 {
let (dir, endpoint, disk) = make_formatted_local_disk(index, &format).await;
assert!(disk.is_online().await, "initialize the original drive identity");
dirs.push(dir);
endpoints.push(endpoint);
disks.push(Some(disk));
}
let set = SetDisks::new(
"partial-reconnect".to_owned(),
Arc::new(RwLock::new(disks.clone())),
16,
4,
0,
0,
endpoints,
format,
Vec::new(),
)
.await;
let bucket = "partial-reconnect";
let opts = ObjectOptions {
no_lock: true,
..Default::default()
};
set.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket before the outage");
let payload = vec![b'r'; 1_300_000];
set.put_object(bucket, "before", &mut PutObjReader::from_vec(payload.clone()), &opts)
.await
.expect("write the original object");
let mut saved_formats = Vec::new();
for (dir, disk) in dirs.iter().zip(&disks).take(8) {
let disk = disk.as_ref().expect("original disk");
disk.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
let path = dir.path().join(disk::RUSTFS_META_BUCKET).join(disk::FORMAT_CONFIG_FILE);
saved_formats.push(tokio::fs::read(&path).await.expect("save original format"));
// A corrupt format makes reconnect fail without admitting a blank replacement.
tokio::fs::write(&path, b"invalid-format")
.await
.expect("make the original drive unavailable");
let disk::Disk::Local(local) = disk.as_ref() else {
panic!("fixture must use local drives");
};
local.get_disk().format_info.write().await.last_check = None;
}
set.connect_disks_with_active_monitoring(active_monitoring).await;
let unavailable = set.get_disks_internal().await;
for (original, retained) in disks.iter().zip(&unavailable).take(8) {
assert!(
Arc::ptr_eq(original.as_ref().expect("original"), retained.as_ref().expect("retained")),
"failed reconnect must leave the old handle to exercise subsequent recovery"
);
}
assert!(
set.put_object(bucket, "during", &mut PutObjReader::from_vec(payload.clone()), &opts)
.await
.is_err(),
"eight available drives must not satisfy a 16-drive EC:4 write"
);
for (dir, saved) in dirs.iter().zip(&saved_formats).take(4) {
let path = dir.path().join(disk::RUSTFS_META_BUCKET).join(disk::FORMAT_CONFIG_FILE);
tokio::fs::write(path, saved)
.await
.expect("restore four original drive identities");
}
set.connect_disks_with_active_monitoring(active_monitoring).await;
assert_eq!(
set.drive_membership_snapshot().await.online.len(),
12,
"the four restored drives must be usable while four others remain unavailable"
);
set.put_object(bucket, "after", &mut PutObjReader::from_vec(payload.clone()), &opts)
.await
.expect("write at the restored EC:4 quorum");
for object in ["before", "after"] {
let mut reader = set
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
.await
.expect("read at the restored EC:4 quorum");
let mut actual = Vec::new();
reader
.stream
.read_to_end(&mut actual)
.await
.expect("read every reconstructed byte");
assert_eq!(actual, payload, "the complete object must survive partial recovery");
}
for disk in set.get_disks_internal().await.into_iter().flatten() {
disk.close().await.expect("stop the recovered drive monitors");
}
}
}
@@ -1144,162 +985,6 @@ mod tests {
drop(temp_dirs);
}
#[tokio::test]
async fn reconnect_sweep_collects_all_blank_targets_and_excludes_original_disks() {
use rustfs_heal_contracts::heal_channel::{HealAdmissionResult, HealChannelCommand, init_heal_channel};
const CHILD: &str = "RUSTFS_TEST_GROUPED_RECONNECT_CHILD";
if std::env::var_os(CHILD).is_none() {
// The heal sender is process-global; keep this receiver isolated.
let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
.args([
"--exact",
"set_disk::ops::locking::tests::reconnect_sweep_collects_all_blank_targets_and_excludes_original_disks",
"--nocapture",
])
.env(CHILD, "1")
.output()
.expect("isolated reconnect test should run");
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(output.status.success(), "isolated reconnect test failed: {stdout} {stderr}");
assert!(stdout.contains("1 passed"), "child must execute the reconnect test: {stdout}");
return;
}
let mut heal_rx = init_heal_channel().expect("isolated heal receiver");
let disk_count = 4;
let format = FormatV3::new(1, disk_count);
let mut directories = Vec::new();
let mut endpoints = Vec::new();
let mut disks = Vec::new();
for disk_idx in 0..2 {
let (directory, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
directories.push(directory);
endpoints.push(endpoint);
disks.push(Some(disk));
}
for disk_idx in 2..disk_count {
let directory = tempfile::tempdir().expect("replacement directory");
let mut endpoint =
Endpoint::try_from(directory.path().to_str().expect("UTF-8 replacement path")).expect("replacement endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_idx);
directories.push(directory);
endpoints.push(endpoint);
disks.push(None);
}
let mut ready_targets = endpoints[2..].iter().map(ToString::to_string).collect::<Vec<_>>();
ready_targets.sort_unstable();
// Disk 1 returns with its format and data; disks 2 and 3 are blank.
disks[1]
.take()
.expect("original disk handle")
.close()
.await
.expect("close original disk");
let retained_data = directories[1].path().join("retained-data");
tokio::fs::write(&retained_data, b"original data")
.await
.expect("write original data");
let set = Arc::new(
SetDisks::new(
"reconnect-sweep".to_string(),
Arc::new(RwLock::new(disks)),
disk_count,
2,
0,
0,
endpoints.clone(),
format,
Vec::new(),
)
.await,
);
// Block disk 0's format check, then make it a late blank replacement.
let slow = set.get_disks_internal().await[0].clone().expect("original disk");
assert!(slow.is_online().await, "initialize original disk identity");
let disk::Disk::Local(wrapper) = slow.as_ref() else {
panic!("fixture must use a local disk");
};
let slow_local = wrapper.get_disk();
let mut slow_format_guard = slow_local.format_info.write().await;
slow_format_guard.last_check = None;
std::fs::remove_file(
directories[0]
.path()
.join(disk::RUSTFS_META_BUCKET)
.join(disk::FORMAT_CONFIG_FILE),
)
.expect("make the blocked disk a blank replacement");
let sweep = tokio::spawn({
let set = set.clone();
async move { set.connect_disks_with_active_monitoring(true).await }
});
tokio::time::timeout(std::time::Duration::from_secs(10), async {
loop {
let attached = set.get_disks_internal().await;
if attached[1..].iter().all(Option::is_some) {
break;
}
tokio::task::yield_now().await;
}
})
.await
.expect("other drives must reconnect while the first drive is blocked");
let command = tokio::time::timeout(std::time::Duration::from_secs(10), heal_rx.recv())
.await
.expect("ready targets must be submitted while a drive is blocked")
.expect("heal receiver should remain open");
let HealChannelCommand::Start { request, response_tx } = command else {
panic!("expected a replacement heal request");
};
assert_eq!(request.heal_endpoints, ready_targets, "ready blank disks must share one request");
assert!(!sweep.is_finished(), "the blocked drive must still be pending");
drop(slow_format_guard);
// Do not acknowledge the first request yet: a delayed admission
// response must not stop the late drive reconnecting and submitting.
let late_command = tokio::time::timeout(std::time::Duration::from_secs(10), heal_rx.recv())
.await
.expect("late target must be submitted while the first response is pending")
.expect("heal receiver should remain open");
let HealChannelCommand::Start {
request: late_request,
response_tx: late_response,
} = late_command
else {
panic!("expected a late replacement heal request");
};
assert_eq!(late_request.heal_endpoints, [endpoints[0].to_string()]);
response_tx
.send(Ok(HealAdmissionResult::Accepted))
.expect("acknowledge ready targets");
late_response
.send(Ok(HealAdmissionResult::Accepted))
.expect("acknowledge late target");
sweep.await.expect("reconnect sweep should finish");
assert!(heal_rx.try_recv().is_err(), "no duplicate request should be sent");
let attached = set.get_disks_internal().await;
for (index, disk) in attached.iter().enumerate() {
let disk = disk.as_ref().expect("each verified endpoint should be attached");
let format = load_format_erasure(disk, false).await;
if index == 1 {
assert!(format.is_ok(), "original disk must retain its format");
} else {
assert_eq!(format.expect_err("replacement must remain unformatted"), DiskError::UnformattedDisk);
assert!(!disk.health_check_enabled_for_test(), "format must precede replacement health checks");
}
disk.close().await.expect("close test disk");
}
assert_eq!(tokio::fs::read(&retained_data).await.expect("read original data"), b"original data");
let local_map = runtime_sources::local_disk_map_handle();
let mut local_map = local_map.write().await;
for endpoint in endpoints {
local_map.remove(&endpoint.to_string());
}
}
#[tokio::test]
async fn renew_disk_attaches_only_a_verified_local_unformatted_replacement() {
let disk_count = 4;
+3 -8
View File
@@ -5617,10 +5617,7 @@ mod tests {
.await
.expect("abort task should join after completion releases")
.expect_err("the committed upload should no longer exist");
assert!(
matches!(abort_err, StorageError::InvalidUploadID(..)),
"the completed upload must be absent after the tail drains, got: {abort_err:?}"
);
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
for (disk_index, disk) in disk_stores.iter().enumerate() {
let fi = disk
@@ -9556,12 +9553,10 @@ mod tests {
).expect("actual completion identity"));
}
}
// The fourth native syscall is held by release_rx even if
// its deadline has already retired the async wrapper.
if published == 3 { break; }
if published == 3 && tasks.running() == 1 { break; }
tokio::task::yield_now().await;
}
}).await.expect("three real native publication legs must succeed while the fourth syscall remains paused");
}).await.expect("three real native publication legs must return before the fourth deadline");
after_publication.as_ref().expect("publication barrier").wait_until_paused().await;
// Quorum has been validated and the parent released its object
// guard. The successful physical tail cannot retain a clone.
+328 -158
View File
@@ -2999,44 +2999,6 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
let object_size = usize::try_from(fi.size)
.map_err(|_| to_object_err(Error::other("inline fast path object size is invalid"), vec![bucket, object]))?;
if can_try_inline_data_shards_direct(object_size, fi.erasure.block_size)
&& let Some(body) = verified_inline_body.filter(|body| body.len() == object_size)
{
if let Some(inline_prepare_stage_start) = inline_prepare_stage_start.take() {
rustfs_io_metrics::record_get_object_stage_duration_by_size(
GET_OBJECT_PATH_INLINE_DIRECT,
GET_STAGE_INLINE_PREPARE,
object_class.as_str(),
size_bucket,
inline_prepare_stage_start.elapsed().as_secs_f64(),
);
}
if rustfs_io_metrics::get_stage_metrics_enabled() {
rustfs_io_metrics::record_get_object_stage_duration_by_size(
GET_OBJECT_PATH_INLINE_DIRECT,
GET_STAGE_READER_SETUP,
object_class.as_str(),
size_bucket,
0.0,
);
rustfs_io_metrics::record_get_object_stage_duration_by_size(
GET_OBJECT_PATH_INLINE_DIRECT,
GET_STAGE_DECODE,
object_class.as_str(),
size_bucket,
0.0,
);
}
record_get_object_reader_path_observation(GET_OBJECT_PATH_INLINE_DIRECT, object_class, size_bucket);
let reader = GetObjectReader {
stream: Box::new(Cursor::new(body.clone())),
object_info,
buffered_body: Some(body),
body_source: GetObjectBodySource::Unprobed,
};
return Ok(reader);
}
let checksum_info = fi.erasure.get_checksum_info(fi.parts[0].number);
let checksum_algo =
if fi.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S {
@@ -3050,6 +3012,42 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
disks.get(index).is_some_and(Option::is_some)
})
{
if let Some(body) = verified_inline_body.filter(|body| body.len() == object_size) {
if let Some(inline_prepare_stage_start) = inline_prepare_stage_start.take() {
rustfs_io_metrics::record_get_object_stage_duration_by_size(
GET_OBJECT_PATH_INLINE_DIRECT,
GET_STAGE_INLINE_PREPARE,
object_class.as_str(),
size_bucket,
inline_prepare_stage_start.elapsed().as_secs_f64(),
);
}
if rustfs_io_metrics::get_stage_metrics_enabled() {
rustfs_io_metrics::record_get_object_stage_duration_by_size(
GET_OBJECT_PATH_INLINE_DIRECT,
GET_STAGE_READER_SETUP,
object_class.as_str(),
size_bucket,
0.0,
);
rustfs_io_metrics::record_get_object_stage_duration_by_size(
GET_OBJECT_PATH_INLINE_DIRECT,
GET_STAGE_DECODE,
object_class.as_str(),
size_bucket,
0.0,
);
}
record_get_object_reader_path_observation(GET_OBJECT_PATH_INLINE_DIRECT, object_class, size_bucket);
let reader = GetObjectReader {
stream: Box::new(Cursor::new(body.clone())),
object_info,
buffered_body: Some(body),
body_source: GetObjectBodySource::Unprobed,
};
return Ok(reader);
}
let read_length = inline_erasure_shard_file_offset(
0,
object_size,
@@ -8177,13 +8175,23 @@ fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) {
}
}
#[cfg(test)]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum DeleteObjectCommitPause {
BeforeNamespace,
BeforeCommit,
AfterPublish,
}
#[cfg(test)]
struct DeleteObjectCommitBarrierState {
bucket: String,
object: String,
pause_after_publish: bool,
pause: DeleteObjectCommitPause,
arrived: tokio::sync::Notify,
release: tokio::sync::Notify,
namespace_pending: tokio::sync::Notify,
namespace_acquired: std::sync::atomic::AtomicBool,
}
#[cfg(test)]
@@ -8198,21 +8206,27 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option
#[cfg(test)]
impl DeleteObjectCommitBarrier {
pub(crate) fn install(bucket: &str, object: &str) -> Self {
Self::install_with_mode(bucket, object, false)
Self::install_with_mode(bucket, object, DeleteObjectCommitPause::BeforeCommit)
}
pub(crate) fn install_before_namespace(bucket: &str, object: &str) -> Self {
Self::install_with_mode(bucket, object, DeleteObjectCommitPause::BeforeNamespace)
}
#[cfg(feature = "test-util")]
pub(crate) fn install_after_publish(bucket: &str, object: &str) -> Self {
Self::install_with_mode(bucket, object, true)
Self::install_with_mode(bucket, object, DeleteObjectCommitPause::AfterPublish)
}
fn install_with_mode(bucket: &str, object: &str, pause_after_publish: bool) -> Self {
fn install_with_mode(bucket: &str, object: &str, pause: DeleteObjectCommitPause) -> Self {
let state = Arc::new(DeleteObjectCommitBarrierState {
bucket: bucket.to_string(),
object: object.to_string(),
pause_after_publish,
pause,
arrived: tokio::sync::Notify::new(),
release: tokio::sync::Notify::new(),
namespace_pending: tokio::sync::Notify::new(),
namespace_acquired: std::sync::atomic::AtomicBool::new(false),
});
let mut slot = DELETE_OBJECT_COMMIT_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
@@ -8232,6 +8246,19 @@ impl DeleteObjectCommitBarrier {
pub(crate) fn release(&self) {
self.state.release.notify_one();
}
pub(crate) async fn release_and_wait_until_namespace_pending(&self) {
assert_eq!(self.state.pause, DeleteObjectCommitPause::BeforeNamespace);
let pending = self.state.namespace_pending.notified();
self.release();
tokio::time::timeout(Duration::from_secs(5), pending)
.await
.expect("delete object must poll its actual namespace acquisition to Pending");
}
pub(crate) fn namespace_acquired(&self) -> bool {
self.state.namespace_acquired.load(std::sync::atomic::Ordering::Acquire)
}
}
#[cfg(test)]
@@ -8248,6 +8275,43 @@ impl Drop for DeleteObjectCommitBarrier {
}
}
#[cfg(test)]
async fn pause_delete_object_before_namespace(bucket: &str, object: &str) {
let barrier = DELETE_OBJECT_COMMIT_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("delete object commit barrier mutex should not poison")
.as_ref()
.filter(|barrier| {
barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeNamespace
})
.cloned();
if let Some(barrier) = barrier {
barrier.arrived.notify_one();
barrier.release.notified().await;
}
}
#[cfg(test)]
fn notify_delete_object_namespace(bucket: &str, object: &str, acquired: bool) {
let barrier = DELETE_OBJECT_COMMIT_BARRIER
.get_or_init(|| std::sync::Mutex::new(None))
.lock()
.expect("delete object commit barrier mutex should not poison")
.as_ref()
.filter(|barrier| {
barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeNamespace
})
.cloned();
if let Some(barrier) = barrier {
if acquired {
barrier.namespace_acquired.store(true, std::sync::atomic::Ordering::Release);
} else {
barrier.namespace_pending.notify_one();
}
}
}
#[cfg(test)]
async fn pause_delete_object_commit(bucket: &str, object: &str) {
let barrier = DELETE_OBJECT_COMMIT_BARRIER
@@ -8255,7 +8319,9 @@ async fn pause_delete_object_commit(bucket: &str, object: &str) {
.lock()
.expect("delete object commit barrier mutex should not poison")
.as_ref()
.filter(|barrier| barrier.bucket == bucket && barrier.object == object && !barrier.pause_after_publish)
.filter(|barrier| {
barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::BeforeCommit
})
.cloned();
if let Some(barrier) = barrier {
barrier.arrived.notify_one();
@@ -8270,7 +8336,9 @@ async fn pause_delete_object_commit_after_publish(bucket: &str, object: &str) {
.lock()
.expect("delete object commit barrier mutex should not poison")
.as_ref()
.filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause_after_publish)
.filter(|barrier| {
barrier.bucket == bucket && barrier.object == object && barrier.pause == DeleteObjectCommitPause::AfterPublish
})
.cloned();
if let Some(barrier) = barrier {
barrier.arrived.notify_one();
@@ -10529,7 +10597,20 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
// Guard lock for single object delete
let _lock_guard = if (!opts.delete_prefix || opts.delete_prefix_object) && !opts.no_lock {
Some(Arc::new(self.acquire_write_lock_diag("delete_object", bucket, object).await?))
#[cfg(test)]
let guard = {
pause_delete_object_before_namespace(bucket, object).await;
let guard = self
.acquire_write_lock_diag_with_pending_hook("delete_object", bucket, object, || {
notify_delete_object_namespace(bucket, object, false);
})
.await?;
notify_delete_object_namespace(bucket, object, true);
guard
};
#[cfg(not(test))]
let guard = self.acquire_write_lock_diag("delete_object", bucket, object).await?;
Some(Arc::new(guard))
} else {
None
};
@@ -11089,7 +11170,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
// shared lock and permit metadata caching can early-stop once write
// quorum agrees on the object identity.
let allow_early_stop = !opts.no_lock || opts.metadata_cache_safe;
let read = self.get_object_fileinfo_for_object_read(bucket, object, opts, true, allow_early_stop);
let read = self.get_object_fileinfo(bucket, object, opts, true, allow_early_stop);
#[cfg(feature = "diagnostic-read-version-caller")]
let read = crate::cluster::rpc::runtime_sources::with_read_version_entry(
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ObjectInfoRead,
@@ -16906,7 +16987,6 @@ mod transition_commit_failure_tests {
parts_metadata,
online_disks,
read_quorum: 2,
verified_inline_body: None,
}),
)
.await;
@@ -21435,6 +21515,206 @@ mod put_object_tmp_cleanup_tests {
.await;
}
#[tokio::test]
async fn deep_heal_preserves_committed_generation_during_old_data_cleanup() {
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
for (disk_count, parity, read_repair) in [(4, 2, false), (4, 2, true), (16, 4, false), (16, 4, true)] {
let (dirs, disks, set) = super::hermetic_set_disks_support::hermetic_set_disks_for_pool_with_default_parity_isolated(
disk_count, 0, parity,
)
.await;
let bucket = "native-heal-commit-cleanup";
let object = format!("generation-{disk_count}-{parity}-{read_repair}.bin");
for disk in &disks {
disk.make_volume(bucket).await.expect("create actual fixture volume");
}
let options = ObjectOptions {
write_completion: crate::object_api::WriteCompletion::TailDrained,
shard_integrity_write_mode: Some(crate::object_api::ShardIntegrityWriteMode::Protected),
..Default::default()
};
set.put_object(bucket, &object, &mut PutObjReader::from_vec(vec![0x17; TEST_OBJECT_SIZE]), &options)
.await
.expect("acknowledge original generation on every disk");
let old = disks[0]
.read_version("", bucket, &object, "", &ReadOptions::default())
.await
.expect("read actual original generation");
let old_dir = old.data_dir.expect("original external data directory");
let old_part = dirs[0]
.path()
.join(bucket)
.join(&object)
.join(old_dir.to_string())
.join("part.1");
assert!(tokio::fs::try_exists(&old_part).await.expect("original part exists"));
// Cleanup slots follow the writer's shuffled erasure distribution,
// not the physical fixture order. Pause every real cleanup slot so
// no physical old shard can disappear before the proof below.
let barriers: Vec<_> = (0..disk_count)
.map(|slot| rename_fanout_barrier::arm(&object, slot, rename_fanout_barrier::PHASE_CLEANUP))
.collect();
let writer_set = Arc::clone(&set);
let writer_object = object.clone();
let writer_options = options.clone();
let payload = vec![0xa3; TEST_OBJECT_SIZE];
let writer_payload = payload.clone();
let writer = tokio::spawn(async move {
writer_set
.put_object(bucket, &writer_object, &mut PutObjReader::from_vec(writer_payload), &writer_options)
.await
});
tokio::time::timeout(Duration::from_secs(30), async {
for barrier in &barriers {
barrier.wait_until_paused().await;
}
})
.await
.expect("overwrite commits before its real old-data cleanup pause");
assert!(!writer.is_finished(), "tail-drained ACK must remain pending at cleanup");
assert!(
tokio::fs::try_exists(&old_part)
.await
.expect("old source still present at pause")
);
let mut committed = Vec::new();
let mut healthy_metadata = Vec::new();
for disk in &disks {
let info = disk
.read_version("", bucket, &object, "", &ReadOptions::default())
.await
.expect("read independently committed physical generation without GET");
assert_eq!((info.erasure.data_blocks, info.erasure.parity_blocks), (disk_count - parity, parity));
assert!(info.data.is_none(), "use actual external parts");
assert_ne!(info.data_dir, Some(old_dir));
committed.push(info);
healthy_metadata.push(
disk.read_all(bucket, &format!("{object}/xl.meta"))
.await
.expect("committed raw metadata"),
);
}
let new_dir = committed[0].data_dir.expect("new committed data directory");
assert!(committed.iter().all(|info| info.data_dir == Some(new_dir)));
let new_part = dirs[0]
.path()
.join(bucket)
.join(&object)
.join(new_dir.to_string())
.join("part.1");
let expected_part = tokio::fs::read(&new_part).await.expect("exact new source bytes");
let mut committed_parts = Vec::new();
for dir in &dirs {
let original = dir.path().join(bucket).join(&object).join(old_dir.to_string()).join("part.1");
assert!(
tokio::fs::try_exists(original)
.await
.expect("every old cleanup slot is paused")
);
committed_parts.push(
tokio::fs::read(dir.path().join(bucket).join(&object).join(new_dir.to_string()).join("part.1"))
.await
.expect("every new encoded shard before fault"),
);
}
tokio::fs::remove_file(&new_part)
.await
.expect("remove only the newly committed target shard");
println!(
"native_g01: geometry={disk_count}/{parity} read_repair={read_repair} old_cleanup_paused new_generation_committed target_part_missing"
);
let (result, error) = tokio::time::timeout(
Duration::from_secs(30),
set.heal_object(
bucket,
&object,
"",
&HealOpts {
scan_mode: HealScanMode::Deep,
read_repair,
..Default::default()
},
),
)
.await
.expect("real locked Heal must progress during old cleanup")
.expect("complete actual deep Heal");
assert!(error.is_none(), "new generation must remain repairable: {error:?}");
assert_eq!(result.drives_healed(), Some(1));
// Shared data-phase repair deliberately does not certify full integrity.
assert_eq!(result.integrity_verified, !read_repair);
assert_eq!(tokio::fs::read(&new_part).await.expect("rebuilt part before any GET"), expected_part);
let repaired = disks[0]
.read_version("", bucket, &object, "", &ReadOptions::default())
.await
.expect("read actual repaired generation");
assert_eq!(repaired.data_dir, Some(new_dir));
assert_eq!(repaired.mod_time, committed[0].mod_time);
for (disk, expected) in disks.iter().zip(&healthy_metadata).skip(1) {
assert_eq!(
disk.read_all(bucket, &format!("{object}/xl.meta"))
.await
.expect("healthy raw metadata after Heal"),
*expected
);
}
println!(
"native_g01: geometry={disk_count}/{parity} read_repair={read_repair} locked_heal_completed new_generation_part_rebuilt_before_get"
);
for barrier in &barriers {
barrier.release();
}
let acknowledged = tokio::time::timeout(Duration::from_secs(30), writer)
.await
.expect("cleanup drains")
.expect("writer joins")
.expect("overwrite ACK remains successful");
assert_eq!(acknowledged.size, i64::try_from(payload.len()).expect("fixture size fits i64"));
drop(barriers);
for (slot, dir) in dirs.iter().enumerate() {
let root = dir.path().join(bucket).join(&object);
assert!(
!tokio::fs::try_exists(root.join(old_dir.to_string()))
.await
.expect("old cleanup completed")
);
assert_eq!(
tokio::fs::read(root.join(new_dir.to_string()).join("part.1"))
.await
.expect("new shard survives cleanup before GET"),
committed_parts[slot],
"old cleanup must preserve every exact committed encoded shard"
);
}
let mut reader = set
.get_object_reader(bucket, &object, None, HeaderMap::new(), &options)
.await
.expect("read final acknowledged generation");
let mut body = Vec::new();
reader
.stream
.read_to_end(&mut body)
.await
.expect("read complete final payload");
assert_eq!(body, payload);
for disk in &disks {
let final_info = disk
.read_version("", bucket, &object, "", &ReadOptions::default())
.await
.expect("final physical generation");
assert_eq!(final_info.data_dir, Some(new_dir), "old cleanup must not revert the healed generation");
}
println!(
"native_g01: geometry={disk_count}/{parity} read_repair={read_repair} overwrite_ack_preserved final_physical_generation_and_full_body_verified"
);
}
}
#[tokio::test]
async fn committed_put_releases_namespace_lock_before_old_data_cleanup() {
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
@@ -22845,116 +23125,6 @@ mod put_object_tmp_cleanup_tests {
);
}
/// Per-drive view of an object: every version (including free versions)
/// recorded in its xl.meta, and the data directories next to it. A missing
/// xl.meta and one without versions both hold no version.
async fn per_drive_object_state(
dirs: &[TempDir],
bucket: &str,
object: &str,
) -> Vec<(Vec<(Option<Uuid>, Option<Uuid>, Option<OffsetDateTime>)>, Vec<String>)> {
let mut drives = Vec::with_capacity(dirs.len());
for dir in dirs {
let object_dir = dir.path().join(bucket).join(object);
let versions = match tokio::fs::read(object_dir.join(STORAGE_FORMAT_FILE)).await {
Ok(encoded) => {
let all = FileMeta::load(&encoded)
.expect("xl.meta should decode")
.get_all_file_info_versions(bucket, object, true)
.expect("xl.meta versions should be readable");
all.versions
.iter()
.chain(all.free_versions.iter())
.map(|version| (version.version_id, version.data_dir, version.mod_time))
.collect()
}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Vec::new(),
Err(err) => panic!("xl.meta at {object_dir:?} should be readable: {err}"),
};
let mut data_dirs = Vec::new();
if let Ok(mut entries) = tokio::fs::read_dir(&object_dir).await {
while let Some(entry) = entries.next_entry().await.expect("object directory entry should be readable") {
if entry
.file_type()
.await
.expect("object entry type should be readable")
.is_dir()
{
data_dirs.push(entry.file_name().to_string_lossy().into_owned());
}
}
}
data_dirs.sort();
drives.push((versions, data_dirs));
}
drives
}
/// Stage F lists any version present on half of the drives, so a PUT that
/// misses write quorum must not leave its version behind once its rollback
/// completes: no drive may keep the new xl.meta entry or its data directory.
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn failed_put_rollback_leaves_no_new_version_on_any_drive() {
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
for write_completion in [WriteCompletion::Quorum, WriteCompletion::TailDrained] {
for size in [4096, TEST_OBJECT_SIZE] {
// (versioned, seed an earlier version first)
for (versioned, seeded) in [(false, false), (false, true), (true, true)] {
let case = format!("{write_completion:?} size={size} versioned={versioned} seeded={seeded}");
let (dirs, disks, set) = hermetic_set_disks(4).await;
let bucket = "put-rollback-no-residue";
let object = "rolled-back-object";
make_completion_test_bucket(&disks, bucket).await;
if seeded {
set.put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![b'0'; size]),
&ObjectOptions {
versioned,
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.unwrap_or_else(|err| panic!("{case}: seed version should commit on every drive: {err}"));
}
wait_for_tmp_workspace_to_drain(&dirs, "seed PUT must leave no staging").await;
let before = per_drive_object_state(&dirs, bucket, object).await;
let tasks = rename_fanout_barrier::observe_tasks(object);
let _fault = rename_fault_injection::fail_rename_on(object, &[0, 1]);
let err = set
.put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![b'1'; size]),
&ObjectOptions {
versioned,
write_completion,
..Default::default()
},
)
.await
.expect_err("two renamed drives cannot satisfy write quorum three");
assert!(
matches!(err, Error::ErasureWriteQuorum | Error::InsufficientWriteQuorum(_, _)),
"{case}: original quorum error expected: {err}"
);
assert_eq!(tasks.running(), 0, "{case}: rename and undo tasks must be reaped before return");
// An incomplete rollback retains staging for recovery, so a
// drained workspace shows the rollback completed.
wait_for_tmp_workspace_to_drain(&dirs, "a completed rollback must release staging").await;
let after = per_drive_object_state(&dirs, bucket, object).await;
assert_eq!(after, before, "{case}: no drive may keep the failed PUT's version or data");
}
}
}
})
.await;
}
#[tokio::test]
#[serial_test::serial(capacity_dirty_scope)]
async fn put_incomplete_rollback_preserves_staging_and_old_version_backup() {
+135 -523
View File
@@ -264,66 +264,6 @@ enum MetadataReadPurpose {
GetObjectReader,
}
/// How a metadata read-quorum failure is reported when the replies prove the
/// object or version dangling. Only the object read entry points
/// (`get_object_info`, `get_object_reader` and prepared GET metadata) report
/// it as not found; read-before-write callers of `get_object_fileinfo*` keep
/// the quorum error so fences and delete recovery never treat a torn write as
/// an absent key.
#[derive(Clone, Copy, PartialEq, Eq)]
enum DanglingRead {
KeepQuorumError,
ReportNotFound,
}
/// Read-only form of MinIO's GET-path dangling check: once
/// `totalResp == setDriveCount`, `getObjectFileInfo` calls `deleteIfDangling`
/// and a dangling verdict replaces the read-quorum error with
/// `errFileNotFound` or `errFileVersionNotFound`.
///
/// Each deliberate difference keeps the quorum error instead:
/// - nothing is deleted; heal owns cleanup and its recent-write grace window;
/// - MinIO counts an offline drive as a response and lets `isObjectDangling`
/// reject it, while here every slot must hold valid metadata or a not-found
/// reply, so an offline drive, a timeout, an unanswered slot, or a corrupt or
/// unusable xl.meta stays uncertain;
/// - the valid replies must be one metadata generation and not a remote
/// (transitioned) version, the candidates heal's `dangling_delete_safety`
/// also refuses to delete.
fn dangling_read_not_found(
bucket: &str,
version_id: &str,
parts_metadata: &[FileInfo],
errs: &[Option<DiskError>],
fanout_complete: bool,
) -> Option<DiskError> {
if !fanout_complete || crate::bucket::utils::is_meta_bucketname(bucket) {
return None;
}
let mut generation = None;
for (metadata, err) in parts_metadata.iter().zip(errs) {
match err {
Some(DiskError::FileNotFound | DiskError::FileVersionNotFound) => {}
Some(_) => return None,
None => {
if !super::file_info_is_valid_for_metadata(metadata) || metadata.is_remote() {
return None;
}
let identity = SetDisks::file_info_quorum_hash(metadata);
if *generation.get_or_insert(identity) != identity {
return None;
}
}
}
}
let (_, dangling) = super::is_object_dangling(parts_metadata, errs, &std::collections::HashMap::new());
dangling.then_some(if version_id.is_empty() {
DiskError::FileNotFound
} else {
DiskError::FileVersionNotFound
})
}
impl SetDisks {
/// A startup quorum can leave remote slots unregistered even after their
/// peers become ready. Refresh those slots before taking the GET metadata
@@ -487,7 +427,6 @@ impl SetDisks {
}
}
#[cfg(test)]
async fn cache_get_object_fileinfo(
&self,
identity: (&str, &str),
@@ -498,32 +437,8 @@ impl SetDisks {
read_quorum: usize,
) {
let (fi, capability) = selected;
let _ = self
.cache_get_object_fileinfo_owned(
identity,
generation,
(fi.clone(), capability, None),
parts_metadata.to_vec(),
online_disks.to_vec(),
read_quorum,
)
.await;
}
async fn cache_get_object_fileinfo_owned(
&self,
identity: (&str, &str),
generation: Option<GetObjectMetadataCacheGeneration>,
selected: (FileInfo, GetObjectMetadataCacheCapability, Option<Bytes>),
parts_metadata: Vec<FileInfo>,
online_disks: Vec<Option<DiskStore>>,
read_quorum: usize,
) -> GetObjectFileInfo {
let (fi, capability, verified_inline_body) = selected;
if fi.deleted || !fi.has_valid_erasure_geometry() {
let mut snapshot = GetObjectFileInfo::owned(fi, parts_metadata, online_disks);
snapshot.set_verified_inline_body(verified_inline_body);
return snapshot;
return;
}
let (bucket, object) = identity;
@@ -534,24 +449,20 @@ impl SetDisks {
GET_METADATA_CACHE_DECISION_REJECT,
GET_METADATA_CACHE_REASON_STALE_PUBLICATION,
);
let mut snapshot = GetObjectFileInfo::owned(fi, parts_metadata, online_disks);
snapshot.set_verified_inline_body(verified_inline_body);
return snapshot;
return;
};
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
let entry = Arc::new(GetObjectMetadataCacheEntry {
created_at: Instant::now(),
fi,
parts_metadata,
online_disks,
fi: fi.clone(),
parts_metadata: parts_metadata.to_vec(),
online_disks: online_disks.to_vec(),
read_quorum,
capability,
verified_inline_body,
});
self.insert_get_object_metadata_cache_entry_after_insert(key, generation, Arc::clone(&entry), || {})
self.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || {})
.await;
GetObjectFileInfo::owned_published(entry)
}
async fn insert_get_object_metadata_cache_entry_after_insert(
@@ -672,31 +583,6 @@ impl SetDisks {
read_data,
caller_allows_early_stop,
MetadataReadPurpose::FullLayout,
DanglingRead::KeepQuorumError,
)
.await
}
/// `get_object_fileinfo` for `get_object_info` and prepared GET metadata:
/// a provably dangling object or version reads as not found.
#[tracing::instrument(level = "debug", skip(self))]
#[hotpath::measure(impl_type = "SetDisks")]
pub(super) async fn get_object_fileinfo_for_object_read(
&self,
bucket: &str,
object: &str,
opts: &ObjectOptions,
read_data: bool,
caller_allows_early_stop: bool,
) -> Result<GetObjectFileInfo> {
self.get_object_fileinfo_gated_inner(
bucket,
object,
opts,
read_data,
caller_allows_early_stop,
MetadataReadPurpose::FullLayout,
DanglingRead::ReportNotFound,
)
.await
}
@@ -718,7 +604,6 @@ impl SetDisks {
read_data,
caller_allows_early_stop,
MetadataReadPurpose::GetObjectReader,
DanglingRead::ReportNotFound,
)
.await
}
@@ -736,16 +621,8 @@ impl SetDisks {
read_data: bool,
allow_early_stop: bool,
) -> Result<GetObjectFileInfo> {
self.get_object_fileinfo_gated_inner(
bucket,
object,
opts,
read_data,
allow_early_stop,
MetadataReadPurpose::FullLayout,
DanglingRead::KeepQuorumError,
)
.await
self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, allow_early_stop, MetadataReadPurpose::FullLayout)
.await
}
#[allow(clippy::too_many_arguments)]
@@ -757,7 +634,6 @@ impl SetDisks {
read_data: bool,
allow_early_stop: bool,
purpose: MetadataReadPurpose,
dangling_read: DanglingRead,
) -> Result<GetObjectFileInfo> {
let vid = opts.version_id.clone().unwrap_or_default();
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
@@ -917,17 +793,7 @@ impl SetDisks {
let _min_disks = self.set_drive_count - self.default_parity_count;
let metadata_resolve_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
// Like MinIO, check for a dangling object after either metadata quorum
// reduction fails; only a read-quorum failure is reclassified.
let dangling_not_found = |err: &DiskError| {
if dangling_read == DanglingRead::ReportNotFound && *err == DiskError::ErasureReadQuorum {
dangling_read_not_found(bucket, &vid, &parts_metadata, &errs, metadata_fanout_complete)
} else {
None
}
};
let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)
.map_err(|err| dangling_not_found(&err).unwrap_or(err))
.map_err(|err| to_object_err(err.into(), vec![bucket, object, &vid]))
{
Ok(v) => v,
@@ -947,15 +813,6 @@ impl SetDisks {
let write_quorum = usize::try_from(write_quorum)
.map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
if let Some(not_found) = dangling_not_found(&err) {
rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "not_found");
record_get_stage_duration_if_enabled(
GET_OBJECT_PATH_SET_DISK,
GET_STAGE_METADATA_RESOLVE,
metadata_resolve_stage_start,
);
return Err(to_object_err(not_found.into(), vec![bucket, object, &vid]));
}
rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "not_reached");
error!("reduce_read_quorum_errs: {:?}, bucket: {}, object: {}", &err, bucket, object);
record_get_stage_duration_if_enabled(
@@ -988,11 +845,7 @@ impl SetDisks {
}
}
metadata_fanout_diagnostics.record_quorum_candidate_latency(metadata_metrics_path, fileinfo_selection_quorum);
let needs_late_metadata_fanout = !metadata_fanout_complete
&& allow_early_stop
&& non_inline_data_read_early_stop_allowed(read_data, bucket, object)
&& late_materialization_candidate_is_safe(&fi);
let mut snapshot = if !opts.suppress_read_repair && errs.iter().any(|err| err.is_some()) {
if !opts.suppress_read_repair && errs.iter().any(|err| err.is_some()) {
let version_id = resolved_read_repair_version_id(&fi, opts.version_id.as_deref());
submit_read_repair_heal(
&fi.volume,
@@ -1004,9 +857,6 @@ impl SetDisks {
"metadata_read_error",
)
.await;
let mut snapshot = GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks);
snapshot.set_verified_inline_body(verified_inline_body);
snapshot
} else if use_metadata_cache
&& (metadata_fanout_complete
|| condition_stopped
@@ -1019,11 +869,11 @@ impl SetDisks {
{
#[cfg(test)]
metadata_cache_tests::wait_before_metadata_cache_publish(bucket, object).await;
self.cache_get_object_fileinfo_owned(
self.cache_get_object_fileinfo(
(bucket, object),
metadata_cache_generation,
(
fi,
&fi,
if metadata_fanout_complete {
GetObjectMetadataCacheCapability::Full
} else if condition_stopped {
@@ -1031,23 +881,29 @@ impl SetDisks {
} else {
GetObjectMetadataCacheCapability::QuorumSelected
},
verified_inline_body,
),
parts_metadata,
op_online_disks,
&parts_metadata,
&op_online_disks,
read_quorum,
)
.await
} else {
let mut snapshot = GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks);
snapshot.set_verified_inline_body(verified_inline_body);
snapshot
};
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_METADATA_RESOLVE, metadata_resolve_stage_start);
if needs_late_metadata_fanout {
snapshot.set_late_metadata_fanout_disks(disks);
.await;
}
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_METADATA_RESOLVE, metadata_resolve_stage_start);
// debug!("get_object_fileinfo pick fi {:?}", &fi);
// let online_disks: Vec<Option<DiskStore>> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect();
let mut snapshot = if !metadata_fanout_complete
&& allow_early_stop
&& non_inline_data_read_early_stop_allowed(read_data, bucket, object)
&& late_materialization_candidate_is_safe(&fi)
{
GetObjectFileInfo::owned_with_late_metadata_fanout(fi, parts_metadata, op_online_disks, disks)
} else {
GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks)
};
snapshot.condition_stopped = condition_stopped;
snapshot.set_verified_inline_body(verified_inline_body);
Ok(snapshot)
}
@@ -1576,7 +1432,10 @@ impl SetDisks {
"Create bitrot reader failed read quorum"
);
record_get_object_pipeline_failure(GET_STAGE_READER_SETUP, reason);
return Err(to_object_err(read_err.into(), vec![bucket, object]));
// The metadata selected this object, but the part cannot
// be reconstructed. A dominant disk I/O/not-found error
// must not erase the object's read-quorum failure.
return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
}
let reason = GetObjectFailureReason::ReadQuorum;
error!(
@@ -1601,7 +1460,7 @@ impl SetDisks {
"Create bitrot reader did not have enough disks"
);
record_get_object_pipeline_failure(GET_STAGE_READER_SETUP, reason);
return Err(Error::other(format!("not enough disks to read: {:?}", reader_setup.errors)));
return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
}
// Check if we have missing shards even though we can read successfully
@@ -2136,10 +1995,10 @@ impl SetDisks {
if let Some(read_err) = reduce_read_quorum_errs(&reader_setup.errors, OBJECT_OP_IGNORED_ERRS, erasure.data_shards) {
let reason = classify_disk_error(&read_err);
record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, reason);
return Err(to_object_err(read_err.into(), vec![bucket, object]));
return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
}
record_get_object_pipeline_failure_for_path(metrics_path, GET_STAGE_READER_SETUP, GetObjectFailureReason::ReadQuorum);
return Err(Error::other(format!("not enough disks to read: {:?}", reader_setup.errors)));
return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
}
let missing_shards = reader_setup.completed_failed_shards();
@@ -4174,7 +4033,6 @@ mod metadata_cache_tests {
parts_metadata: vec![fi],
online_disks: vec![None],
read_quorum: 1,
verified_inline_body: None,
}),
)
.await;
@@ -4836,7 +4694,6 @@ mod metadata_cache_tests {
online_disks: Vec::new(),
read_quorum: 0,
capability,
verified_inline_body: None,
})
}
@@ -5329,7 +5186,6 @@ mod metadata_cache_tests {
parts_metadata: vec![fi],
online_disks: vec![None],
read_quorum: 1,
verified_inline_body: None,
}),
)
.await;
@@ -5448,7 +5304,6 @@ mod metadata_cache_tests {
parts_metadata: vec![fi],
online_disks: Vec::new(),
read_quorum: 0,
verified_inline_body: None,
});
set.insert_get_object_metadata_cache_entry_after_insert(key, generation, entry, || {
@@ -5584,7 +5439,6 @@ mod metadata_cache_tests {
fi,
online_disks: Vec::new(),
read_quorum: 0,
verified_inline_body: None,
})
};
@@ -7950,6 +7804,105 @@ mod tests {
(data_dir, files)
}
#[tokio::test]
#[serial_test::serial]
async fn external_missing_shards_keep_typed_read_quorum() {
use tokio::io::AsyncReadExt;
let payload = vec![0x37; 256 * 1024];
for readable_shards in [1, 2, 4] {
let erasure = coding::Erasure::new(4, 2, 1024 * 1024);
let mut fi = codec_streaming_test_fileinfo(i64::try_from(payload.len()).expect("fixture size fits i64"), 1);
fi.erasure.data_blocks = erasure.data_shards;
fi.erasure.parity_blocks = erasure.parity_shards;
fi.erasure.block_size = erasure.block_size;
fi.erasure.distribution = (1..=erasure.total_shard_count()).collect();
let (dirs, disks) = local_test_disks(erasure.total_shard_count(), CODEC_STREAMING_TEST_BUCKET).await;
let (data_dir, files) = codec_streaming_external_files(&erasure, &payload, &disks).await;
fi.data_dir = Some(data_dir);
for directory in dirs.iter().skip(readable_shards) {
tokio::fs::remove_file(
directory
.path()
.join(CODEC_STREAMING_TEST_BUCKET)
.join(CODEC_STREAMING_TEST_OBJECT)
.join(data_dir.to_string())
.join("part.1"),
)
.await
.expect("remove real part while retaining its selected metadata");
}
let mut output = Vec::new();
let legacy = SetDisks::get_object_with_fileinfo(
CODEC_STREAMING_TEST_BUCKET,
CODEC_STREAMING_TEST_OBJECT,
Arc::new(ErasureCache::new()),
0,
fi.size,
&mut output,
fi.clone(),
files.clone(),
&disks,
0,
0,
false,
true,
false,
false,
GET_OBJECT_PATH_LEGACY_DUPLEX,
"plain_single_part",
"le_1mib",
)
.await;
let streaming = SetDisks::get_object_mid_size_reader_with_fileinfo(
CODEC_STREAMING_TEST_BUCKET,
CODEC_STREAMING_TEST_OBJECT,
Arc::new(ErasureCache::new()),
&fi,
&files,
&disks,
0,
0,
false,
true,
"plain_single_part",
"le_1mib",
false,
)
.await;
if readable_shards < erasure.data_shards {
for error in [
legacy.expect_err("legacy read must reject quorum minus one"),
streaming.err().expect("streaming setup must reject quorum minus one"),
] {
assert_eq!(
error,
Error::InsufficientReadQuorum(
CODEC_STREAMING_TEST_BUCKET.to_owned(),
CODEC_STREAMING_TEST_OBJECT.to_owned(),
)
);
}
assert!(output.is_empty(), "quorum rejection must precede all response bytes");
} else {
legacy.expect("legacy reader retains reconstructable quorum");
assert_eq!(output, payload, "legacy read returns every original byte at quorum");
let GetCodecStreamingReaderBuildOutcome::Reader(mut reader) =
streaming.expect("streaming quorum remains readable")
else {
panic!("mid-size quorum must yield an actual reader");
};
let mut decoded = Vec::new();
reader
.read_to_end(&mut decoded)
.await
.expect("streaming reader returns the complete body");
assert_eq!(decoded, payload, "streaming read returns every original byte at quorum");
}
}
}
#[test]
#[serial]
fn payload_read_repair_is_suppressed_only_for_read_only_requests() {
@@ -9740,344 +9693,3 @@ mod tests {
assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
}
}
#[cfg(test)]
mod dangling_read_tests {
//! GET and HEAD report a metadata read-quorum failure as not found only when
//! every drive answered and the surviving copies prove the object or version
//! dangling; any uncertainty keeps the read-quorum error (503 SlowDownRead).
use super::*;
use crate::disk::{DeleteOptions, STORAGE_FORMAT_FILE};
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
use crate::storage_api_contracts::object::ObjectOperations;
use http::HeaderMap;
use std::path::PathBuf;
use tempfile::TempDir;
use tokio::io::AsyncReadExt;
/// EC 4+2: read quorum is four, so three surviving copies are below it
/// while still reaching the half-set probe.
const DRIVES: usize = 6;
const PARITY: usize = 2;
async fn dangling_test_set(bucket: &str) -> (Vec<TempDir>, Arc<SetDisks>) {
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
ctx.update_erasure_type(crate::layout::endpoints::SetupType::Erasure).await;
let (dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(DRIVES, PARITY, ctx).await;
set.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("test bucket should be created on every drive");
(dirs, set)
}
async fn put(set: &SetDisks, bucket: &str, object: &str, body: &[u8], versioned: bool) -> ObjectInfo {
set.put_object(
bucket,
object,
&mut PutObjReader::from_vec(body.to_vec()),
&ObjectOptions {
versioned,
..Default::default()
},
)
.await
.expect("seed object should commit on every drive")
}
fn object_dir(dirs: &[TempDir], disk: usize, bucket: &str, object: &str) -> PathBuf {
dirs[disk].path().join(bucket).join(object)
}
async fn remove_object_from(dirs: &[TempDir], bucket: &str, object: &str, disks: &[usize]) {
for &disk in disks {
tokio::fs::remove_dir_all(object_dir(dirs, disk, bucket, object))
.await
.expect("object directory should be removable to model a drive that never received the write");
}
}
fn read_opts(version_id: Option<&str>) -> ObjectOptions {
ObjectOptions {
version_id: version_id.map(str::to_owned),
..Default::default()
}
}
async fn head(set: &SetDisks, bucket: &str, object: &str, version_id: Option<&str>) -> Result<ObjectInfo> {
set.get_object_info(bucket, object, &read_opts(version_id)).await
}
async fn get(set: &SetDisks, bucket: &str, object: &str, version_id: Option<&str>) -> Result<Vec<u8>> {
let mut reader = set
.get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts(version_id))
.await?;
let mut body = Vec::new();
reader
.stream
.read_to_end(&mut body)
.await
.expect("an opened object should stream its body");
Ok(body)
}
fn is_read_quorum_error(err: &StorageError) -> bool {
matches!(err, StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))
}
async fn assert_get_and_head_keep_read_quorum_error(set: &SetDisks, bucket: &str, object: &str, case: &str) {
let Err(err) = head(set, bucket, object, None).await else {
panic!("{case}: HEAD must not serve an object below read quorum");
};
assert!(is_read_quorum_error(&err), "{case}: HEAD must keep the read-quorum error: {err:?}");
let Err(err) = get(set, bucket, object, None).await else {
panic!("{case}: GET must not serve an object below read quorum");
};
assert!(is_read_quorum_error(&err), "{case}: GET must keep the read-quorum error: {err:?}");
}
#[tokio::test]
async fn dangling_object_reads_as_not_found_on_get_and_head() {
let bucket = "dangling-read-unversioned";
let object = "torn/object";
let (dirs, set) = dangling_test_set(bucket).await;
put(&set, bucket, object, b"torn write payload", false).await;
// Three copies stay below read quorum four; the other drives answer FileNotFound.
remove_object_from(&dirs, bucket, object, &[3, 4, 5]).await;
let is_not_found = |err: &StorageError| matches!(err, StorageError::ObjectNotFound(b, o) if b == bucket && o == object);
let Err(err) = head(&set, bucket, object, None).await else {
panic!("HEAD must not serve a dangling object");
};
assert!(is_not_found(&err), "HEAD of a dangling object must be NoSuchKey: {err:?}");
let Err(err) = get(&set, bucket, object, None).await else {
panic!("GET must not serve a dangling object");
};
assert!(is_not_found(&err), "GET of a dangling object must be NoSuchKey: {err:?}");
let Err(err) = set
.prepare_get_object_metadata(bucket, object, &ObjectOptions::default())
.await
else {
panic!("prepared GET metadata must not resolve a dangling object");
};
assert!(is_not_found(&err), "prepared GET metadata must be NoSuchKey: {err:?}");
// Read-before-write callers keep the quorum error so fences and delete
// recovery never treat a torn write as an absent key.
let Err(err) = set
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, false)
.await
else {
panic!("internal metadata reads must not resolve a dangling object");
};
assert!(is_read_quorum_error(&err), "internal metadata reads keep the quorum error: {err:?}");
// The classification is read-only; heal owns dangling cleanup.
for disk in 0..3 {
assert!(
object_dir(&dirs, disk, bucket, object).join(STORAGE_FORMAT_FILE).exists(),
"GET/HEAD must not delete the surviving copy on drive {disk}"
);
}
}
#[tokio::test]
async fn dangling_classification_requires_every_drive_to_answer() {
let bucket = "dangling-read-offline";
let object = "torn/object";
let (dirs, set) = dangling_test_set(bucket).await;
let payload = b"offline drive may hold the fourth copy";
put(&set, bucket, object, payload, false).await;
remove_object_from(&dirs, bucket, object, &[3, 4]).await;
let offline = set.disks.write().await[5].take().expect("drive five should start online");
assert_get_and_head_keep_read_quorum_error(&set, bucket, object, "one drive offline").await;
// The offline drive held a copy: once it answers, read quorum is met.
set.disks.write().await[5] = Some(offline);
let body = get(&set, bucket, object, None)
.await
.expect("four copies satisfy read quorum once the drive returns");
assert_eq!(body, payload);
}
#[tokio::test]
async fn dangling_version_reads_as_no_such_version_while_other_versions_stay_readable() {
let bucket = "dangling-read-versioned";
let object = "torn/versioned";
let (_dirs, set) = dangling_test_set(bucket).await;
let v1 = put(&set, bucket, object, b"version one", true)
.await
.version_id
.expect("versioned PUT should assign a version id");
let v2 = put(&set, bucket, object, b"version two", true)
.await
.version_id
.expect("versioned PUT should assign a version id");
let disks = set.disks.read().await.clone();
for disk in disks.iter().skip(3) {
let disk = disk.as_ref().expect("every drive should be online");
let mut remove = FileInfo {
volume: bucket.to_string(),
name: object.to_string(),
version_id: Some(v2),
..Default::default()
};
remove.set_skip_tier_free_version();
disk.delete_version(bucket, object, remove, false, DeleteOptions::default())
.await
.expect("dropping version two should model a drive that never received it");
assert!(matches!(
disk.read_version("", bucket, object, &v2.to_string(), &ReadOptions::default())
.await,
Err(DiskError::FileVersionNotFound | DiskError::FileNotFound)
));
}
let v1 = v1.to_string();
let v2 = v2.to_string();
let is_no_such_version =
|err: &StorageError| matches!(err, StorageError::VersionNotFound(b, o, v) if b == bucket && o == object && v == &v2);
let Err(err) = head(&set, bucket, object, Some(&v2)).await else {
panic!("HEAD must not serve a dangling version");
};
assert!(is_no_such_version(&err), "HEAD of a dangling version must be NoSuchVersion: {err:?}");
let Err(err) = get(&set, bucket, object, Some(&v2)).await else {
panic!("GET must not serve a dangling version");
};
assert!(is_no_such_version(&err), "GET of a dangling version must be NoSuchVersion: {err:?}");
assert_eq!(
get(&set, bucket, object, Some(&v1))
.await
.expect("the committed version must stay readable"),
b"version one"
);
head(&set, bucket, object, Some(&v1))
.await
.expect("HEAD of the committed version must succeed");
// The latest read sees two generations (version two on three drives,
// version one on the rest), which is not proof of absence.
let Err(err) = head(&set, bucket, object, None).await else {
panic!("a split latest version must not resolve without quorum");
};
assert!(is_read_quorum_error(&err), "a split latest version keeps the quorum error: {err:?}");
}
#[tokio::test]
async fn dangling_classification_keeps_quorum_error_for_corrupt_metadata() {
let bucket = "dangling-read-corrupt";
let object = "torn/object";
let (dirs, set) = dangling_test_set(bucket).await;
put(&set, bucket, object, b"corrupt copies are present, not absent", false).await;
for disk in 3..DRIVES {
tokio::fs::write(object_dir(&dirs, disk, bucket, object).join(STORAGE_FORMAT_FILE), b"corrupt metadata")
.await
.expect("xl.meta should be corruptible for the test");
}
assert_get_and_head_keep_read_quorum_error(&set, bucket, object, "corrupt xl.meta").await;
}
#[tokio::test]
async fn dangling_classification_keeps_quorum_error_for_two_generations() {
let bucket = "dangling-read-two-generations";
let object = "torn/object";
let (dirs, set) = dangling_test_set(bucket).await;
put(&set, bucket, object, b"first generation", false).await;
let mut first_generation = Vec::new();
for disk in 0..2 {
let path = object_dir(&dirs, disk, bucket, object).join(STORAGE_FORMAT_FILE);
let bytes = tokio::fs::read(&path)
.await
.expect("first generation xl.meta should be readable");
first_generation.push((path, bytes));
}
put(&set, bucket, object, b"second generation", false).await;
// First generation on two drives, second on one, three drives absent:
// each generation is below read quorum and neither proves the other absent.
for (path, bytes) in first_generation {
tokio::fs::write(&path, bytes)
.await
.expect("first generation xl.meta should be restorable");
}
remove_object_from(&dirs, bucket, object, &[3, 4, 5]).await;
assert_get_and_head_keep_read_quorum_error(&set, bucket, object, "two generations").await;
}
/// Writes `fi` as the only version on the first `copies` drives; the
/// remaining drives answer FileNotFound.
async fn write_metadata_copies(set: &SetDisks, bucket: &str, object: &str, fi: &FileInfo, copies: usize) {
let disks = set.disks.read().await.clone();
for (index, disk) in disks.iter().take(copies).enumerate() {
let mut copy = fi.clone();
copy.erasure.index = index + 1;
disk.as_ref()
.expect("every drive should be online")
.write_metadata(bucket, bucket, object, copy)
.await
.expect("hand-written metadata should be stored");
}
}
fn handwritten_version(bucket: &str, object: &str, data_blocks: usize, parity_blocks: usize) -> FileInfo {
let mut fi = FileInfo::new(object, data_blocks, parity_blocks);
fi.volume = bucket.to_string();
fi.name = object.to_string();
fi.size = 1;
fi.mod_time = Some(OffsetDateTime::now_utc());
fi.metadata.insert("etag".to_string(), "handwritten-etag".to_string());
fi.add_object_part(1, "part-etag".to_string(), 1, None, 1, None, None);
fi
}
#[tokio::test]
async fn dangling_zero_parity_version_reads_as_not_found() {
// An EC:0 version on a default EC:2 set passes the common-parity check
// with four of six copies (n/2 + 1), so its read fails at the later
// read-quorum reduction, which must classify it as well.
let bucket = "dangling-read-zero-parity";
let object = "torn/zero-parity";
let (_dirs, set) = dangling_test_set(bucket).await;
write_metadata_copies(&set, bucket, object, &handwritten_version(bucket, object, DRIVES, 0), 4).await;
let is_not_found = |err: &StorageError| matches!(err, StorageError::ObjectNotFound(b, o) if b == bucket && o == object);
let Err(err) = head(&set, bucket, object, None).await else {
panic!("HEAD must not serve a zero-parity object missing two shards");
};
assert!(is_not_found(&err), "HEAD of a dangling zero-parity object must be NoSuchKey: {err:?}");
let Err(err) = get(&set, bucket, object, None).await else {
panic!("GET must not serve a zero-parity object missing two shards");
};
assert!(is_not_found(&err), "GET of a dangling zero-parity object must be NoSuchKey: {err:?}");
}
#[tokio::test]
async fn dangling_classification_keeps_quorum_error_for_remote_versions() {
// Local metadata is the only pointer to tiered data, and heal never
// deletes a remote candidate as dangling, so GET must not report it absent.
let bucket = "dangling-read-remote";
let object = "torn/transitioned";
let (_dirs, set) = dangling_test_set(bucket).await;
let mut fi = handwritten_version(bucket, object, DRIVES - PARITY, PARITY);
fi.transition_status = crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE.to_string();
fi.transition_tier = "WARM".to_string();
fi.transitioned_objname = "remote/torn/transitioned".to_string();
assert!(fi.is_remote(), "the fixture must be a remote version");
write_metadata_copies(&set, bucket, object, &fi, 3).await;
assert_get_and_head_keep_read_quorum_error(&set, bucket, object, "remote version").await;
}
#[tokio::test]
async fn dangling_classification_skips_the_meta_bucket() {
// Like MinIO, internal objects are never classified: their readers
// treat not-found as absent configuration.
let object = "config/dangling-read.json";
let (dirs, set) = dangling_test_set("dangling-read-meta").await;
put(&set, RUSTFS_META_BUCKET, object, b"internal config", false).await;
remove_object_from(&dirs, RUSTFS_META_BUCKET, object, &[3, 4, 5]).await;
assert_get_and_head_keep_read_quorum_error(&set, RUSTFS_META_BUCKET, object, "meta bucket").await;
}
}
+204 -3
View File
@@ -2167,6 +2167,210 @@ mod tests {
shutdown.cancel();
}
#[tokio::test]
#[serial_test::serial]
async fn bucket_incarnation_fence_survives_caller_abort_at_real_repair_commit() {
use crate::set_disk::ReadRepairPauseScope;
let (_root, store, shutdown) = multi_pool_heal_store().await;
let bucket = format!("heal-commit-owner-{}", Uuid::new_v4().simple());
let object = "history.bin";
let version = Uuid::new_v4().to_string();
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("original bucket");
let old = store
.bucket_incarnation_id_from_disk(&bucket)
.await
.expect("original identity");
let set = store.pools[0].get_disks(0);
let mut reader = PutObjReader::from_vec(vec![0x17; 1024 * 1024]);
set.put_object(
&bucket,
object,
&mut reader,
&ObjectOptions {
versioned: true,
version_id: Some(version.clone()),
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("original acknowledged version");
let missing = set.disks.read().await[0].clone().expect("repair target");
let healthy = set.disks.read().await[1].clone().expect("healthy target");
let baseline = healthy
.read_all(&bucket, &format!("{object}/xl.meta"))
.await
.expect("healthy metadata baseline");
let original = healthy
.read_version("", &bucket, object, &version, &Default::default())
.await
.expect("original version metadata");
assert!(original.data.is_none(), "the commit barrier fixture must use actual external data shards");
missing
.delete(
&bucket,
object,
DeleteOptions {
recursive: true,
immediate: true,
..Default::default()
},
)
.await
.expect("remove one physical shard");
// The scope pauses the real repair after reconstruction, immediately
// before its generation check and final shard rename.
let (_pause, arrived, release) = ReadRepairPauseScope::install_incarnation_commit(&bucket, object);
let (drained_tx, drained_rx) = tokio::sync::oneshot::channel();
let worker_store = store.clone();
let worker_bucket = bucket.clone();
let worker_version = version.clone();
let opts = HealOpts {
pool: Some(0),
set: Some(0),
scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
..Default::default()
};
let mut waiter = tokio::spawn(async move {
worker_store
.run_bucket_heal_at_incarnation(&worker_bucket, old, &opts, move |store, bucket, opts| async move {
let result = store.heal_object_with_proof(&bucket, object, &worker_version, &opts).await?;
drained_tx
.send(result)
.expect("retain the actual storage result after caller cancellation");
Ok(())
})
.await
});
tokio::select! {
paused = tokio::time::timeout(std::time::Duration::from_secs(10), arrived.notified()) => {
paused.expect("real missing-shard repair reaches the final commit pause");
}
completed = &mut waiter => panic!("repair ended before the actual final commit barrier: {completed:?}"),
}
assert!(
missing.read_xl(&bucket, object, false).await.is_err(),
"final shard is not published at the pause"
);
println!("incarnation_commit_barrier: final_rename_paused");
waiter.abort();
assert!(waiter.await.expect_err("caller aborted").is_cancelled());
let mut writer = Box::pin(store.acquire_bucket_lifecycle_write_lock(&bucket));
assert!(
futures::poll!(&mut writer).is_pending(),
"bucket deletion/recreation cannot acquire its generation writer while the cancelled repair still owns commit"
);
println!("incarnation_commit_barrier: caller_aborted_writer_pending");
release.notify_one();
let result = tokio::time::timeout(std::time::Duration::from_secs(10), drained_rx)
.await
.expect("actual repair drains after release")
.expect("retained repair result");
assert!(
result.error.is_none(),
"physical repair must succeed before lifecycle handoff: {:?}",
result.error
);
let guard = tokio::time::timeout(std::time::Duration::from_secs(10), writer)
.await
.expect("generation writer acquires only after physical repair drains")
.expect("generation writer");
assert!(
missing.read_xl(&bucket, object, false).await.is_ok(),
"the physical shard is published before writer admission"
);
let repaired = missing
.read_version("", &bucket, object, &version, &Default::default())
.await
.expect("repaired exact version metadata");
assert_eq!(repaired.version_id, original.version_id);
assert_eq!(repaired.data_dir, original.data_dir);
let checked = missing
.check_parts(&bucket, object, &repaired)
.await
.expect("verify the published physical shard");
assert_eq!(
checked.results,
vec![crate::disk::CHECK_PART_SUCCESS],
"the actual published part passes bitrot verification"
);
assert_eq!(
healthy
.read_all(&bucket, &format!("{object}/xl.meta"))
.await
.expect("healthy after repair"),
baseline,
"repair does not overwrite the surviving version"
);
println!("incarnation_commit_barrier: repair_drained_writer_acquired");
drop(guard);
set.delete_object(
&bucket,
object,
ObjectOptions {
versioned: true,
version_id: Some(version.clone()),
..Default::default()
},
)
.await
.expect("normal explicit deletion of the old version");
store
.delete_bucket(&bucket, &DeleteBucketOptions::default())
.await
.expect("normal empty bucket deletion");
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("same-name successor bucket");
let new = store
.bucket_incarnation_id_from_disk(&bucket)
.await
.expect("successor identity");
assert_ne!(old, new);
let successor_version = Uuid::new_v4().to_string();
let mut successor_reader = PutObjReader::from_vec(vec![0x29; 512 * 1024]);
set.put_object(
&bucket,
object,
&mut successor_reader,
&ObjectOptions {
versioned: true,
version_id: Some(successor_version),
write_completion: crate::object_api::WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("acknowledged successor version");
let successor = healthy
.read_all(&bucket, &format!("{object}/xl.meta"))
.await
.expect("successor metadata");
assert!(
store
.heal_object_at_incarnation(&bucket, object, &version, old, &opts)
.await
.is_err()
);
assert_eq!(
healthy
.read_all(&bucket, &format!("{object}/xl.meta"))
.await
.expect("successor after obsolete admission"),
successor,
"obsolete admission cannot change the same-name successor"
);
println!("incarnation_commit_barrier: successor_rejects_obsolete_admission");
shutdown.cancel();
}
#[tokio::test]
#[serial_test::serial]
async fn bucket_incarnation_heal_owner_survives_cancelled_waiter_and_queued_writer() {
@@ -3221,9 +3425,6 @@ mod tests {
#[serial_test::serial]
async fn full_format_heal_lost_after_last_save_does_not_publish_or_renew_and_latches() {
let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
tokio::time::timeout(std::time::Duration::from_secs(30), store.pools[0].stop_endpoint_monitor_for_test())
.await
.expect("endpoint reconnection must stop before the format-loss fixture takes ownership");
let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
store.pools[0].disk_set[0].disks.write().await[3] = None;
let barrier = HealFormatAfterSaveBarrier::install(&store.pools[0], 3);
+3 -5
View File
@@ -5090,8 +5090,8 @@ mod tests {
assert!(Arc::ptr_eq(disk, expected), "endpoint monitor must not replace an offline test handle");
assert_eq!(disk.runtime_state(), crate::disk::health_state::RuntimeDriveHealthState::Offline);
assert!(
!disk.is_online().await,
"the explicit drive fault must survive renewal despite a valid disk identity"
disk.is_online().await,
"the valid disk identity must prevent endpoint renewal while the IO health gate remains offline"
);
}
OfflineTestDisks { disks }
@@ -12018,9 +12018,7 @@ mod tests {
Ok::<(), Error>(())
}
})
// Pagination, not write throughput, is the oracle. Drain each pair of
// durable receipts before preparing the next to bound fixture IO.
.buffer_unordered(1)
.buffer_unordered(32)
.try_collect::<Vec<_>>()
.await
.expect("more than one receipt page should persist");
File diff suppressed because it is too large Load Diff
@@ -14,14 +14,14 @@
use super::*;
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::disk::{DiskAPI, STORAGE_FORMAT_FILE, WalkDirOptions};
use crate::disk::STORAGE_FORMAT_FILE;
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
use bytes::Bytes;
use rustfs_filemeta::{FileInfo, MetacacheReader, ObjectPartInfo};
use rustfs_filemeta::{FileInfo, ObjectPartInfo};
use std::collections::BTreeMap;
use std::pin::Pin;
use std::task::{Context, Poll};
use tokio::io::{AsyncWrite, DuplexStream, duplex};
use tokio::io::{AsyncWrite, DuplexStream};
use tokio::sync::Notify;
use tokio::time::timeout;
@@ -468,50 +468,6 @@ async fn list_objects_shared_corpus_offline_corrupt_and_minority_stale() {
fixture.assert_walk("", None, None, 3).await;
}
#[tokio::test]
#[serial_test::serial]
async fn list_objects_shared_corpus_quorum_delete_leaves_no_resurrectable_stale_copy() {
let mut fixture = Fixture::new("list-corpus-stale-copies", namespace(&shared_keys()), true).await;
// A delete that reached write quorum (three of four drives) leaves one
// stale copy, the most any quorum delete can leave on this set.
let deleted = ["a0", "z"];
for key in deleted {
let physical_key = rustfs_utils::path::encode_dir_object(key);
for dir in fixture.dirs.iter().take(3) {
tokio::fs::remove_dir_all(dir.path().join(&fixture.bucket).join(&physical_key))
.await
.expect("quorum delete fixture should remove the object directory");
}
}
fixture.objects.retain(|object| !deleted.contains(&object.key.as_str()));
temp_env::async_with_vars([(ENV_API_LIST_OBJECTS_QUORUM, Some("strict"))], async {
for max in [1, 3, 1000] {
fixture.assert_walk("", None, None, max).await;
fixture.assert_walk("", Some("/"), None, max).await;
}
})
.await;
// A sampling mode promises only that no committed key goes missing.
temp_env::async_with_vars([(ENV_API_LIST_OBJECTS_QUORUM, Some("optimal"))], async {
let mut listed = Vec::new();
let mut marker = None;
for _ in 0..=fixture.objects.len() + deleted.len() {
let page = fixture.page("", marker.clone(), None, 3).await;
listed.extend(page.objects.into_iter().map(|object| object.name));
if !page.is_truncated {
break;
}
marker = page.next_marker;
}
for object in &fixture.objects {
assert!(listed.contains(&object.key), "optimal LIST lost committed key {}", object.key);
}
})
.await;
}
#[tokio::test]
async fn list_objects_shared_corpus_versions_delete_markers_and_free_versions() {
let mut objects = vec![
+212 -4
View File
@@ -446,6 +446,7 @@ fn bench_metacache_resolve(c: &mut Criterion) {
requested_versions: 1,
bucket: "bucket".to_string(),
strict: false,
write_quorum_slack: 0,
candidates: Vec::new(),
});
black_box(resolved);
@@ -462,6 +463,24 @@ fn bench_metacache_resolve(c: &mut Criterion) {
requested_versions: 1,
bucket: "bucket".to_string(),
strict: false,
write_quorum_slack: 0,
candidates: Vec::new(),
});
black_box(resolved);
}
});
});
group.bench_function("8_disks_write_quorum/1000_objects", |b| {
b.iter(|| {
for entries in &sets_8_agree {
let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams {
dir_quorum: 4,
obj_quorum: 4,
requested_versions: 1,
bucket: "bucket".to_string(),
strict: true,
write_quorum_slack: 0,
candidates: Vec::new(),
});
black_box(resolved);
@@ -472,12 +491,13 @@ fn bench_metacache_resolve(c: &mut Criterion) {
group.bench_function("8_disks_disagree_merge/1000_objects", |b| {
b.iter(|| {
for entries in &sets_8_disagree {
let resolved = entries.resolve(MetadataResolutionParams {
let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams {
dir_quorum: 4,
obj_quorum: 4,
requested_versions: 1,
bucket: "bucket".to_string(),
strict: true,
write_quorum_slack: 0,
candidates: Vec::new(),
});
black_box(resolved);
@@ -488,12 +508,13 @@ fn bench_metacache_resolve(c: &mut Criterion) {
group.bench_function("8_disks_agree_13_versions/1000_objects", |b| {
b.iter(|| {
for entries in &sets_8_agree_13v {
let resolved = entries.resolve(MetadataResolutionParams {
let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams {
dir_quorum: 4,
obj_quorum: 4,
requested_versions: 1,
bucket: "bucket".to_string(),
strict: true,
write_quorum_slack: 0,
candidates: Vec::new(),
});
black_box(resolved);
@@ -504,12 +525,13 @@ fn bench_metacache_resolve(c: &mut Criterion) {
group.bench_function("8_disks_disagree_13_versions/1000_objects", |b| {
b.iter(|| {
for entries in &sets_8_disagree_13v {
let resolved = entries.resolve(MetadataResolutionParams {
let resolved = entries.resolve_with_write_quorum(MetadataResolutionParams {
dir_quorum: 4,
obj_quorum: 4,
requested_versions: 1,
bucket: "bucket".to_string(),
strict: true,
write_quorum_slack: 0,
candidates: Vec::new(),
});
black_box(resolved);
@@ -574,6 +596,191 @@ fn bench_metacache_resolve(c: &mut Criterion) {
group.finish();
}
fn listing_history(versions: usize, cached: bool) -> MetaCacheEntry {
let mut metadata = FileMeta::new();
for index in 0..versions {
let (data, parity) = if index + 1 == versions { (12, 4) } else { (8, 8) };
let mut info = FileInfo::new("object", data, parity);
info.volume = "bucket".to_owned();
info.erasure.index = 1;
info.data_dir = Some(Uuid::from_u128(0x1000 + u128::try_from(index).expect("data directory index")));
info.versioned = true;
info.version_id = Some(Uuid::from_u128(u128::try_from(index + 1).expect("version index")));
info.mod_time = Some(
OffsetDateTime::from_unix_timestamp(1_700_000_000 + i64::try_from(index).expect("timestamp")).expect("fixture time"),
);
info.size = 1;
info.parts = vec![ObjectPartInfo {
number: 1,
size: 1,
actual_size: 1,
..Default::default()
}];
info.metadata.insert("etag".to_owned(), format!("etag-{index}"));
info.metadata.insert("x-amz-meta-description".to_owned(), "m".repeat(1024));
metadata.add_version(info).expect("valid version history");
}
MetaCacheEntry {
name: "object".to_owned(),
metadata: metadata.marshal_msg().expect("serialize history"),
cached: cached.then_some(metadata),
reusable: false,
}
}
#[inline]
fn owned_agreed_latest_reference(
mut entry: MetaCacheEntry,
replicas: usize,
params: &rustfs_filemeta::MetadataResolutionParams,
enforce_write_quorum: bool,
) -> rustfs_filemeta::MetaCacheEntryResolution {
use rustfs_filemeta::MetaCacheEntryResolution::{NeedsSupplement, Rejected, Resolved};
if !enforce_write_quorum {
return Resolved(entry);
}
if entry.is_dir() {
return if replicas >= params.dir_quorum {
Resolved(entry)
} else {
Rejected
};
}
if replicas < params.obj_quorum {
return Rejected;
}
let metadata = match entry.xl_meta_ref() {
Ok(metadata) => metadata,
Err(_) if params.obj_quorum <= 1 => return Resolved(entry),
Err(_) => return Rejected,
};
if params.requested_versions == 1
&& let Some(first) = metadata.versions.first()
&& !first.header.free_version()
&& replicas >= first.listing_quorum(params.obj_quorum, params.write_quorum_slack)
{
return Resolved(entry);
}
// These fixtures cover the fast prefix, never older-version reconciliation.
NeedsSupplement(entry)
}
fn bench_listing_borrowed_metadata(c: &mut Criterion) {
use rustfs_filemeta::{FileMetaVersion, MetaCacheEntries, MetadataResolutionParams};
let mut group = c.benchmark_group("listing_borrowed_metadata");
group
.sample_size(20)
.warm_up_time(Duration::from_millis(150))
.measurement_time(Duration::from_millis(700));
for versions in [2, 16, 256] {
for cached in [false, true] {
let entry = listing_history(versions, cached);
let params = MetadataResolutionParams {
dir_quorum: 8,
obj_quorum: 8,
requested_versions: 1,
write_quorum_slack: 8,
bucket: "bucket".to_owned(),
..Default::default()
};
let slots = vec![Some(&entry); 8];
let owned = MetaCacheEntries(vec![Some(entry.clone()); 8])
.resolve_with_write_quorum(params.clone())
.expect("previous committed version");
let borrowed =
MetaCacheEntries::resolve_borrowed_with_write_quorum(&slots, params.clone()).expect("borrowed previous version");
assert_eq!(owned.metadata, borrowed.metadata);
let id = format!("{versions}_versions/{}", if cached { "cached" } else { "cold" });
group.throughput(Throughput::Bytes(u64::try_from(entry.metadata.len()).expect("metadata size")));
// ABBA order permits an explicit baseline-drift check for each fixture.
for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] {
group.bench_function(BenchmarkId::new("replicas", format!("{id}/{phase}")), |b| {
b.iter(|| {
if phase.starts_with("owned") {
black_box(
MetaCacheEntries(vec![Some(black_box(&entry).clone()); 8])
.resolve_with_write_quorum(params.clone()),
)
} else {
let slots = vec![Some(black_box(&entry)); 8];
black_box(MetaCacheEntries::resolve_borrowed_with_write_quorum(&slots, params.clone()))
}
});
});
}
let mut agreed_metadata = entry
.cached
.clone()
.unwrap_or_else(|| FileMeta::load(&entry.metadata).expect("agreed fixture"));
let mut latest = FileMetaVersion::try_from(agreed_metadata.versions[0].meta.as_slice()).expect("latest body");
let object = latest.object.as_mut().expect("latest object");
object.erasure_m = 8;
object.erasure_n = 8;
agreed_metadata.versions[0] = latest.try_into().expect("balanced latest header");
let agreed = MetaCacheEntry {
name: entry.name.clone(),
metadata: agreed_metadata.marshal_msg().expect("agreed history"),
cached: cached.then_some(agreed_metadata),
reusable: false,
};
for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] {
group.bench_function(BenchmarkId::new("agreed", format!("{id}/{phase}")), |b| {
b.iter_batched(
|| agreed.clone(),
|candidate| {
let resolved = if phase.starts_with("owned") {
owned_agreed_latest_reference(candidate, 8, &params, true)
} else {
rustfs_filemeta::resolve_agreed_entry(candidate, 8, &params, true)
};
let rustfs_filemeta::MetaCacheEntryResolution::Resolved(candidate) = resolved else {
panic!("balanced latest must use agreed fast prefix");
};
black_box(candidate)
},
BatchSize::SmallInput,
);
});
}
let replicas: Vec<_> = (1..=8)
.map(|disk| {
let mut metadata = entry
.cached
.clone()
.unwrap_or_else(|| FileMeta::load(&entry.metadata).expect("decode fixture"));
for version in &mut metadata.versions {
let mut payload = FileMetaVersion::try_from(version.meta.as_slice()).expect("version body");
payload.object.as_mut().expect("object body").erasure_index = disk;
version.meta = payload.marshal_msg().expect("per-disk body");
}
MetaCacheEntry {
name: entry.name.clone(),
metadata: metadata.marshal_msg().expect("per-disk metadata"),
cached: cached.then_some(metadata),
reusable: false,
}
})
.collect();
assert!(replicas[0].headers_agree(&replicas, false));
for phase in ["owned_a1", "borrowed_b1", "borrowed_b2", "owned_a2"] {
group.bench_function(BenchmarkId::new("headers", format!("{id}/{phase}")), |b| {
b.iter(|| {
let agreed = if phase.starts_with("owned") {
replicas
.iter()
.all(|candidate| replicas[0].matches(Some(black_box(candidate)), false).1)
} else {
replicas[0].headers_agree(black_box(&replicas), false)
};
black_box(agreed)
});
});
}
}
}
group.finish();
}
criterion_group!(
benches,
bench_create_real_xlmeta,
@@ -590,7 +797,8 @@ criterion_group!(
bench_list_page_to_fileinfo,
bench_delete_inline_versions,
bench_metacache_stream_page,
bench_metacache_resolve
bench_metacache_resolve,
bench_listing_borrowed_metadata
);
criterion_main!(benches);
-11
View File
@@ -764,17 +764,6 @@ pub(crate) fn is_valid_distribution(distribution: &[usize], n: usize) -> bool {
}
impl FileInfo {
pub fn distribution_start(bucket: &str, object: &str, cardinality: usize) -> usize {
if cardinality == 0 {
return 0;
}
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
hasher.update(bucket.as_bytes());
hasher.update(b"/");
hasher.update(object.as_bytes());
(hasher.finalize() as u32 as usize) % cardinality
}
pub fn new(object: &str, data_blocks: usize, parity_blocks: usize) -> Self {
let indices = {
let cardinality = data_blocks + parity_blocks;
+59 -15
View File
@@ -115,7 +115,6 @@ pub(crate) fn data_key_for_version(version_id: Option<Uuid>) -> String {
}
}
#[cfg(test)]
fn legacy_data_key_for_version(version_id: Option<Uuid>) -> Option<String> {
if version_id.is_none() || version_id == Some(Uuid::nil()) {
Some(Uuid::nil().to_string())
@@ -155,6 +154,7 @@ pub fn is_skip_meta_key(key: &str) -> bool {
}
mod codec;
pub(crate) use codec::{BorrowedFileMeta, reconcile_headers};
mod inline_data;
pub(crate) mod msgp_decode;
mod validation;
@@ -348,21 +348,19 @@ impl FileMeta {
self.versions.sort_by(cmp_shallow_versions_for_order);
}
fn find_inline_data_slice(inline_data: &[u8], version_id: Option<Uuid>) -> Result<Option<&[u8]>> {
if let Some(vid) = version_id.filter(|v| !v.is_nil()) {
let mut buf = Uuid::encode_buffer();
let key = vid.hyphenated().encode_lower(&mut buf);
InlineData::find_slice_in_raw(inline_data, key)
} else {
if let Some(data) = InlineData::find_slice_in_raw(inline_data, NULL_VERSION_ID)? {
return Ok(Some(data));
}
InlineData::find_slice_in_raw(inline_data, "00000000-0000-0000-0000-000000000000")
}
}
fn find_inline_data_in_slice(inline_data: &[u8], version_id: Option<Uuid>) -> Result<Option<Vec<u8>>> {
Ok(Self::find_inline_data_slice(inline_data, version_id)?.map(<[u8]>::to_vec))
let key = data_key_for_version(version_id);
if let Some(data) = InlineData::find_in_slice(inline_data, key.as_str())? {
return Ok(Some(data));
}
if let Some(legacy_key) = legacy_data_key_for_version(version_id)
&& legacy_key != key
{
return InlineData::find_in_slice(inline_data, legacy_key.as_str());
}
Ok(None)
}
fn find_inline_data_for_version(&self, version_id: Option<Uuid>) -> Result<Option<Vec<u8>>> {
@@ -2429,6 +2427,52 @@ mod test {
}
}
#[test]
fn truncated_inline_xlmeta_is_file_corrupt_only_for_supported_formats() {
let mut meta = FileMeta::new();
meta.data
.replace("fixture", vec![0x5a; 4095])
.expect("seed real inline framing");
let inline_len = meta.data.as_slice().len();
let original = meta.marshal_msg().expect("serialize valid inline metadata");
let inline_start = original.len() - inline_len;
let parsed = FileMeta::load(&original).expect("complete inline metadata must decode");
assert_eq!(parsed.data, meta.data);
for cut in inline_start + 1..original.len() {
assert_eq!(
FileMeta::load(&original[..cut]).expect_err("incomplete inline framing must fail"),
Error::FileCorrupt,
"inline truncation at byte {cut} must remain repairable"
);
}
for inline_version in [0, 2] {
let mut unsupported = original[..original.len() / 2].to_vec();
unsupported[inline_start] = inline_version;
assert_ne!(
FileMeta::load(&unsupported).expect_err("malformed unsupported inline format must fail"),
Error::FileCorrupt,
"unknown inline version must not authorize repair"
);
}
for (header_ver, meta_ver) in [
(XL_HEADER_VERSION + 1, XL_META_VERSION),
(XL_HEADER_VERSION, XL_META_VERSION + 1),
] {
let mut headers = Vec::new();
rmp::encode::write_uint(&mut headers, u64::from(header_ver)).expect("write future header");
rmp::encode::write_uint(&mut headers, u64::from(meta_ver)).expect("write future metadata version");
rmp::encode::write_uint(&mut headers, 0).expect("write empty version count");
let mut unsupported = build_xl_buffer(&headers);
unsupported.extend_from_slice(&meta.data.as_slice()[..inline_len / 2]);
assert_ne!(
FileMeta::load(&unsupported).expect_err("unsupported metadata with truncated inline data must fail"),
Error::FileCorrupt,
"future metadata schema must not authorize repair"
);
}
}
#[test]
fn truncated_xlmeta_index_and_format_reads_are_file_corrupt() {
let buf = FileMeta::default()
+7 -1
View File
@@ -188,7 +188,13 @@ impl FileMeta {
if !buf.is_empty() {
InlineData::validate_slice(buf).map_err(|e| {
error!("data validation failed: {}", e);
e
// Validation only examines bytes already read. Classify damage
// as repairable corruption only for formats we understand.
if InlineData::version_ok_slice(buf) && Self::decode_xl_headers(meta).is_ok() {
Error::FileCorrupt
} else {
e
}
})?;
}
+258 -65
View File
@@ -622,6 +622,18 @@ pub struct FileMetaShallowVersion {
pub meta: Vec<u8>, // FileMetaVersion.marshal_msg
}
fn write_quorum_from_erasure(data_blocks: usize, parity_blocks: usize) -> Option<usize> {
if data_blocks == 0 {
return None;
}
Some(if data_blocks == parity_blocks {
data_blocks.saturating_add(1)
} else {
data_blocks
})
}
impl FileMetaShallowVersion {
/// Parse version meta with legacy format compatibility.
/// Use this instead of `FileMetaVersion::default()` + `unmarshal_msg()` to handle old-version xl.meta.
@@ -629,11 +641,60 @@ impl FileMetaShallowVersion {
FileMetaVersion::try_from(self.meta.as_slice())
}
pub fn write_quorum(&self, fallback_quorum: usize) -> usize {
write_quorum_from_header_and_meta(&self.header, self.meta.as_slice(), fallback_quorum)
}
/// Quorum for listing this version without accepting an unreadable payload.
pub fn listing_quorum(&self, fallback_quorum: usize, unreachable_disks: usize) -> usize {
listing_quorum_from_header_and_meta(&self.header, &self.meta, fallback_quorum, unreachable_disks)
}
pub fn into_fileinfo(&self, volume: &str, path: &str, all_parts: bool) -> Result<FileInfo> {
FileMetaVersion::decode_fileinfo_from_slice(self.meta.as_slice(), volume, path, all_parts, true)
}
}
#[inline]
pub(crate) fn write_quorum_from_header_and_meta(header: &FileMetaVersionHeader, meta: &[u8], fallback_quorum: usize) -> usize {
let header_quorum = header.write_quorum(fallback_quorum);
if header.has_ec() || !matches!(header.version_type, VersionType::Object | VersionType::Legacy) {
return header_quorum;
}
FileMetaVersion::try_from(meta)
.ok()
.and_then(|version| version.erasure_write_quorum())
.unwrap_or(header_quorum)
}
pub(crate) fn listing_quorum_from_header_and_meta(
header: &FileMetaVersionHeader,
meta: &[u8],
fallback_quorum: usize,
unreachable_disks: usize,
) -> usize {
let geometry = if header.version_type == VersionType::Object && header.has_ec() {
Some((usize::from(header.ec_m), usize::from(header.ec_n)))
} else if matches!(header.version_type, VersionType::Object | VersionType::Legacy) {
FileMetaVersion::try_from(meta)
.ok()
.and_then(|version| version.erasure_geometry())
} else {
None
};
let Some((data_blocks, parity_blocks)) = geometry else {
return fallback_quorum;
};
let Some(write_quorum) = write_quorum_from_erasure(data_blocks, parity_blocks) else {
return fallback_quorum;
};
write_quorum
.saturating_sub(unreachable_disks)
.max(data_blocks)
.max(fallback_quorum)
}
impl TryFrom<FileMetaVersion> for FileMetaShallowVersion {
type Error = Error;
@@ -676,6 +737,26 @@ pub(super) struct V2ObjectSliceSummary {
}
impl FileMetaVersion {
fn erasure_geometry(&self) -> Option<(usize, usize)> {
Some(match self.version_type {
VersionType::Object | VersionType::Legacy => {
if let Some(object) = &self.object {
(object.erasure_m, object.erasure_n)
} else if let Some(object) = &self.legacy_object {
(object.erasure.data_blocks, object.erasure.parity_blocks)
} else {
return None;
}
}
_ => return None,
})
}
fn erasure_write_quorum(&self) -> Option<usize> {
let (data_blocks, parity_blocks) = self.erasure_geometry()?;
write_quorum_from_erasure(data_blocks, parity_blocks)
}
/// Fast borrowed scanner for `V2Obj` slices. Returning `Ok` guarantees
/// `FileMetaVersion::try_from` would succeed with the identical summary fields;
/// any non-`V2Obj` or invalid input returns `Err` so callers fall back to `try_from`.
@@ -1694,6 +1775,14 @@ impl FileMetaVersionHeader {
self.ec_m > 0 && self.ec_n > 0
}
pub fn write_quorum(&self, fallback_quorum: usize) -> usize {
if self.version_type != VersionType::Object || !self.has_ec() {
return fallback_quorum;
}
write_quorum_from_erasure(usize::from(self.ec_m), usize::from(self.ec_n)).unwrap_or(fallback_quorum)
}
pub fn matches_not_strict(&self, o: &FileMetaVersionHeader) -> bool {
let mut ok = self.version_id == o.version_id && self.version_type == o.version_type && self.matches_ec(o);
// Disk-loaded headers keep the null version as Some(nil), not None: all null
@@ -4078,49 +4167,15 @@ pub fn merge_file_meta_versions(
.iter()
.map(|v| v.iter().map(|ver| (&ver.header, ver.meta.as_slice())).collect())
.collect();
merge_file_meta_borrowed_streams(quorum, strict, requested_versions, streams)
}
/// Picks the most common header among the tops that share `latest`'s version
/// id; a tie keeps the header that sorts first (the newest).
fn most_common_header_for_version_id<'a>(
tops: &[(&'a FileMetaVersionHeader, &'a [u8])],
mut latest: (&'a FileMetaVersionHeader, &'a [u8]),
strict: bool,
) -> ((&'a FileMetaVersionHeader, &'a [u8]), usize) {
let version_id = latest.0.version_id;
let comparable = |header: &FileMetaVersionHeader| {
let mut header = header.clone();
if !strict {
header.signature = [0; 4];
}
header
};
let mut counts: HashMap<FileMetaVersionHeader, usize> = HashMap::new();
for &top in tops {
if top.0.version_id == version_id {
*counts.entry(comparable(top.0)).or_insert(0) += 1;
}
}
let mut latest_count = 0;
for (header, count) in &counts {
if *count < latest_count || (*count == latest_count && latest.0.sorts_before(header)) {
continue;
}
for &top in tops {
if comparable(top.0) == *header {
latest = top;
}
}
latest_count = *count;
}
(latest, latest_count)
merge_file_meta_borrowed_streams(quorum, strict, requested_versions, false, 0, streams)
}
pub(crate) fn merge_file_meta_borrowed_streams(
mut quorum: usize,
mut strict: bool,
requested_versions: usize,
enforce_write_quorum: bool,
write_quorum_slack: usize,
mut streams: Vec<Vec<(&FileMetaVersionHeader, &[u8])>>,
) -> Vec<FileMetaShallowVersion> {
let to_shallow = |(header, meta): (&FileMetaVersionHeader, &[u8])| FileMetaShallowVersion {
@@ -4137,13 +4192,32 @@ pub(crate) fn merge_file_meta_borrowed_streams(
}
if streams.len() == 1 {
return streams[0].iter().copied().map(to_shallow).collect();
if !enforce_write_quorum {
return streams[0].iter().copied().map(to_shallow).collect();
}
let required_quorum = streams[0]
.first()
.map(|&(hdr, meta)| listing_quorum_from_header_and_meta(hdr, meta, quorum, write_quorum_slack))
.unwrap_or(quorum);
if streams.len() >= required_quorum {
return streams[0].iter().copied().map(to_shallow).collect();
}
return Vec::new();
}
if quorum == 1 {
strict = true;
}
let required_quorum = |(hdr, meta): (&FileMetaVersionHeader, &[u8])| {
if enforce_write_quorum {
listing_quorum_from_header_and_meta(hdr, meta, quorum, write_quorum_slack)
} else {
quorum
}
};
let mut tops: Vec<(&FileMetaVersionHeader, &[u8])> = Vec::with_capacity(streams.len());
let mut n_versions = 0;
@@ -4175,7 +4249,7 @@ pub(crate) fn merge_file_meta_borrowed_streams(
let latest: (&FileMetaVersionHeader, &[u8]);
if consistent {
latest = tops[0];
if tops.len() >= quorum {
if tops.len() >= required_quorum(latest) {
merged.push(to_shallow(latest));
if !latest.0.free_version() {
n_versions += 1;
@@ -4184,7 +4258,6 @@ pub(crate) fn merge_file_meta_borrowed_streams(
} else {
let mut latest_ref = tops[0];
let mut latest_count = 1;
let mut settled_by_version_id = false;
for &ver in &tops[1..] {
if ver.0 == latest_ref.0 {
latest_count += 1;
@@ -4208,26 +4281,39 @@ pub(crate) fn merge_file_meta_borrowed_streams(
}
if ver.0.version_id == latest_ref.0.version_id {
(latest_ref, latest_count) = most_common_header_for_version_id(&tops, latest_ref, strict);
settled_by_version_id = true;
let mut x: HashMap<FileMetaVersionHeader, usize> = HashMap::new();
for &a in &tops {
if a.0.version_id != ver.0.version_id {
continue;
}
let mut hdr = a.0.clone();
if !strict {
hdr.signature = [0; 4];
}
*x.entry(hdr).or_insert(0) += 1;
}
latest_count = 0;
for (k, v) in x.iter() {
if *v < latest_count || (*v == latest_count && latest_ref.0.sorts_before(k)) {
continue;
}
for &a in &tops {
let mut hdr = a.0.clone();
if !strict {
hdr.signature = [0; 4];
}
if hdr == *k {
latest_ref = a;
}
}
latest_count = *v;
}
break;
}
}
// Readers return their heads in any order. When every older header
// sharing the newest header's version id (an interrupted null-version
// overwrite) came first, the scan above never compared them, and the
// stream trim below would then drop that older committed generation.
if !settled_by_version_id
&& tops.iter().any(|ver| {
ver.0 != latest_ref.0
&& ver.0.version_id == latest_ref.0.version_id
&& (strict || !ver.0.matches_not_strict(latest_ref.0))
})
{
(latest_ref, latest_count) = most_common_header_for_version_id(&tops, latest_ref, strict);
}
latest = latest_ref;
if latest_count >= quorum {
if latest_count >= required_quorum(latest) {
if !latest.0.free_version() {
n_versions += 1;
}
@@ -4419,14 +4505,14 @@ fn read_more_sync<R: std::io::Read>(
}
let extra = read_size - has;
buf.try_reserve(extra.saturating_add(1)).map_err(Error::other)?;
std::io::Read::read_to_end(&mut std::io::Read::take(reader, extra as u64), buf)?;
if buf.len() < read_size {
return Err(Error::from(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"failed to fill whole buffer",
)));
if buf.capacity() >= read_size {
// Extend the buffer if we have enough space.
buf.resize(read_size, 0);
} else {
buf.extend(vec![0u8; extra]);
}
reader.read_exact(&mut buf[has..])?;
Ok(())
}
@@ -4448,8 +4534,8 @@ pub fn read_xl_meta_no_data_sync<R: std::io::Read>(reader: &mut R, size: usize)
has_full = false;
}
let mut buf = Vec::new();
read_more_sync(reader, &mut buf, size, initial, false)?;
let mut buf = vec![0u8; initial];
reader.read_exact(&mut buf)?;
let (tmp_buf, major, minor) = FileMeta::check_xl2_v1(&buf)?;
@@ -5016,6 +5102,39 @@ mod tests {
wr
}
fn encode_legacy_v2_object_body(data_blocks: usize, parity_blocks: usize) -> Vec<u8> {
let drive_count = data_blocks + parity_blocks;
let payload = LegacyObjectVersionFixture {
version_type: LegacyObjectVersionTypeFixture::Object,
object: Some(LegacyObjectFixture {
version_id: Some(sample_version_id().as_bytes().to_vec()),
data_dir: Some(Uuid::from_u128(42).as_bytes().to_vec()),
erasure_algorithm: "ReedSolomon".to_string(),
erasure_m: data_blocks,
erasure_n: parity_blocks,
erasure_block_size: 1_048_576,
erasure_index: 1,
erasure_dist: (1..=drive_count)
.map(|idx| u8::try_from(idx).expect("test drive index should fit u8"))
.collect(),
bitrot_checksum_algo: "HighwayHash".to_string(),
part_numbers: vec![1],
part_etags: vec!["etag-1".to_string()],
part_sizes: vec![11],
part_actual_sizes: vec![11],
part_indices: vec![Vec::new()],
size: 11,
mod_time: Some(sample_mod_time()),
meta_sys: HashMap::new(),
meta_user: HashMap::from([("content-type".to_string(), "text/plain".to_string())]),
}),
delete_marker: None,
write_version: 3,
};
rmp_serde::to_vec_named(&payload).expect("legacy object payload should marshal")
}
#[test]
fn version_header_unmarshal_v1_uses_legacy_layout_defaults() {
let expected = sample_header();
@@ -5050,6 +5169,80 @@ mod tests {
assert_eq!(decoded.ec_m, 0);
}
#[test]
fn shallow_version_write_quorum_uses_legacy_object_payload_when_header_lacks_ec() {
let expected = sample_header();
let encoded = encode_v2_header(&expected);
let mut header = FileMetaVersionHeader::default();
header.unmarshal_v(2, &encoded).expect("legacy v2 header should decode");
let version = FileMetaShallowVersion {
header,
meta: encode_legacy_v2_object_body(7, 1),
};
assert_eq!(version.header.write_quorum(5), 5);
assert_eq!(version.write_quorum(5), 7);
}
#[test]
fn listing_quorum_uses_each_versions_data_floor() {
for (data, parity, offline, fallback, expected) in
[(8, 8, 0, 9, 9), (8, 8, 8, 8, 8), (12, 4, 8, 8, 12), (12, 4, 0, 9, 12)]
{
let version = FileMetaShallowVersion {
header: FileMetaVersionHeader {
version_type: VersionType::Object,
ec_m: data,
ec_n: parity,
..Default::default()
},
// Modern headers must not decode the payload to decide quorum.
meta: vec![0xff],
};
assert_eq!(version.listing_quorum(fallback, offline), expected);
}
let marker = FileMetaShallowVersion {
header: FileMetaVersionHeader {
version_type: VersionType::Delete,
..Default::default()
},
meta: Vec::new(),
};
assert_eq!(marker.listing_quorum(9, 0), 9, "healthy marker needs majority metadata");
assert_eq!(marker.listing_quorum(8, 8), 8, "degraded marker retains metadata floor");
}
#[test]
fn listing_quorum_preserves_legacy_and_zero_parity_data_floor() {
for (data, parity, fallback, expected) in [(12, 4, 8, 12), (8, 8, 8, 8), (4, 0, 2, 4)] {
let version = FileMetaShallowVersion {
header: FileMetaVersionHeader {
version_type: VersionType::Object,
..Default::default()
},
meta: encode_legacy_v2_object_body(data, parity),
};
assert_eq!(version.listing_quorum(fallback, 8), expected);
}
}
#[test]
fn shallow_version_write_quorum_uses_zero_parity_object_payload() {
let expected = sample_header();
let encoded = encode_v2_header(&expected);
let mut header = FileMetaVersionHeader::default();
header.unmarshal_v(2, &encoded).expect("legacy v2 header should decode");
let version = FileMetaShallowVersion {
header,
meta: encode_legacy_v2_object_body(4, 0),
};
assert_eq!(version.header.write_quorum(2), 2);
assert_eq!(version.write_quorum(2), 4);
}
#[test]
fn version_header_unmarshal_v3_round_trips_current_layout() {
let expected = sample_header();
+6 -10
View File
@@ -168,11 +168,11 @@ impl InlineData {
self.0.as_slice()
}
pub fn version_ok(&self) -> bool {
if self.0.is_empty() {
return true;
}
Self::version_ok_slice(&self.0)
}
self.0[0] > 0 && self.0[0] <= INLINE_DATA_VER
pub(crate) fn version_ok_slice(raw: &[u8]) -> bool {
raw.first().is_none_or(|version| *version > 0 && *version <= INLINE_DATA_VER)
}
pub fn after_version(&self) -> &[u8] {
@@ -230,7 +230,7 @@ impl InlineData {
Self::find_in_slice(&self.0, key)
}
pub(crate) fn find_slice_in_raw<'a>(raw: &'a [u8], key: &str) -> Result<Option<&'a [u8]>> {
pub(crate) fn find_in_slice(raw: &[u8], key: &str) -> Result<Option<Vec<u8>>> {
if raw.is_empty() || raw[0] == 0 || raw[0] > INLINE_DATA_VER {
return Ok(None);
}
@@ -243,17 +243,13 @@ impl InlineData {
fields_len -= 1;
let (field, value) = Self::read_entry_slice(buf, &mut cur, false)?;
if field == key {
return Ok(Some(value));
return Ok(Some(value.to_vec()));
}
}
Ok(None)
}
pub(crate) fn find_in_slice(raw: &[u8], key: &str) -> Result<Option<Vec<u8>>> {
Ok(Self::find_slice_in_raw(raw, key)?.map(<[u8]>::to_vec))
}
pub fn validate(&self) -> Result<()> {
Self::validate_slice(&self.0)
}
File diff suppressed because it is too large Load Diff
+7 -31
View File
@@ -683,17 +683,14 @@ fn create_auto_heal_disk_request(set_disk_id: String, priority: Option<HealChann
}
}
/// Build one automatic replacement request for the verified targets of a set.
pub fn create_auto_replacement_disks_request(
fn create_auto_replacement_disk_request(
pool_index: usize,
set_index: usize,
mut replacement_endpoints: Vec<String>,
replacement_endpoint: String,
priority: Option<HealChannelPriority>,
) -> HealChannelRequest {
replacement_endpoints.sort_unstable();
replacement_endpoints.dedup();
let mut request = create_auto_heal_disk_request(format!("pool_{pool_index}_set_{set_index}"), priority);
request.heal_endpoints = replacement_endpoints;
request.heal_endpoints = vec![replacement_endpoint];
request.pool_index = Some(pool_index);
request.set_index = Some(set_index);
request
@@ -714,10 +711,10 @@ pub async fn send_heal_replacement_disk(
replacement_endpoint: String,
priority: Option<HealChannelPriority>,
) -> Result<(), String> {
send_heal_request(create_auto_replacement_disks_request(
send_heal_request(create_auto_replacement_disk_request(
pool_index,
set_index,
vec![replacement_endpoint],
replacement_endpoint,
priority,
))
.await
@@ -729,12 +726,8 @@ mod auto_heal_disk_request_tests {
#[test]
fn replacement_disk_request_carries_its_exact_endpoint() {
let request = create_auto_replacement_disks_request(
2,
3,
vec!["http://node2:9000/drive3".to_string()],
Some(HealChannelPriority::Normal),
);
let request =
create_auto_replacement_disk_request(2, 3, "http://node2:9000/drive3".to_string(), Some(HealChannelPriority::Normal));
assert_eq!(request.disk.as_deref(), Some("pool_2_set_3"));
assert_eq!(request.heal_endpoints, ["http://node2:9000/drive3"]);
@@ -743,23 +736,6 @@ mod auto_heal_disk_request_tests {
assert_eq!(request.source, HealRequestSource::AutoHeal);
}
#[test]
fn replacement_disk_request_keeps_every_distinct_target() {
let request = create_auto_replacement_disks_request(
2,
3,
vec!["drive-b".to_string(), "drive-a".to_string(), "drive-b".to_string()],
Some(HealChannelPriority::Normal),
);
assert_eq!(request.disk.as_deref(), Some("pool_2_set_3"));
assert_eq!(request.heal_endpoints, ["drive-a", "drive-b"]);
assert_eq!(request.pool_index, Some(2));
assert_eq!(request.set_index, Some(3));
assert_eq!(request.priority, HealChannelPriority::Normal);
assert_eq!(request.source, HealRequestSource::AutoHeal);
}
#[test]
fn legacy_auto_heal_disk_request_has_no_replacement_endpoint() {
let request = create_auto_heal_disk_request("pool_2_set_3".to_string(), None);
+27 -17
View File
@@ -196,6 +196,20 @@ impl HealChannelProcessor {
.map_err(Error::other)
}
/// Query a local automatic task; other tokens require coordinator routing.
pub async fn execute_local_automatic_query(
&self,
heal_path: &str,
client_token: &str,
since_seq: Option<u64>,
) -> Result<Option<HealChannelResponse>> {
self.heal_manager
.get_local_automatic_task_report(heal_path, client_token, since_seq)
.await?
.map(|report| Self::query_report_response(client_token.to_owned(), Ok(report)))
.transpose()
}
/// Execute cancellation directly against the manager.
pub async fn execute_cancel_request(&self, heal_path: String, client_token: String) -> Result<HealChannelResponse> {
let (response_tx, response_rx) = oneshot::channel();
@@ -463,6 +477,13 @@ impl HealChannelProcessor {
.await
};
let response = Self::query_report_response(client_token, report)?;
let _ = response_tx.send(Ok(response.clone()));
self.publish_response(response);
Ok(())
}
fn query_report_response(client_token: String, report: Result<HealTaskReport>) -> Result<HealChannelResponse> {
let outcome = report.as_ref().ok().and_then(|report| report.outcome.clone());
let settings = report
.as_ref()
@@ -581,27 +602,21 @@ impl HealChannelProcessor {
0,
),
Err(crate::Error::InvalidClientToken) => {
let response = HealChannelResponse {
return Ok(HealChannelResponse {
request_id: client_token,
success: false,
data: None,
error: Some("invalid heal client token".to_string()),
};
let _ = response_tx.send(Ok(response.clone()));
self.publish_response(response);
return Ok(());
});
}
Err(err) => {
let error_text = err.to_string();
let response = HealChannelResponse {
return Ok(HealChannelResponse {
request_id: client_token,
success: false,
data: None,
error: Some(error_text),
};
let _ = response_tx.send(Ok(response.clone()));
self.publish_response(response);
return Ok(());
});
}
};
@@ -618,17 +633,12 @@ impl HealChannelProcessor {
},
)?;
let response = HealChannelResponse {
Ok(HealChannelResponse {
request_id: client_token,
success: true,
data: Some(data),
error: detail,
};
let _ = response_tx.send(Ok(response.clone()));
self.publish_response(response);
Ok(())
})
}
/// Process cancel request
+80 -113
View File
@@ -234,6 +234,8 @@ fn record_displaced_terminal(
request: &HealRequest,
) -> Arc<CompletedHealStatus> {
let terminal = Arc::new(CompletedHealStatus {
task_id: request.id.clone(),
source: request.source,
outcome: None,
progress: None,
retained_bytes: std::sync::OnceLock::new(),
@@ -669,7 +671,10 @@ fn retry_budget_for_result(task: &HealTask, result: &Result<()>, retryable_batch
if task.retry_attempts >= MAX_RECOVERABLE_HEAL_RETRIES {
return None;
}
if task.has_batch_failure() && !retryable_batch_failure {
// Admin traversals already exhaust their per-object retries. Restarting
// the whole traversal under the same token discards those receipts and
// reports previously repaired objects as unchanged on the next scan.
if task.has_batch_failure() && (task.source == HealRequestSource::Admin || !retryable_batch_failure) {
return None;
}
@@ -825,21 +830,6 @@ fn admin_heal_options_compatible(left: &HealOptions, right: &HealOptions) -> boo
&& left.no_lock == right.no_lock
}
fn replacement_request_compatible(
request: &HealRequest,
owner_type: &HealType,
owner_options: &HealOptions,
owner_source: HealRequestSource,
) -> bool {
request.source == HealRequestSource::AutoHeal
&& owner_source == HealRequestSource::AutoHeal
&& matches!(request.heal_type, HealType::ErasureSet { .. })
&& request.heal_type == *owner_type
&& request.options.pool_index == owner_options.pool_index
&& request.options.set_index == owner_options.set_index
&& admin_heal_options_compatible(&request.options, owner_options)
}
fn admin_overlap_rejection(
request: &HealRequest,
owner_type: &HealType,
@@ -1071,9 +1061,9 @@ pub struct HealManager {
/// cascade without re-locking.
enum TaskStateLookup {
Active(Arc<HealTask>),
Retrying(HealTaskStatus, HealOptions),
Retrying(HealTaskStatus, HealOptions, HealRequestSource),
Completed(Arc<CompletedHealStatus>),
Queued(HealOptions),
Queued(HealOptions, HealRequestSource),
/// A durable root heal record that cannot be admitted or quarantined;
/// it answers status queries with the reason until an operator repairs it.
Unrunnable(String),
@@ -1291,9 +1281,7 @@ impl HealManager {
request: HealRequest,
config: &HealConfig,
context: &'static str,
replacement_targets_frozen: bool,
) -> HealAdmissionDecision {
let replacement_targets_frozen = replacement_targets_frozen || request.retry_attempts != 0;
let queue_len = queue.len();
publish_heal_queue_length(queue);
let queue_capacity = config.queue_size;
@@ -1301,25 +1289,6 @@ impl HealManager {
let best_effort_reserve = best_effort_queue_reserve(queue_capacity);
let best_effort_source_reserves = best_effort_source_reserves(queue_capacity);
// Scanner and retry admission bypass submit_heal_request's duplicate
// checks. Keep their target ownership checks ahead of queue capacity.
if !request.force_start
&& request.source == HealRequestSource::AutoHeal
&& !request.heal_endpoints.is_empty()
&& let Some(owner_id) = queue
.queued_request_id_for_dedup_key(&PriorityHealQueue::make_dedup_key(&request))
.map(ToOwned::to_owned)
{
let mut admission = Self::duplicate_admission_for_request(&request, config);
if admission == HealAdmissionResult::Merged
&& !queue.merge_replacement_targets(&owner_id, &request, true, replacement_targets_frozen)
{
admission = HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning);
}
Self::record_admission_metric(request.source, admission, context);
return HealAdmissionDecision::new(admission);
}
if queue_len >= queue_capacity && !request.force_start {
if Self::can_displace_queued_work(&request)
&& queue.can_displace_lower_priority_where(request.priority, Self::queued_request_can_be_displaced)
@@ -1479,7 +1448,7 @@ impl HealManager {
let priority = request.priority;
let source = request.source;
match queue.push_with_replacement_scope(request, replacement_targets_frozen) {
match queue.push(request) {
QueuePushOutcome::Accepted => {
publish_heal_queue_length(queue);
Self::record_admission_metric(source, HealAdmissionResult::Accepted, context);
@@ -2296,7 +2265,7 @@ impl HealManager {
// HS-06: under the minio_error overlap policy an exact duplicate
// admin start reports the typed AlreadyRunning rejection instead
// of the silent merge (MinIO's ErrHealAlreadyRunning).
let mut admission = if (request.source == HealRequestSource::Admin
let admission = if (request.source == HealRequestSource::Admin
&& config.overlap_policy == HealOverlapPolicy::MinioError)
|| (duplicate_state != "queued" && mrf_notice_target.as_ref().is_some_and(|target| target.kind.is_durable()))
{
@@ -2306,44 +2275,6 @@ impl HealManager {
} else {
Self::duplicate_admission_for_request(&request, &config)
};
if matches!(admission, HealAdmissionResult::Merged)
&& request.source == HealRequestSource::AutoHeal
&& !request.heal_endpoints.is_empty()
{
let (owner_recovered, incoming_recovered) = {
// Match queue -> recovery-anchor ordering in the scanner.
let anchors = self
.replacement_recovery_anchors
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
(anchors.contains_key(&merged_task_id), anchors.contains_key(&request.id))
};
let incoming_frozen = incoming_recovered || request.retry_attempts != 0;
let covered = if incoming_frozen && request.id != merged_task_id {
false
} else if duplicate_state == "queued" {
queue.merge_replacement_targets(&merged_task_id, &request, !owner_recovered, incoming_frozen)
} else {
let owner = active_heals
.get(&merged_task_id)
.map(|task| (&task.heal_type, &task.options, task.source, &task.heal_endpoints))
.or_else(|| {
retrying_heals.get(&merged_task_id).map(|retry| {
let owner = &retry.request;
(&owner.heal_type, &owner.options, owner.source, &owner.heal_endpoints)
})
});
owner.is_some_and(|(heal_type, options, source, endpoints)| {
replacement_request_compatible(&request, heal_type, options, source)
&& request.heal_endpoints.iter().all(|endpoint| endpoints.contains(endpoint))
})
};
if !covered {
// A frozen generation cannot certify newly arriving disks.
// Leave them discoverable by the scanner for the next pass.
admission = HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning);
}
}
if matches!(admission, HealAdmissionResult::Merged)
&& let Some(target) = mrf_notice_target
{
@@ -2500,14 +2431,7 @@ impl HealManager {
let request_id = request.id.clone();
let request_heal_type = request.heal_type.clone();
let request_source = request.source;
let recovered_replacement = request.source == HealRequestSource::AutoHeal
&& !request.heal_endpoints.is_empty()
&& self
.replacement_recovery_anchors
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.contains_key(&request.id);
let admission_decision = Self::admit_request_to_queue(&mut queue, request, &config, "submit", recovered_replacement);
let admission_decision = Self::admit_request_to_queue(&mut queue, request, &config, "submit");
let admission = admission_decision.result;
if durable_handoff_required && !admission.is_admitted() {
self.root_recovery
@@ -2586,6 +2510,23 @@ impl HealManager {
/// completed entry. `heal_path` additionally constrains the map matches
/// the way the `*_for_path` variants always have.
async fn lookup_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> Result<TaskStateLookup> {
let state = self.lookup_memory_task_state(canonical_task_id, heal_path).await;
if !matches!(state, TaskStateLookup::NotFound) {
return Ok(state);
}
if let Some(completed) = self.root_recovery.completed(canonical_task_id).await?
&& heal_path.is_none_or(|path| heal_type_matches_path(&completed.heal_type, path))
{
return Ok(TaskStateLookup::Completed(Arc::new(completed)));
}
// A durable record that cannot run still answers with its reason.
if let Some(reason) = self.root_recovery.record_issue(canonical_task_id).await {
return Ok(TaskStateLookup::Unrunnable(reason));
}
Ok(TaskStateLookup::NotFound)
}
async fn lookup_memory_task_state(&self, canonical_task_id: &str, heal_path: Option<&str>) -> TaskStateLookup {
let matches_path = |heal_type: &HealType| heal_path.is_none_or(|path| heal_type_matches_path(heal_type, path));
{
@@ -2594,7 +2535,7 @@ impl HealManager {
.get(canonical_task_id)
.filter(|task| matches_path(&task.heal_type))
{
return Ok(TaskStateLookup::Active(Arc::clone(task)));
return TaskStateLookup::Active(Arc::clone(task));
}
}
@@ -2604,7 +2545,7 @@ impl HealManager {
.get(canonical_task_id)
.filter(|retrying| matches_path(&retrying.request.heal_type))
{
return Ok(TaskStateLookup::Retrying(retrying.status(), retrying.request.options.clone()));
return TaskStateLookup::Retrying(retrying.status(), retrying.request.options.clone(), retrying.request.source);
}
}
@@ -2617,7 +2558,7 @@ impl HealManager {
prune_completed_heal_statuses(&mut completed_heals);
if let Some(completed) = completed_heals.get(canonical_task_id).filter(|c| matches_path(&c.heal_type)) {
if completed_status_is_retrying(&completed.status) {
return Ok(TaskStateLookup::Completed(Arc::clone(completed)));
return TaskStateLookup::Completed(Arc::clone(completed));
}
terminal_completed = Some(Arc::clone(completed));
}
@@ -2626,7 +2567,7 @@ impl HealManager {
{
let queue = self.heal_queue.lock().await;
if let Some(request) = queue.request_matching_id_and_path(canonical_task_id, heal_path) {
return Ok(TaskStateLookup::Queued(request.options.clone()));
return TaskStateLookup::Queued(request.options.clone(), request.source);
}
}
@@ -2639,22 +2580,10 @@ impl HealManager {
.cloned();
}
if terminal_completed.is_none()
&& let Some(completed) = self.root_recovery.completed(canonical_task_id).await?
&& matches_path(&completed.heal_type)
{
terminal_completed = Some(Arc::new(completed));
}
if let Some(completed) = terminal_completed {
return Ok(TaskStateLookup::Completed(completed));
return TaskStateLookup::Completed(completed);
}
// A durable root record that can neither run nor be quarantined still
// answers status queries with its reason.
if let Some(reason) = self.root_recovery.record_issue(canonical_task_id).await {
return Ok(TaskStateLookup::Unrunnable(reason));
}
Ok(TaskStateLookup::NotFound)
TaskStateLookup::NotFound
}
async fn publish_admin_terminal(
@@ -2678,6 +2607,8 @@ impl HealManager {
) -> Result<bool> {
let previous = self.completed_heals.lock().await.get(task_id).cloned();
let mut completed = CompletedHealStatus {
task_id: task_id.to_owned(),
source,
outcome: None,
progress: None,
retained_bytes: std::sync::OnceLock::new(),
@@ -2713,9 +2644,9 @@ impl HealManager {
let canonical_task_id = self.canonical_task_id(task_id).await;
match self.lookup_task_state(&canonical_task_id, None).await? {
TaskStateLookup::Active(task) => Ok(task.get_status().await),
TaskStateLookup::Retrying(status, _) => Ok(status),
TaskStateLookup::Retrying(status, _, _) => Ok(status),
TaskStateLookup::Completed(completed) => Ok(completed.status.clone()),
TaskStateLookup::Queued(_) => Ok(HealTaskStatus::Pending),
TaskStateLookup::Queued(_, _) => Ok(HealTaskStatus::Pending),
TaskStateLookup::Unrunnable(error) => Ok(HealTaskStatus::Failed { error }),
TaskStateLookup::NotFound => Err(Error::TaskNotFound {
task_id: task_id.to_string(),
@@ -2734,9 +2665,9 @@ impl HealManager {
let canonical_task_id = self.canonical_task_id(task_id).await;
match self.lookup_task_state(&canonical_task_id, None).await? {
TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await),
TaskStateLookup::Retrying(status, options) => Ok(empty_task_report_with_options(status, options)),
TaskStateLookup::Retrying(status, options, _) => Ok(empty_task_report_with_options(status, options)),
TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)),
TaskStateLookup::Queued(options) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
TaskStateLookup::Queued(options, _) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
TaskStateLookup::Unrunnable(error) => {
Ok(empty_task_report_with_options(HealTaskStatus::Failed { error }, HealOptions::default()))
}
@@ -2750,6 +2681,42 @@ impl HealManager {
self.get_task_report_for_path_since(heal_path, task_id, None).await
}
/// Resolve a node-owned automatic task without granting local control over
/// Admin tasks. The source and report come from the same state snapshot.
pub async fn get_local_automatic_task_report(
&self,
heal_path: &str,
task_id: &str,
since: Option<u64>,
) -> Result<Option<HealTaskReport>> {
let canonical_task_id = self.canonical_task_id(task_id).await;
if canonical_task_id != task_id {
return Ok(None);
}
let state = self.lookup_memory_task_state(&canonical_task_id, Some(heal_path)).await;
let source = match &state {
TaskStateLookup::Active(task) => task.source,
TaskStateLookup::Retrying(_, _, source) | TaskStateLookup::Queued(_, source) => *source,
TaskStateLookup::Completed(completed) if completed.task_id == task_id => completed.source,
TaskStateLookup::Completed(_) => return Ok(None),
TaskStateLookup::Unrunnable(_) | TaskStateLookup::NotFound => return Ok(None),
};
if !matches!(
source,
HealRequestSource::Scanner | HealRequestSource::Mrf | HealRequestSource::AutoHeal | HealRequestSource::ReadRepair
) {
return Ok(None);
}
let report = match state {
TaskStateLookup::Active(task) => active_task_report(&task, since).await,
TaskStateLookup::Retrying(status, options, _) => empty_task_report_with_options(status, options),
TaskStateLookup::Completed(completed) => completed_task_report(&completed, since),
TaskStateLookup::Queued(options, _) => empty_task_report_with_options(HealTaskStatus::Pending, options),
TaskStateLookup::Unrunnable(_) | TaskStateLookup::NotFound => return Ok(None),
};
Ok(Some(report))
}
/// Incremental variant of [`Self::get_task_report_for_path`] (HS-06).
pub async fn get_task_report_for_path_since(
&self,
@@ -2760,9 +2727,9 @@ impl HealManager {
let canonical_task_id = self.canonical_task_id(task_id).await;
match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? {
TaskStateLookup::Active(task) => Ok(active_task_report(&task, since).await),
TaskStateLookup::Retrying(status, options) => Ok(empty_task_report_with_options(status, options)),
TaskStateLookup::Retrying(status, options, _) => Ok(empty_task_report_with_options(status, options)),
TaskStateLookup::Completed(completed) => Ok(completed_task_report(&completed, since)),
TaskStateLookup::Queued(options) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
TaskStateLookup::Queued(options, _) => Ok(empty_task_report_with_options(HealTaskStatus::Pending, options)),
TaskStateLookup::Unrunnable(error) => {
Ok(empty_task_report_with_options(HealTaskStatus::Failed { error }, HealOptions::default()))
}
@@ -2786,9 +2753,9 @@ impl HealManager {
let canonical_task_id = self.canonical_task_id(task_id).await;
match self.lookup_task_state(&canonical_task_id, Some(heal_path)).await? {
TaskStateLookup::Active(task) => Ok(task.get_status().await),
TaskStateLookup::Retrying(status, _) => Ok(status),
TaskStateLookup::Retrying(status, _, _) => Ok(status),
TaskStateLookup::Completed(completed) => Ok(completed.status.clone()),
TaskStateLookup::Queued(_) => Ok(HealTaskStatus::Pending),
TaskStateLookup::Queued(_, _) => Ok(HealTaskStatus::Pending),
TaskStateLookup::Unrunnable(error) => Ok(HealTaskStatus::Failed { error }),
TaskStateLookup::NotFound => {
if self.path_has_task(heal_path).await {
+1 -3
View File
@@ -682,7 +682,6 @@ impl HealManager {
HealPriority::Low,
);
let recovery_anchor = recovery.as_ref().map(|(_, _, _, anchor)| anchor.clone());
let recovered_replacement = recovery.is_some();
if let Some((task_id, _, _, _)) = recovery {
req.id = task_id;
}
@@ -692,8 +691,7 @@ impl HealManager {
let endpoint_count = req.heal_endpoints.len();
let config = config.read().await;
let mut queue = heal_queue.lock().await;
let admission_decision =
Self::admit_request_to_queue(&mut queue, req, &config, "auto_scan", recovered_replacement);
let admission_decision = Self::admit_request_to_queue(&mut queue, req, &config, "auto_scan");
let admission = admission_decision.result;
let should_notify =
matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable;
+5 -52
View File
@@ -70,8 +70,6 @@ pub(super) struct PriorityQueueItem {
pub(super) sequence: u64,
pub(super) dedup_key: String,
pub(super) request: HealRequest,
/// Execution or durable recovery fixed these targets; grace waits can have zero retries.
pub(super) replacement_targets_frozen: bool,
}
impl Eq for PriorityQueueItem {}
@@ -109,7 +107,9 @@ pub(super) enum QueuePushOutcome {
#[derive(Debug, Clone)]
pub(super) struct CompletedHealStatus {
pub(super) task_id: String,
pub(super) heal_type: HealType,
pub(super) source: HealRequestSource,
/// Options used to execute the task, retained for token-scoped status.
pub(super) options: HealOptions,
pub(super) status: HealTaskStatus,
@@ -137,6 +137,7 @@ impl CompletedHealStatus {
fn measure_retained_bytes(&self) -> usize {
let mut bytes = size_of::<Self>();
let mut add = |amount: usize| bytes = bytes.saturating_add(amount);
add(self.task_id.capacity());
add(self.outcome.as_ref().map_or(0, |outcome| outcome.retained_bytes()));
match &self.heal_type {
HealType::Cluster => {}
@@ -258,7 +259,9 @@ impl CompletedHealStatus {
outcome.finish(Some(crate::heal::outcome::HealAbortReason::Cancelled));
}
let mut snapshot = Self {
task_id: task.id.clone(),
heal_type: task.heal_type.clone(),
source: task.source,
options: task.options.clone(),
status,
progress: Some(task.get_progress().await),
@@ -318,10 +321,6 @@ impl PriorityHealQueue {
}
pub(super) fn push(&mut self, request: HealRequest) -> QueuePushOutcome {
self.push_with_replacement_scope(request, false)
}
pub(super) fn push_with_replacement_scope(&mut self, request: HealRequest, frozen: bool) -> QueuePushOutcome {
let key = Self::make_dedup_key(&request);
// Check for duplicates unless the caller explicitly forces admission.
@@ -346,7 +345,6 @@ impl PriorityHealQueue {
sequence: self.sequence,
dedup_key: key,
request,
replacement_targets_frozen: frozen,
});
if is_best_effort_source(source) {
self.best_effort_queued += 1;
@@ -687,51 +685,6 @@ impl PriorityHealQueue {
self.heap.iter().map(|item| &item.request)
}
pub(super) fn merge_replacement_targets(
&mut self,
owner_id: &str,
request: &HealRequest,
allow_expand: bool,
incoming_frozen: bool,
) -> bool {
let Some(item) = self.heap.iter().find(|item| item.request.id == owner_id) else {
return false;
};
let owner = &item.request;
let same_id = request.id == owner.id;
let eligible = replacement_request_compatible(request, &owner.heal_type, &owner.options, owner.source)
&& (!incoming_frozen || same_id);
let covered = eligible
&& request
.heal_endpoints
.iter()
.all(|endpoint| owner.heal_endpoints.contains(endpoint));
let mark_frozen = incoming_frozen && same_id && !item.replacement_targets_frozen;
let can_expand = eligible
&& allow_expand
&& !item.replacement_targets_frozen
&& !incoming_frozen
&& owner.retry_attempts == 0
&& request.retry_attempts == 0
&& !owner.heal_endpoints.is_empty();
if !mark_frozen && (covered || !can_expand) {
return covered;
}
// Neither targets nor their frozen flag affect heap ordering.
let mut items = std::mem::take(&mut self.heap).into_vec();
if let Some(item) = items.iter_mut().find(|item| item.request.id == owner_id) {
item.replacement_targets_frozen |= mark_frozen;
if !covered && can_expand {
item.request.heal_endpoints.extend(request.heal_endpoints.iter().cloned());
item.request.heal_endpoints.sort_unstable();
item.request.heal_endpoints.dedup();
}
}
self.heap = BinaryHeap::from(items);
covered || can_expand
}
#[cfg(test)]
pub(super) fn contains_request_id(&self, request_id: &str) -> bool {
self.heap.iter().any(|item| item.request.id == request_id)
@@ -460,6 +460,8 @@ impl RootHealTerminal {
fn into_completed(self) -> CompletedHealStatus {
CompletedHealStatus {
task_id: self.task_id,
source: HealRequestSource::Admin,
outcome: None,
progress: self.progress,
retained_bytes: std::sync::OnceLock::new(),
+1 -1
View File
@@ -582,7 +582,7 @@ impl HealManager {
return;
}
let admission_decision =
Self::admit_request_to_queue(&mut queue, retry_request.clone(), &retry_config, "retry", true);
Self::admit_request_to_queue(&mut queue, retry_request.clone(), &retry_config, "retry");
let admission = admission_decision.result;
let should_notify = matches!(admission, HealAdmissionResult::Accepted)
&& retry_config.event_driven_scheduler_enable;
+217 -295
View File
@@ -118,6 +118,8 @@ struct MockStorage;
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
CompletedHealStatus {
task_id: "fixture-token".to_string(),
source: HealRequestSource::Internal,
outcome: None,
heal_type: HealType::Cluster,
options: HealOptions::default(),
@@ -2505,301 +2507,6 @@ fn test_heal_request_and_task_metric_labels_match() {
assert_eq!(task.metric_set_label(), "pool_0_set_1");
}
#[tokio::test]
async fn automatic_replacement_queued_targets_merge_without_changing_fifo_or_token() {
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
let owner = automatic_replacement_request(&["replacement-a"]);
let owner_id = owner.id.clone();
let created_at = owner.created_at;
let first = manager
.submit_heal_request_with_receipt(owner)
.await
.expect("queue first target");
assert_eq!(first.result, HealAdmissionResult::Accepted);
let other = HealRequest::object("bucket".to_string(), "unrelated".to_string(), None);
let other_id = other.id.clone();
manager.submit_heal_request(other).await.expect("queue unrelated work");
let merged = manager
.submit_heal_request_with_receipt(automatic_replacement_request(&["replacement-b", "replacement-a"]))
.await
.expect("merge all fresh targets");
assert_eq!(merged.result, HealAdmissionResult::Merged);
assert_eq!(merged.task_id, owner_id);
let mut queue = manager.heal_queue.lock().await;
assert_eq!(queue.len(), 2);
let owner = queue.pop_next().expect("original task remains first");
assert_eq!(owner.id, owner_id);
assert_eq!(owner.created_at, created_at);
assert_eq!(owner.heal_endpoints, ["replacement-a", "replacement-b"]);
assert_eq!(queue.pop_next().expect("unrelated task remains queued").id, other_id);
}
fn automatic_replacement_request(targets: &[&str]) -> HealRequest {
let mut request = HealRequest::new(
HealType::ErasureSet {
buckets: Vec::new(),
set_disk_id: "pool_0_set_1".to_string(),
},
HealOptions {
pool_index: Some(0),
set_index: Some(1),
..Default::default()
},
HealPriority::Normal,
);
request.source = HealRequestSource::AutoHeal;
request.heal_endpoints = targets.iter().map(|target| (*target).to_string()).collect();
request
}
#[tokio::test]
async fn automatic_replacement_frozen_targets_reject_late_disks_but_cover_duplicates() {
for state in ["active", "retrying", "grace", "recovered", "recovery_anchor"] {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
let manager = HealManager::new_without_root_recovery_for_test(storage.clone(), None);
let mut owner = automatic_replacement_request(&["replacement-a"]);
let owner_id = owner.id.clone();
if state == "grace" {
let now = SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("test clock")
.as_secs();
let task = HealTask::from_request(owner, storage.clone());
owner = retry_request_for_result_with_budget(
&task,
&Err(Error::DanglingDeleteDeferred {
retry_not_before: now + 60,
}),
)
.await
.expect("grace wait keeps the generation replayable")
.0;
assert_eq!(owner.retry_attempts, 0, "grace must not consume a failure retry");
}
match state {
"active" => {
manager
.active_heals
.lock()
.await
.insert(owner_id.clone(), Arc::new(HealTask::from_request(owner, storage.clone())));
}
"retrying" => {
manager.retrying_heals.lock().await.insert(
owner_id.clone(),
RetryingHeal {
request: owner,
error: "target unavailable".to_string(),
cancel_token: CancellationToken::new(),
},
);
}
"grace" | "recovered" => {
let config = manager.config.read().await.clone();
let mut queue = manager.heal_queue.lock().await;
let context = if state == "grace" { "retry" } else { "auto_scan" };
assert_eq!(
HealManager::admit_request_to_queue(&mut queue, owner, &config, context, true).result,
HealAdmissionResult::Accepted
);
assert!(
queue
.heap
.iter()
.any(|item| item.request.id == owner_id && item.replacement_targets_frozen),
"zero retries cannot mean the scope is fresh"
);
}
_ => {
manager.submit_heal_request(owner).await.expect("queue frozen generation");
if state == "recovery_anchor" {
manager
.replacement_recovery_anchors
.lock()
.expect("recovery anchors")
.insert(owner_id.clone(), "survivor".to_string());
}
}
}
let late = manager
.submit_heal_request(automatic_replacement_request(&["replacement-b"]))
.await
.expect("classify late target");
assert_eq!(late, HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning), "{state}");
let covered = manager
.submit_heal_request_with_receipt(automatic_replacement_request(&["replacement-a"]))
.await
.expect("covered target remains owned");
assert_eq!(covered.result, HealAdmissionResult::Merged, "{state}");
assert_eq!(covered.task_id, owner_id, "{state}");
let targets = match state {
"active" => manager.active_heals.lock().await[&owner_id].heal_endpoints.clone(),
"retrying" => manager.retrying_heals.lock().await[&owner_id].request.heal_endpoints.clone(),
_ => {
manager
.heal_queue
.lock()
.await
.pop_next()
.expect("retained generation")
.heal_endpoints
}
};
assert_eq!(targets, ["replacement-a"], "{state}: frozen scope must not grow");
}
}
#[tokio::test]
async fn automatic_replacement_merge_preserves_options_policy_and_recovery_identity() {
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
manager
.submit_heal_request(automatic_replacement_request(&["replacement-a"]))
.await
.expect("queue fresh generation");
for variant in [
"dry_run",
"bucket_scope",
"pool_scope",
"retried_incoming",
"recovery_anchor",
"merge_disabled",
] {
let mut incoming = automatic_replacement_request(&["replacement-b"]);
match variant {
"dry_run" => incoming.options.dry_run = true,
"bucket_scope" => {
if let HealType::ErasureSet { buckets, .. } = &mut incoming.heal_type {
buckets.push("other-bucket".to_string());
}
}
"pool_scope" => incoming.options.pool_index = Some(2),
"retried_incoming" => {
incoming.heal_endpoints = vec!["replacement-a".to_string()];
incoming.retry_attempts = 1;
}
"recovery_anchor" => {
incoming.heal_endpoints = vec!["replacement-a".to_string()];
manager
.replacement_recovery_anchors
.lock()
.expect("recovery anchors")
.insert(incoming.id.clone(), "survivor".to_string());
}
"merge_disabled" => {
manager.config.write().await.low_priority_merge_enable = false;
incoming.priority = HealPriority::Low;
}
_ => unreachable!("test variant"),
}
let result = manager
.submit_heal_request(incoming)
.await
.expect("classify incompatible generation");
let reason = if variant == "merge_disabled" {
HealAdmissionDropReason::PolicyDropped
} else {
HealAdmissionDropReason::AlreadyRunning
};
assert_eq!(result, HealAdmissionResult::Dropped(reason), "{variant}");
let queue = manager.heal_queue.lock().await;
assert_eq!(queue.len(), 1, "{variant}");
assert_eq!(
queue.requests().next().expect("original generation").heal_endpoints,
["replacement-a"],
"{variant}"
);
}
}
#[test]
fn automatic_replacement_direct_admission_checks_frozen_targets_before_capacity() {
let config = HealConfig {
queue_size: 1,
..HealConfig::default()
};
let mut queue = PriorityHealQueue::new();
let owner = automatic_replacement_request(&["replacement-a"]);
let owner_id = owner.id.clone();
assert_eq!(queue.push(owner), QueuePushOutcome::Accepted);
let admit = |queue: &mut PriorityHealQueue, request, frozen| {
HealManager::admit_request_to_queue(queue, request, &config, "auto_scan", frozen).result
};
assert_eq!(
admit(&mut queue, automatic_replacement_request(&["replacement-b"]), false),
HealAdmissionResult::Merged
);
let mut replay = automatic_replacement_request(&["replacement-a"]);
replay.id = owner_id.clone();
assert_eq!(admit(&mut queue, replay, true), HealAdmissionResult::Merged);
assert!(
queue
.heap
.iter()
.any(|item| item.request.id == owner_id && item.replacement_targets_frozen)
);
assert_eq!(
admit(&mut queue, automatic_replacement_request(&["replacement-c"]), false),
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
);
assert_eq!(
admit(&mut queue, automatic_replacement_request(&["replacement-a"]), false),
HealAdmissionResult::Merged
);
assert_eq!(
admit(&mut queue, automatic_replacement_request(&["replacement-a"]), true),
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
);
assert_eq!(queue.len(), 1);
let owner = queue.pop_next().expect("frozen owner retained");
assert_eq!(owner.id, owner_id);
assert_eq!(owner.heal_endpoints, ["replacement-a", "replacement-b"]);
let owner = automatic_replacement_request(&["replacement-a"]);
let owner_id = owner.id.clone();
assert_eq!(queue.push(owner), QueuePushOutcome::Accepted);
let mut conflicting_replay = automatic_replacement_request(&["replacement-b"]);
conflicting_replay.id = owner_id.clone();
assert_eq!(
admit(&mut queue, conflicting_replay, true),
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
);
assert!(
queue
.heap
.iter()
.any(|item| item.request.id == owner_id && item.replacement_targets_frozen)
);
assert_eq!(
admit(&mut queue, automatic_replacement_request(&["replacement-c"]), false),
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
);
let mut queue = PriorityHealQueue::new();
let owner = automatic_replacement_request(&["replacement-a"]);
let owner_id = owner.id.clone();
assert_eq!(queue.push(owner), QueuePushOutcome::Accepted);
let mut incompatible_replay = automatic_replacement_request(&["replacement-a"]);
incompatible_replay.id = owner_id.clone();
incompatible_replay.options.dry_run = true;
assert_eq!(
admit(&mut queue, incompatible_replay, true),
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
);
assert!(
queue
.heap
.iter()
.any(|item| item.request.id == owner_id && item.replacement_targets_frozen)
);
assert_eq!(
admit(&mut queue, automatic_replacement_request(&["replacement-b"]), false),
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
);
}
#[tokio::test]
async fn test_submit_heal_request_returns_merged_for_duplicate() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
@@ -3071,6 +2778,181 @@ async fn test_active_duplicate_token_can_query_and_cancel_original_task() {
);
}
#[tokio::test]
async fn local_automatic_query_preserves_source_path_token_and_state() {
use crate::heal::channel::HealChannelProcessor;
use crate::heal::outcome::{
HealExecutionOutcome, HealObjectDisposition, HealObjectIdentity, HealObjectKind, HealObjectOutcome, HealTaskOutcome,
HealTraversalCoverage,
};
for source in [
HealRequestSource::Scanner,
HealRequestSource::Mrf,
HealRequestSource::AutoHeal,
HealRequestSource::ReadRepair,
HealRequestSource::Admin,
HealRequestSource::Internal,
] {
for state in ["queued", "active", "retrying", "completed", "displaced"] {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(storage.clone(), None));
let mut request = HealRequest::object("bucket".to_string(), "object".to_string(), None);
request.source = source;
let token = request.id.clone();
let task = Arc::new(HealTask::from_request(request.clone(), storage));
let mut snapshot = CompletedHealStatus::snapshot(&task, HealTaskStatus::Completed).await;
assert_eq!(snapshot.source, source, "completion must retain actual task source");
let mut outcome = HealTaskOutcome::default();
outcome.execution = HealExecutionOutcome::Completed;
outcome.coverage = HealTraversalCoverage::Complete;
outcome.record(HealObjectOutcome {
identity: HealObjectIdentity {
kind: HealObjectKind::Object,
bucket: "bucket".to_string(),
object: "object".to_string(),
version_id: None,
bucket_incarnation_id: Some(uuid::Uuid::nil()),
pool_index: Some(0),
set_index: Some(0),
},
disposition: HealObjectDisposition::VerifiedHealthy,
detail: None,
});
snapshot.outcome = Some(Arc::new(outcome));
snapshot.seqed_items = vec![(
3,
HealResultItem {
bucket: "bucket".to_string(),
object: "object".to_string(),
..Default::default()
},
)];
snapshot.min_seq = 3;
snapshot.next_seq = 4;
let expected_status = match state {
"queued" => {
manager.heal_queue.lock().await.push(request.clone());
HealTaskStatus::Pending
}
"active" => {
manager.active_heals.lock().await.insert(token.clone(), task);
HealTaskStatus::Pending
}
"retrying" => {
manager.retrying_heals.lock().await.insert(
token.clone(),
RetryingHeal {
request: request.clone(),
error: "transient".to_string(),
cancel_token: CancellationToken::new(),
},
);
HealTaskStatus::Retrying {
error: "transient".to_string(),
retry_attempt: request.retry_attempts,
}
}
"completed" => {
manager.completed_heals.lock().await.insert(token.clone(), Arc::new(snapshot));
HealTaskStatus::Completed
}
"displaced" => {
let terminal = record_displaced_terminal(&manager.displaced_terminals, &request);
terminal.status.clone()
}
_ => unreachable!("fixed test states"),
};
let automatic = !matches!(source, HealRequestSource::Admin | HealRequestSource::Internal);
let report = manager
.get_local_automatic_task_report("bucket/object", &token, Some(2))
.await
.expect("local source lookup");
assert_eq!(report.is_some(), automatic, "source={source:?} state={state}");
if let Some(report) = report {
assert_eq!(report.status, expected_status);
if state == "completed" {
let outcome = report.outcome.as_ref().expect("retained structured outcome");
assert_eq!(outcome.objects.len(), 1);
assert_eq!(outcome.objects[0].identity.object, "object");
assert_eq!(outcome.objects[0].identity.pool_index, Some(0));
assert_eq!(outcome.objects[0].identity.set_index, Some(0));
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::VerifiedHealthy);
assert_eq!(report.min_seq, 3);
assert_eq!(report.next_seq, 4);
assert_eq!(report.result_items.len(), 1);
assert_eq!(report.result_items[0].object, "object");
}
let processor = HealChannelProcessor::new(manager.clone());
let local = processor
.execute_local_automatic_query("bucket/object", &token, Some(2))
.await
.expect("local response")
.expect("automatic task response");
let direct = processor
.execute_query_request_since("bucket/object".to_string(), token.clone(), Some(2))
.await
.expect("original query response");
assert!(local.success);
assert_eq!(local.request_id, direct.request_id);
assert_eq!(local.data, direct.data, "same report encoding and cursor contract");
assert_eq!(local.error, direct.error);
let coordinator = Arc::new(HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None));
let missing = HealChannelProcessor::new(coordinator)
.execute_query_request("bucket/object".to_string(), token.clone())
.await
.expect("coordinator query");
let body: serde_json::Value =
serde_json::from_slice(missing.data.as_deref().expect("coordinator response")).expect("coordinator JSON");
assert_eq!(body["summary"], "notFound", "a remote manager does not own the automatic task");
if state == "completed" {
let completed = manager
.completed_heals
.lock()
.await
.get(&token)
.cloned()
.expect("original completion");
manager
.completed_heals
.lock()
.await
.insert("retired-alias".to_string(), completed);
assert!(
manager
.get_local_automatic_task_report("bucket/object", "retired-alias", None)
.await
.expect("retired alias lookup")
.is_none()
);
}
}
assert!(
manager
.get_local_automatic_task_report("bucket/other", &token, None)
.await
.expect("wrong path lookup")
.is_none()
);
assert!(
manager
.get_local_automatic_task_report("bucket/object", "unknown", None)
.await
.expect("unknown token lookup")
.is_none()
);
manager.insert_task_alias("admin-alias", &token).await;
assert!(
manager
.get_local_automatic_task_report("bucket/object", "admin-alias", None)
.await
.expect("alias lookup")
.is_none()
);
}
}
}
#[tokio::test]
async fn test_queued_duplicate_token_can_query_and_cancel_original_request() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
@@ -3886,6 +3768,38 @@ async fn test_retry_request_rescans_batch_when_all_exhausted_objects_are_retryab
}
}
#[tokio::test]
async fn admin_exhausted_batches_do_not_restart_the_traversal() {
for heal_type in [
HealType::Bucket {
bucket: "bucket".to_owned(),
},
HealType::Prefix {
bucket: "bucket".to_owned(),
prefix: "prefix/".to_owned(),
},
HealType::Cluster,
] {
let mut request = HealRequest::new(heal_type, HealOptions::default(), HealPriority::Normal);
request.source = HealRequestSource::Admin;
let task = HealTask::from_request(request, Arc::new(MockStorage));
let result = Err(task
.record_batch_failure(BatchHealFailure {
scope: "completed-admin-traversal".to_owned(),
failed: 1,
retryable: 1,
permanent: 0,
first_object: "exhausted-object".to_owned(),
first_error: Error::Storage(EcstoreError::SlowDown).to_string(),
})
.await);
assert!(
retry_request_for_result_with_budget(&task, &result).await.is_none(),
"the exhausted admin batch must retain its original object receipts"
);
}
}
#[tokio::test]
async fn test_retry_request_does_not_rescan_cancelled_or_timed_out_retryable_batch() {
for terminal_error in [Error::TaskCancelled, Error::TaskTimeout] {
@@ -3968,6 +3882,8 @@ async fn insert_retrying_request(manager: &HealManager, request: HealRequest) ->
Arc::new(CompletedHealStatus {
progress: None,
outcome: None,
source: request.source,
task_id: request.id.clone(),
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type,
options: request.options.clone(),
@@ -4816,6 +4732,8 @@ async fn test_retrying_completion_outranks_the_queue_for_the_same_id() {
Arc::new(CompletedHealStatus {
progress: None,
outcome: None,
source: request.source,
task_id: request.id.clone(),
retained_bytes: std::sync::OnceLock::new(),
heal_type: request.heal_type.clone(),
options: request.options.clone(),
@@ -4855,6 +4773,8 @@ async fn test_get_task_status_reads_recent_completed_status() {
Arc::new(CompletedHealStatus {
progress: None,
outcome: None,
source: HealRequestSource::Internal,
task_id: "completed-token".to_string(),
retained_bytes: std::sync::OnceLock::new(),
heal_type: HealType::Bucket {
bucket: "bucket".to_string(),
@@ -4888,6 +4808,8 @@ async fn test_get_task_report_for_path_reads_completed_items() {
Arc::new(CompletedHealStatus {
progress: None,
outcome: None,
source: HealRequestSource::Internal,
task_id: "completed-token".to_string(),
retained_bytes: std::sync::OnceLock::new(),
heal_type: HealType::Object {
bucket: "bucket".to_string(),
@@ -258,16 +258,6 @@ impl ScannerFixture {
assert_eq!(request.id, self.task_id, "scanner must replay the durable UUID");
assert_eq!(request.heal_endpoints, vec![self.local_targets[0].clone()]);
assert_eq!(request.source, HealRequestSource::AutoHeal);
assert!(
self.manager
.heal_queue
.lock()
.await
.heap
.iter()
.any(|item| item.request.id == request.id && item.replacement_targets_frozen),
"replayed durable targets cannot expand"
);
assert!(matches!(
request.heal_type,
HealType::ErasureSet { set_disk_id, buckets }
@@ -117,6 +117,8 @@ fn admin_request(heal_type: HealType) -> HealRequest {
fn completed_admin_status(heal_type: &HealType, completed_at: SystemTime) -> CompletedHealStatus {
CompletedHealStatus {
task_id: "admin-fixture".to_string(),
source: HealRequestSource::Admin,
outcome: None,
heal_type: heal_type.clone(),
options: HealOptions::default(),
@@ -1239,6 +1241,8 @@ async fn root_recovery_completed_non_root_admin_is_queryable_after_restart() {
.expect("durable bucket responsibility");
let completed = CompletedHealStatus {
outcome: None,
source: request.source,
task_id: request.id.clone(),
heal_type: request.heal_type.clone(),
options: request.options.clone(),
status: HealTaskStatus::Completed,
+21 -2
View File
@@ -37,7 +37,7 @@ use std::{
collections::VecDeque,
future::Future,
sync::{
Arc,
Arc, OnceLock,
atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering},
},
time::{Duration, Instant, SystemTime},
@@ -59,6 +59,19 @@ pub(crate) struct OutcomeFinishTestHook {
pub(crate) static OUTCOME_FINISH_TEST_HOOK: std::sync::LazyLock<tokio::sync::Mutex<Option<Arc<OutcomeFinishTestHook>>>> =
std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
#[cfg(test)]
static STORAGE_RECEIPT_TEST_HOOK: std::sync::LazyLock<tokio::sync::Mutex<Option<Arc<OutcomeFinishTestHook>>>> =
std::sync::LazyLock::new(|| tokio::sync::Mutex::new(None));
#[cfg(test)]
async fn pause_storage_receipt(task_id: &str) {
let hook = STORAGE_RECEIPT_TEST_HOOK.lock().await.clone();
if let Some(hook) = hook.filter(|hook| hook.task_id == task_id) {
hook.reached.notify_one();
hook.release.notified().await;
}
}
#[cfg(test)]
async fn pause_outcome_finish(task_id: &str) {
let hook = OUTCOME_FINISH_TEST_HOOK.lock().await.clone();
@@ -436,6 +449,9 @@ pub struct HealTask {
replacement_start_retry_count: Arc<AtomicU32>,
/// Set when the locked storage read classified the single target as absent.
object_absent: Arc<AtomicBool>,
/// The first resolved incarnation belongs to this task even if storage
/// fails or the bucket is recreated before its terminal outcome is recorded.
single_object_incarnation: Arc<OnceLock<Option<Uuid>>>,
/// Task status
pub status: Arc<RwLock<HealTaskStatus>>,
/// Progress tracking
@@ -508,6 +524,7 @@ impl HealTask {
replacement_running: Arc::new(AtomicBool::new(false)),
replacement_start_retry_count: Arc::new(AtomicU32::new(0)),
object_absent: Arc::new(AtomicBool::new(false)),
single_object_incarnation: Arc::new(OnceLock::new()),
status: Arc::new(RwLock::new(HealTaskStatus::Pending)),
progress: Arc::new(RwLock::new(HealProgress::new())),
result_items: Arc::new(RwLock::new(VecDeque::with_capacity(MAX_RETAINED_HEAL_RESULT_ITEMS))),
@@ -669,7 +686,9 @@ impl HealTask {
} => (bucket, object, Some(version_id.as_str())),
_ => return None,
};
Some(self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index))
let mut identity = self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index);
identity.bucket_incarnation_id = self.single_object_incarnation.get().copied().flatten();
Some(identity)
}
pub(super) async fn record_verified_storage_receipt(
+21 -15
View File
@@ -306,9 +306,9 @@ impl HealTask {
let mut progress = self.progress.write().await;
progress.update_stage(3, 3);
}
Err(Error::TaskExecutionFailed {
message: format!("Failed to heal bucket {bucket}: {e}"),
})
// Keep storage/quorum identity for the cluster retry classifier.
// The structured log above already carries the bucket context.
Err(e)
}
}
}
@@ -796,21 +796,27 @@ impl HealTask {
match unavailable_recreate_error(&storage_result.item, &heal_opts) {
Some(error) => Some(error),
None => {
telemetry_unknown |= !increment_counter(&mut healed);
telemetry_unknown |= !add_bytes(
&mut bytes,
u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
);
#[cfg(test)]
super::pause_storage_receipt(&self.id).await;
recorded_authoritative_outcome =
self.record_verified_storage_receipt(&identity, storage_result.receipt).await;
if !recorded_authoritative_outcome
&& !heal_opts.dry_run
&& !storage_result.item.detail.is_empty()
{
detail = Some(storage_result.item.detail.clone());
if !recorded_authoritative_outcome && self.cancel_token.is_cancelled() {
Some(Error::TaskCancelled)
} else {
telemetry_unknown |= !increment_counter(&mut healed);
telemetry_unknown |= !add_bytes(
&mut bytes,
u64::try_from(storage_result.item.object_size).unwrap_or(u64::MAX),
);
if !recorded_authoritative_outcome
&& !heal_opts.dry_run
&& !storage_result.item.detail.is_empty()
{
detail = Some(storage_result.item.detail.clone());
}
self.record_result_item(storage_result.item).await;
None
}
self.record_result_item(storage_result.item).await;
None
}
}
}
+5
View File
@@ -220,6 +220,11 @@ impl HealTask {
let mut expected_identity =
self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index);
expected_identity.bucket_incarnation_id = self.outcome_bucket_incarnation_id(bucket, self.options.dry_run).await?;
if matches!(self.heal_type, HealType::Object { .. }) {
expected_identity.bucket_incarnation_id = *self
.single_object_incarnation
.get_or_init(|| expected_identity.bucket_incarnation_id);
}
let heal_fut = self.storage.heal_object_with_receipt(bucket, object, version_id, &heal_opts);
let heal_result = if self.source == HealRequestSource::ReadRepair {
+257
View File
@@ -305,6 +305,53 @@ mod canonical_outcome {
assert_eq!(storage.healed_objects.lock().expect("healed objects").len(), 2);
}
#[tokio::test]
async fn bucket_cancellation_after_storage_success_keeps_typed_cancelled_responsibility() {
let incarnation = Uuid::new_v4();
let storage = Arc::new(MockStorage {
heal_object_receipts: Mutex::new(HashMap::from([(
"object-a".to_string(),
VecDeque::from([object_receipt("object-a", None, HealObjectDisposition::Repaired, incarnation)]),
)])),
bucket_incarnation_id: Mutex::new(Some(incarnation)),
..Default::default()
});
let task = Arc::new(bucket_task(storage.clone()));
let hook = Arc::new(OutcomeFinishTestHook {
task_id: task.id.clone(),
reached: tokio::sync::Notify::new(),
release: tokio::sync::Notify::new(),
});
*STORAGE_RECEIPT_TEST_HOOK.lock().await = Some(hook.clone());
let runner = {
let task = task.clone();
tokio::spawn(async move { task.execute().await })
};
tokio::time::timeout(Duration::from_secs(10), hook.reached.notified())
.await
.expect("storage success must reach the receipt boundary");
assert_eq!(storage.healed_objects.lock().expect("storage mutations").as_slice(), ["object-a"]);
task.cancel().await.expect("cancel before receipt recording");
hook.release.notify_one();
let result = tokio::time::timeout(Duration::from_secs(10), runner)
.await
.expect("cancelled worker must stop")
.expect("worker must not panic");
STORAGE_RECEIPT_TEST_HOOK.lock().await.take();
assert!(matches!(result, Err(Error::TaskCancelled)));
let outcome = task.get_outcome().await;
assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Cancelled));
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
assert_eq!(outcome.counters.processed, 1);
assert_eq!(outcome.counters.healed, 0);
assert_eq!(outcome.counters.unchanged, 0);
assert_eq!(outcome.counters.unknown, 0);
assert_eq!(outcome.counters.skipped, 1);
assert_eq!(outcome.objects.len(), 1);
assert_eq!(outcome.objects[0].identity.object, "object-a");
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Cancelled);
}
#[tokio::test]
async fn bucket_heal_rejects_stale_receipts_without_double_recording() {
let expected_incarnation = Uuid::new_v4();
@@ -809,6 +856,85 @@ mod canonical_outcome {
);
}
#[tokio::test(start_paused = true)]
async fn cluster_bucket_write_quorum_retries_preserve_budget_and_truthful_coverage() {
for (failures, succeeds) in [(1, true), (4, false)] {
let storage = Arc::new(MockStorage {
listed_buckets: Mutex::new(Some(vec!["bucket-a".to_string(), "bucket-b".to_string()])),
typed_bucket_heal_errors: Mutex::new(HashMap::from([(
"bucket-a".to_string(),
(0..failures)
.map(|_| Error::Storage(EcstoreError::ErasureWriteQuorum))
.collect(),
)])),
..Default::default()
});
let task = HealTask::from_request(
HealRequest::new(
HealType::Cluster,
HealOptions {
recursive: true,
timeout: None,
..Default::default()
},
HealPriority::Normal,
),
storage.clone(),
);
let result = task.execute().await;
assert_eq!(
result.is_ok(),
succeeds,
"original cluster task must honor its bounded bucket retry budget"
);
let outcome = task.get_outcome().await;
if succeeds {
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
} else {
assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
let failure = task.take_batch_failure().await.expect("exhausted bucket retains its failure");
assert_eq!((failure.failed, failure.retryable, failure.permanent), (1, 1, 0));
}
let calls = storage.bucket_heal_calls.lock().expect("original bucket calls");
assert_eq!(
calls.iter().filter(|bucket| bucket.as_str() == "bucket-a").count(),
if succeeds { 2 } else { 4 }
);
assert_eq!(calls.last().map(String::as_str), Some("bucket-b"), "later buckets must still be visited");
}
}
#[tokio::test]
async fn bucket_prepass_preserves_typed_quorum_and_terminal_access_denial() {
for error in [
Error::Storage(EcstoreError::ErasureWriteQuorum),
Error::Disk(DiskError::FileAccessDenied),
] {
let retryable = error.is_recoverable_heal();
let storage = Arc::new(MockStorage {
typed_bucket_heal_errors: Mutex::new(HashMap::from([("bucket-a".to_string(), VecDeque::from([error]))])),
..Default::default()
});
let task = bucket_task(storage.clone());
let observed = task
.heal_bucket("bucket-a")
.await
.expect_err("bucket prepass must retain the storage error");
assert!(matches!(
observed,
Error::Storage(EcstoreError::ErasureWriteQuorum) | Error::Disk(DiskError::FileAccessDenied)
));
assert_eq!(observed.is_recoverable_heal(), retryable);
assert_eq!(storage.bucket_heal_calls.lock().expect("bucket calls").as_slice(), ["bucket-a"]);
assert!(
storage.heal_object_calls.lock().expect("object calls").is_empty(),
"failed prepass must not heal objects"
);
}
}
#[tokio::test(start_paused = true)]
async fn cancellation_and_deadline_leave_partial_coverage() {
for cancel in [false, true] {
@@ -1567,6 +1693,7 @@ struct MockStorage {
listed_buckets: Mutex<Option<Vec<String>>>,
bucket_discovery_incomplete: bool,
bucket_heal_errors: Mutex<HashMap<String, VecDeque<&'static str>>>,
typed_bucket_heal_errors: Mutex<HashMap<String, VecDeque<Error>>>,
bucket_heal_calls: Mutex<Vec<String>>,
delete_recovery_results: Mutex<HashMap<String, Result<Option<HealResultItem>>>>,
delete_recovery_calls: Mutex<Vec<String>>,
@@ -1815,6 +1942,127 @@ async fn failed_object_heal_rejects_matching_positive_storage_receipt() {
| HealObjectDisposition::AuthoritativelyAbsent
)
}));
assert_eq!(outcome.objects.len(), 1);
assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(incarnation));
}
#[tokio::test]
async fn object_failure_retains_incarnation_for_inner_and_outer_storage_errors() {
for failure in [
MockHealObjectOutcome::PermissionDenied,
MockHealObjectOutcome::RetryableReadQuorum,
MockHealObjectOutcome::RetryableLockTimeout,
MockHealObjectOutcome::ErrOther("backend failed"),
] {
let incarnation = Uuid::new_v4();
let storage = Arc::new(MockStorage {
bucket_incarnation_id: Mutex::new(Some(incarnation)),
heal_object_outcome: Mutex::new(Some(failure)),
..Default::default()
});
let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), Some("version-a".to_owned()));
request.options.pool_index = Some(2);
request.options.set_index = Some(3);
let task = HealTask::from_request(request, storage);
task.execute()
.await
.expect_err("the failed storage operation must remain failed");
let outcome = task.get_outcome().await;
assert_eq!(outcome.objects.len(), 1);
assert_eq!(
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
(1, 1, 1)
);
assert_eq!((outcome.counters.healed, outcome.counters.unchanged, outcome.counters.unknown), (0, 0, 0));
let object = &outcome.objects[0];
assert_eq!(object.identity.bucket_incarnation_id, Some(incarnation));
assert_eq!(object.identity.bucket, "bucket-a");
assert_eq!(object.identity.object, "object-a");
assert_eq!(object.identity.version_id.as_deref(), Some("version-a"));
assert_eq!((object.identity.pool_index, object.identity.set_index), (Some(2), Some(3)));
assert!(matches!(object.disposition, HealObjectDisposition::Failed(_)));
}
}
#[tokio::test(start_paused = true)]
async fn object_abort_keeps_pre_storage_incarnation_after_bucket_recreation() {
for cancel in [false, true] {
let original = Uuid::new_v4();
let storage = Arc::new(MockStorage {
bucket_incarnation_id: Mutex::new(Some(original)),
block_heal_object: Mutex::new(true),
..Default::default()
});
let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None);
request.options.timeout = Some(Duration::from_secs(10));
let task = HealTask::from_request(request, storage.clone());
let execution = task.execute();
tokio::pin!(execution);
assert!(futures::poll!(&mut execution).is_pending());
assert_eq!(storage.heal_object_calls.lock().expect("storage boundary").as_slice(), ["object-a"]);
*storage.bucket_incarnation_id.lock().expect("recreated bucket") = Some(Uuid::new_v4());
if cancel {
task.cancel().await.expect("cancel blocked storage");
} else {
tokio::time::advance(Duration::from_secs(11)).await;
}
let result = execution.await;
assert!(matches!(
(&result, cancel),
(Err(Error::TaskCancelled), true) | (Err(Error::TaskTimeout), false)
));
let outcome = task.get_outcome().await;
assert_eq!(outcome.objects.len(), 1);
assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(original));
assert_eq!((outcome.counters.processed, outcome.counters.skipped), (1, 1));
assert_eq!((outcome.counters.failed, outcome.counters.healed, outcome.counters.unknown), (0, 0, 0));
assert_eq!(
outcome.objects[0].disposition,
if cancel {
HealObjectDisposition::Cancelled
} else {
HealObjectDisposition::Deferred {
reason: HealDeferredReason::Deadline,
retry_not_before: None,
}
}
);
}
}
#[tokio::test]
async fn object_unverified_outcome_keeps_original_incarnation_without_claiming_repair() {
let original = Uuid::new_v4();
let successor = Uuid::new_v4();
let storage = Arc::new(MockStorage {
bucket_incarnation_id: Mutex::new(Some(original)),
bucket_incarnation_after_object_heal: Mutex::new(Some(successor)),
..Default::default()
});
let task = HealTask::from_request(HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None), storage.clone());
task.execute().await.expect("legacy storage keeps its unverified result");
let outcome = task.get_outcome().await;
assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, Some(original));
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::Unknown);
assert_eq!((outcome.counters.healed, outcome.counters.unknown), (0, 1));
assert_eq!(*storage.bucket_incarnation_id.lock().expect("successor"), Some(successor));
}
#[tokio::test]
async fn object_dry_run_does_not_adopt_bucket_incarnation() {
let storage = Arc::new(MockStorage {
bucket_incarnation_id: Mutex::new(Some(Uuid::new_v4())),
..Default::default()
});
let mut request = HealRequest::object("bucket-a".to_owned(), "object-a".to_owned(), None);
request.options.dry_run = true;
let task = HealTask::from_request(request, storage);
task.execute().await.expect("dry-run observation");
let outcome = task.get_outcome().await;
assert_eq!(outcome.objects.len(), 1);
assert_eq!(outcome.objects[0].identity.bucket_incarnation_id, None);
assert_eq!(outcome.objects[0].disposition, HealObjectDisposition::DryRunObserved);
assert_eq!(outcome.counters.healed, 0);
}
#[tokio::test]
@@ -2387,6 +2635,15 @@ impl HealStorageAPI for MockStorage {
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
self.bucket_heal_calls.lock().unwrap().push(bucket.to_string());
self.bucket_heal_opts.lock().unwrap().push(*opts);
if let Some(error) = self
.typed_bucket_heal_errors
.lock()
.unwrap()
.get_mut(bucket)
.and_then(VecDeque::pop_front)
{
return Err(error);
}
if let Some(message) = self
.bucket_heal_errors
.lock()
@@ -1,29 +0,0 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use rustfs_heal::{HealOptions, HealPriority, HealRequest, HealType};
use rustfs_heal_contracts::heal_channel::HealRequestSource;
use std::time::SystemTime;
#[test]
fn existing_heal_request_literal_remains_constructible() {
let now = SystemTime::now();
let request = HealRequest {
id: "request".to_string(),
heal_type: HealType::ErasureSet {
set_disk_id: "pool_0_set_0".to_string(),
buckets: Vec::new(),
},
bucket_incarnation_id: None,
expected_mrf_bucket_incarnation_id: None,
options: HealOptions::default(),
priority: HealPriority::Normal,
source: HealRequestSource::AutoHeal,
force_start: false,
retry_attempts: 0,
heal_endpoints: vec!["drive-a".to_string()],
created_at: now,
enqueued_at: now,
};
assert_eq!(request.heal_endpoints, ["drive-a"]);
}
@@ -550,3 +550,159 @@ fn admin_normal_protected_repairs_report_repaired_for_data_and_parity() {
.join()
.expect("heal fixture result");
}
#[test]
#[serial]
fn admin_mixed_batch_retains_repairs_after_object_retry_exhaustion() {
std::thread::Builder::new()
.stack_size(32 * 1024 * 1024)
.spawn(|| {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("mixed batch runtime")
.block_on(async {
use rustfs_heal::heal::{
manager::{HealConfig, HealManager},
outcome::{HealExecutionOutcome, HealFailureClass, HealTraversalCoverage},
task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType},
};
use rustfs_heal_contracts::heal_channel::HealRequestSource;
use std::sync::Arc;
let root = tempfile::tempdir().expect("mixed batch fixture");
let env = TestECStoreEnv::builder().base_dir(root.path()).build().await;
let bucket = "admin-mixed-batch";
env.make_bucket(bucket, false).await;
let set = env.ecstore.pools[0].get_disks(0);
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
let mut originals = Vec::new();
for (object, missing, byte) in [("healthy", 0, 0x11), ("repaired", 1, 0x52), ("failed", 3, 0xa3)] {
env.ecstore
.put_object(
bucket,
object,
&mut PutObjReader::from_vec(vec![byte; 5 * 1024 * 1024 + 123]),
&ObjectOptions {
shard_integrity_write_mode: Some(ShardIntegrityWriteMode::Protected),
write_completion: WriteCompletion::TailDrained,
..Default::default()
},
)
.await
.expect("commit complete fixture");
for (slot, disk) in disks.iter().enumerate() {
let meta = disk
.read_version("", bucket, object, "", &ReadOptions::default())
.await
.expect("fixture metadata");
assert_eq!(meta.erasure.data_blocks, 2, "failed fixture must retain R-1 shards");
let path = env.disk_paths[slot]
.join(bucket)
.join(object)
.join(meta.data_dir.expect("external shard").to_string())
.join("part.1");
let bytes = tokio::fs::read(&path).await.expect("original part");
if slot < missing {
tokio::fs::remove_file(&path).await.expect("inject missing part");
}
originals.push((object, path, bytes, slot < missing));
}
}
let storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
let manager = HealManager::new(
storage.clone(),
Some(HealConfig {
enable_auto_heal: false,
..Default::default()
}),
);
manager.start().await.expect("start mixed batch manager");
let mut request = HealRequest::new(
HealType::Bucket {
bucket: bucket.to_owned(),
},
HealOptions {
recursive: true,
scan_mode: HealScanMode::Deep,
pool_index: Some(0),
set_index: Some(0),
timeout: Some(Duration::from_secs(150)),
..Default::default()
},
HealPriority::High,
);
request.source = HealRequestSource::Admin;
let token = request.id.clone();
manager.submit_heal_request(request).await.expect("admit original mixed task");
let report = tokio::time::timeout(Duration::from_secs(150), async {
loop {
let report = manager.get_task_report(&token).await.expect("same-token report");
if matches!(
report.status,
HealTaskStatus::Completed | HealTaskStatus::Failed { .. } | HealTaskStatus::Timeout
) {
break report;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.expect("mixed task terminal deadline");
manager.stop().await.expect("stop mixed task manager");
for (object, path, original, missing) in originals {
if object == "failed" && missing {
assert!(!path.exists(), "below-R failure must not invent a shard");
} else {
assert_eq!(tokio::fs::read(&path).await.expect("preserved or repaired shard"), original);
}
}
assert!(matches!(report.status, HealTaskStatus::Failed { .. }), "{report:?}");
let outcome = report.outcome.as_ref().expect("canonical mixed outcome");
assert_eq!(outcome.execution, HealExecutionOutcome::CompletedWithErrors);
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
assert_eq!(
(
outcome.counters.processed,
outcome.counters.healed,
outcome.counters.unchanged,
outcome.counters.failed,
outcome.counters.skipped,
outcome.counters.unknown,
outcome.counters.attempt_failures
),
(3, 1, 1, 1, 0, 0, 4),
"{outcome:?}"
);
for (object, disposition) in [
("healthy", HealObjectDisposition::VerifiedHealthy),
("repaired", HealObjectDisposition::Repaired),
("failed", HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)),
] {
let receipt = outcome
.objects
.iter()
.find(|item| item.identity.object == object)
.expect("exact object receipt");
assert_eq!(receipt.disposition, disposition);
}
drop(manager);
let restarted = HealManager::new(
storage,
Some(HealConfig {
enable_auto_heal: false,
..Default::default()
}),
);
restarted.start().await.expect("restart with retained failed batch");
let restored = restarted.get_task_report(&token).await.expect("original token after restart");
assert_eq!(restored.status, report.status);
assert_eq!(restored.outcome, report.outcome);
assert_eq!(restarted.get_queue_length().await, 0, "terminal batch must not replay");
restarted.stop().await.expect("stop restarted mixed batch manager");
});
})
.expect("mixed batch thread")
.join()
.expect("mixed batch result");
}
-18
View File
@@ -26,9 +26,6 @@ use crate::{
/// Default shard count for guard storage (must be power of 2)
const DEFAULT_GUARD_SHARD_COUNT: usize = 64;
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
const LOG_COMPONENT_LOCK: &str = "lock";
const LOG_SUBSYSTEM_NAMESPACE: &str = "namespace";
type GuardShard = Arc<RwLock<HashMap<LockId, LocalGuardEntry>>>;
type GuardStorage = Arc<Vec<GuardShard>>;
@@ -336,22 +333,7 @@ impl LockClient for LocalClient {
};
if let Some(guard) = guard {
// Guard automatically releases the lock when dropped
let physical_guard_id = (!guard.guard.is_disabled()).then(|| guard.guard.guard_id());
drop(guard.guard);
if let Some(physical_guard_id) = physical_guard_id {
tracing::trace!(
target: "rustfs_lock::namespace_owner",
event = EVENT_NAMESPACE_LOCK_RELEASE,
component = LOG_COMPONENT_LOCK,
subsystem = LOG_SUBSYSTEM_NAMESPACE,
state = "mapped",
peer_kind = "local",
lock_id = %lock_id,
process_id = std::process::id(),
physical_guard_id,
"Namespace peer lock mapped to physical guard"
);
}
Ok(true)
} else {
// Lock not found or already released
-5
View File
@@ -412,11 +412,6 @@ impl DistributedLockGuard {
&self.lock_id
}
/// Actual granted peer identities, which may differ from the logical guard ID.
pub fn granted_lock_ids(&self) -> impl Iterator<Item = &LockId> {
self.entries.iter().map(|entry| &entry.lock_id)
}
/// Whether the guard's heartbeat has observed a refresh-quorum loss.
pub fn is_lock_lost(&self) -> bool {
self.lock_lost.is_lost()
-23
View File
@@ -24,9 +24,6 @@ use std::sync::atomic::{AtomicU64, Ordering};
/// Global counter for guard IDs to prevent double-release
static GUARD_ID_COUNTER: AtomicU64 = AtomicU64::new(1);
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
const LOG_COMPONENT_LOCK: &str = "lock";
const LOG_SUBSYSTEM_NAMESPACE: &str = "namespace";
/// RAII guard for fast object locks
///
@@ -112,7 +109,6 @@ impl FastLockGuard {
record_lock_held_release(self.mode);
// Unregister the guard after successful release
shard.unregister_guard(self.guard_id);
self.record_physical_release();
}
success
} else {
@@ -122,22 +118,6 @@ impl FastLockGuard {
}
}
fn record_physical_release(&self) {
tracing::trace!(
target: "rustfs_lock::namespace_owner",
event = EVENT_NAMESPACE_LOCK_RELEASE,
component = LOG_COMPONENT_LOCK,
subsystem = LOG_SUBSYSTEM_NAMESPACE,
state = "released",
peer_kind = "local",
process_id = std::process::id(),
physical_guard_id = self.guard_id,
resource = %self.key,
owner = %self.owner,
"Namespace physical guard release completed"
);
}
/// Check if the lock has been released
pub fn is_released(&self) -> bool {
self.released
@@ -188,9 +168,6 @@ impl Drop for FastLockGuard {
}
// Always unregister the guard to prevent leaks, regardless of release success
shard.unregister_guard(self.guard_id);
if success {
self.record_physical_release();
}
} else {
// If guard was already released or disabled, just unregister it
shard.unregister_guard(self.guard_id);
+8 -10
View File
@@ -139,9 +139,15 @@ impl KeyName {
Self::COMMON_KEYS.iter().filter(|k| k.is_server_derived()).map(|k| k.name())
}
/// Returns the policy variable form of the key, e.g. `${jwt:sub}`.
pub fn var_name(&self) -> String {
format!("${{{}}}", Into::<&str>::into(self))
match self {
KeyName::Aws(s) => format!("${{aws:{}}}", Into::<&str>::into(s)),
KeyName::Jwt(s) => format!("${{jwt:{}}}", Into::<&str>::into(s)),
KeyName::Ldap(s) => format!("${{ldap:{}}}", Into::<&str>::into(s)),
KeyName::Sts(s) => format!("${{sts:{}}}", Into::<&str>::into(s)),
KeyName::Svc(s) => format!("${{svc:{}}}", Into::<&str>::into(s)),
KeyName::S3(s) => format!("${{s3:{}}}", Into::<&str>::into(s)),
}
}
}
@@ -408,14 +414,6 @@ mod tests {
);
}
#[test_case(KeyName::Aws(AwsKeyName::AWSSourceIP), "${aws:SourceIp}")]
#[test_case(KeyName::Jwt(JwtKeyName::JWTPrefUsername), "${jwt:preferred_username}")]
#[test_case(KeyName::Ldap(LdapKeyName::Username), "${ldap:username}")]
#[test_case(KeyName::S3(S3KeyName::S3LocationConstraint), "${s3:LocationConstraint}")]
fn var_name_matches_policy_variable_syntax(key: KeyName, expected: &str) {
assert_eq!(key.var_name(), expected);
}
#[test_case("s3:x-amz-copy-source", KeyName::S3(S3KeyName::S3XAmzCopySource))]
#[test_case("s3:VersionId", KeyName::S3(S3KeyName::S3VersionId) ; "aws_version_id")]
#[test_case("s3:versionid", KeyName::S3(S3KeyName::S3VersionId) ; "minio_version_id")]
+22 -54
View File
@@ -73,20 +73,6 @@ impl StringFunc {
}
}
/// Replaces every `${...}` reference to a common condition key that has a
/// non-empty value in the request, not only the first such key.
fn substitute_common_keys(mut c: String, values: &HashMap<String, Vec<String>>) -> String {
for key in KeyName::COMMON_KEYS {
if let Some(v) = values.get(key.name()).and_then(|x| x.first())
&& !v.is_empty()
{
c = c.replace(&key.var_name(), v);
}
}
c
}
impl FuncKeyValue<StringFuncValue> {
async fn eval(
&self,
@@ -124,7 +110,17 @@ impl FuncKeyValue<StringFuncValue> {
let fvalues = resolved_values
.into_iter()
.flatten()
.map(|resolved_c| Cow::from(substitute_common_keys(resolved_c, values)))
.map(|resolved_c| {
let mut c = Cow::from(resolved_c);
for key in KeyName::COMMON_KEYS {
match values.get(key.name()).and_then(|x| x.first()) {
Some(v) if !v.is_empty() => return Cow::Owned(c.to_mut().replace(&key.var_name(), v)),
_ => continue,
};
}
c
})
.map(|x| if ignore_case { Cow::Owned(x.to_lowercase()) } else { x })
.collect::<Set<_>>();
@@ -163,7 +159,17 @@ impl FuncKeyValue<StringFuncValue> {
let matched = resolved_values
.into_iter()
.flatten()
.map(|resolved_c| substitute_common_keys(resolved_c, values))
.map(|resolved_c| {
let mut c = Cow::from(resolved_c);
for key in KeyName::COMMON_KEYS {
match values.get(key.name()).and_then(|x| x.first()) {
Some(v) if !v.is_empty() => return Cow::Owned(c.to_mut().replace(&key.var_name(), v)),
_ => continue,
};
}
c
})
.any(|x| wildcard::is_match(x, v));
// Unqualified evaluation reports the plain aggregate match and lets the
@@ -383,44 +389,6 @@ mod tests {
));
}
// Real requests always carry earlier common keys such as `signatureversion`
// and `SourceIp`; substitution must still reach later keys like `jwt:*`.
#[test]
fn test_condition_values_substitute_variables_after_other_common_keys() {
let request = || {
vec![
("signatureversion", vec!["AWS4-HMAC-SHA256"]),
("SourceIp", vec!["192.0.2.10"]),
("preferred_username", vec!["alice"]),
("prefix", vec!["alice/photos/"]),
]
};
assert!(test_eval_like(
new_fkv("s3:prefix", vec!["${jwt:preferred_username}/*"]),
false,
false,
request()
));
assert!(test_eval(
new_fkv("s3:prefix", vec!["${jwt:preferred_username}/photos/"]),
false,
false,
false,
request()
));
assert!(!test_eval_like(
new_fkv("s3:prefix", vec!["${jwt:preferred_username}/*"]),
false,
false,
vec![
("signatureversion", vec!["AWS4-HMAC-SHA256"]),
("preferred_username", vec!["bob"]),
("prefix", vec!["alice/photos/"]),
]
));
}
#[test_case(new_fkv("s3:x-amz-copy-source", vec!["mybucket/myobject"]), false, vec![("x-amz-copy-source", vec!["mybucket/myobject"])] => true ; "1")]
#[test_case(new_fkv("s3:x-amz-copy-source", vec!["mybucket/myobject"]), false, vec![("x-amz-copy-source", vec!["yourbucket/myobject"])] => false ; "2")]
#[test_case(new_fkv("s3:x-amz-copy-source", vec!["mybucket/myobject"]), false, vec![] => false ; "3")]
-13
View File
@@ -352,19 +352,6 @@ mod tests {
pollster::block_on(resource.is_match(object, &HashMap::new()))
}
#[test_case("arn:aws:s3:::mybucket/${jwt:preferred_username}/*", "mybucket/alice/photo.jpg" => true; "jwt variable")]
#[test_case("arn:aws:s3:::mybucket/${jwt:preferred_username}/*", "mybucket/bob/photo.jpg" => false; "jwt variable other user")]
#[test_case("arn:aws:s3:::mybucket/${ldap:username}/*", "mybucket/alice-ldap/photo.jpg" => true; "ldap variable")]
fn test_resource_is_match_substitutes_condition_variables(resource: &str, object: &str) -> bool {
let resource: Resource = resource.try_into().unwrap();
let conditions = HashMap::from([
("preferred_username".to_string(), vec!["alice".to_string()]),
("username".to_string(), vec!["alice-ldap".to_string()]),
]);
pollster::block_on(resource.is_match(object, &conditions))
}
#[test]
fn test_wildcard_resource_serialization_round_trip() {
use crate::policy::resource::ResourceSet;
+3 -77
View File
@@ -639,7 +639,7 @@ mod hot_reload_tests {
format!("{d}BEGIN {l}{d}\n{body}\n{d}END {l}{d}\n", d = PEM_BOUNDARY_DASHES, l = PEM_OPENSSH_LABEL,)
}
pub(super) fn test_ed25519_pem() -> String {
fn test_ed25519_pem() -> String {
build_pem_block(
"b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\n\
QyNTUxOQAAACCkeMEUpnJEbOMBXiQfjZcHZMEbHW3DlNRL+Jbi1cIqMgAAAKDviRiQ74kY\n\
@@ -1010,7 +1010,7 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
fn auth_publickey_offered(
&mut self,
user: &str,
_key: russh::server::PublicKeyOffer<'_>,
_key: &keys::PublicKey,
) -> impl std::future::Future<Output = Result<Auth, Self::Error>> + Send {
async { Ok(Auth::reject()) }
}
@@ -1019,7 +1019,7 @@ impl<S: StorageBackend + Send + Sync + 'static> russh::server::Handler for SshSe
fn auth_publickey(
&mut self,
user: &str,
_key: russh::server::VerifiedPublicKey<'_>,
_key: &keys::PublicKey,
) -> impl std::future::Future<Output = Result<Auth, Self::Error>> + Send {
async { Ok(Auth::reject()) }
}
@@ -1594,80 +1594,6 @@ mod tests {
);
}
#[cfg(unix)]
#[tokio::test]
async fn ssh_handshake_rejects_public_key_authentication() {
use crate::common::dummy_storage::DummyBackend;
use russh::client::{self, AuthResult};
use russh::keys::{PrivateKeyWithHashAlg, PublicKeyOrCertificate};
struct PinnedHostKey(keys::PublicKey);
impl client::Handler for PinnedHostKey {
type Error = russh::Error;
async fn check_server_key(&mut self, offered: &PublicKeyOrCertificate) -> Result<bool, Self::Error> {
Ok(matches!(offered, PublicKeyOrCertificate::PublicKey { key, .. } if key.key_data() == self.0.key_data()))
}
}
timeout(Duration::from_secs(30), async {
let key = PrivateKey::from_openssh(super::hot_reload_tests::test_ed25519_pem()).expect("decode test SSH key");
let trusted_host_key = key.public_key().clone();
let config = build_ssh_config(vec![key.clone()], 600, "SSH-2.0-RustFS");
let peer_addr = "127.0.0.1:12345".parse().expect("test peer address");
let handler = SshSessionHandler {
storage: Arc::new(DummyBackend::new()),
peer_addr,
session_context: None,
channels: HashMap::new(),
read_only: false,
part_size: super::super::test_support::TEST_PART_SIZE,
handles_per_session: DEFAULT_HANDLES_PER_SESSION,
backend_op_timeout_secs: DEFAULT_BACKEND_OP_TIMEOUT_SECS,
read_cache_window: READ_CACHE_WINDOW_DEFAULT,
read_cache_total_mem_limit: READ_CACHE_TOTAL_MEM_DEFAULT,
read_cache_in_use: Arc::new(AtomicU64::new(0)),
session_diag: Arc::new(SessionDiag::new(peer_addr, peer_addr)),
};
let (client_stream, server_stream) = tokio::io::duplex(64 * 1024);
let server = tokio::spawn(async move {
let session = russh::server::run_stream(config, server_stream, handler)
.await
.expect("start test SSH session");
// The client deliberately disconnects after the rejected authentication.
let _ = session.await;
});
let mut client =
client::connect_stream(Arc::new(client::Config::default()), client_stream, PinnedHostKey(trusted_host_key))
.await
.expect("complete SSH handshake with the pinned host key");
let auth = client
.authenticate_publickey("public-key-auth-disabled", PrivateKeyWithHashAlg::new(Arc::new(key), None))
.await
.expect("receive public-key authentication response");
match auth {
AuthResult::Failure {
remaining_methods,
partial_success,
} => {
assert!(!partial_success, "public keys must not grant partial authentication");
assert!(remaining_methods.contains(&MethodKind::Password));
assert!(!remaining_methods.contains(&MethodKind::PublicKey));
}
AuthResult::Success => panic!("public-key authentication must remain disabled"),
}
client
.disconnect(russh::Disconnect::ByApplication, "test complete", "")
.await
.expect("disconnect test client");
drop(client);
server.await.expect("join test SSH server");
})
.await
.expect("SSH authentication rejection must complete within the test deadline");
}
#[test]
fn ssh_config_advertises_only_password() {
let config = build_ssh_config(Vec::new(), 600, "SSH-2.0-RustFS");
+34 -47
View File
@@ -1410,46 +1410,27 @@ impl DataUsageCache {
Some((flattened, visited.len()))
}
fn flatten_with_guard<'a>(
&'a self,
root: &'a DataUsageEntry,
visited: &mut HashSet<&'a str>,
depth: usize,
) -> DataUsageEntry {
// A summary never owns child links. Cloning and clearing the set
// retains its allocation in every completed subtree and update.
let mut flat = DataUsageEntry {
children: HashSet::new(),
size: root.size,
objects: root.objects,
versions: root.versions,
delete_markers: root.delete_markers,
obj_sizes: root.obj_sizes.clone(),
obj_versions: root.obj_versions.clone(),
replication_stats: root.replication_stats.clone(),
compacted: root.compacted,
failed_objects: root.failed_objects,
all_tier_stats: root.all_tier_stats.clone(),
unknown_tier_stats: root.unknown_tier_stats.clone(),
tier_accounting_proof: root.tier_accounting_proof,
};
fn flatten_with_guard(&self, root: &DataUsageEntry, visited: &mut HashSet<String>, depth: usize) -> DataUsageEntry {
let mut root = root.clone();
if depth >= MAX_DATA_USAGE_CACHE_DEPTH {
return flat;
root.children.clear();
return root;
}
for id in &root.children {
if !visited.insert(id.as_str()) {
for id in root.children.clone().iter() {
if !visited.insert(id.clone()) {
continue;
}
if let Some(e) = self.cache.get(id) {
if e.children.is_empty() {
flat.merge(e);
} else {
flat.merge(&self.flatten_with_guard(e, visited, depth + 1));
let mut e = e.clone();
if !e.children.is_empty() {
e = self.flatten_with_guard(&e, visited, depth + 1);
}
root.merge(&e);
}
}
flat
root.children.clear();
root
}
pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
@@ -1486,31 +1467,38 @@ impl DataUsageCache {
}
pub fn delete_recursive(&mut self, hash: &DataUsageHash) {
self.delete_recursive_guard(&hash.0, 0);
let mut visited = HashSet::new();
self.delete_recursive_guard(hash, &mut visited, 0);
}
fn delete_recursive_guard(&mut self, path: &str, depth: usize) {
// Removal transfers child ownership and makes cyclic/shared references
// no-ops, without cloning paths or maintaining a second visited set.
let Some(children) = self.cache.remove(path).map(|entry| entry.children) else {
fn delete_recursive_guard(&mut self, hash: &DataUsageHash, visited: &mut HashSet<String>, depth: usize) {
if !visited.insert(hash.key()) {
return;
};
}
let mut need_remove = Vec::new();
if let Some(v) = self.cache.get(&hash.string()) {
for child in v.children.iter() {
need_remove.push(child.clone());
}
}
self.cache.remove(&hash.string());
if depth >= MAX_DATA_USAGE_CACHE_DEPTH {
return;
}
for child in children {
self.delete_recursive_guard(&child, depth + 1);
for child in need_remove {
self.delete_recursive_guard(&DataUsageHash(child), visited, depth + 1);
}
}
pub fn size_recursive(&self, path: &str) -> Option<DataUsageEntry> {
match self.cache.get_key_value(&hash_path(path).key()) {
Some((key, root)) => {
match self.find(path) {
Some(root) => {
if root.children.is_empty() {
return Some(root.clone());
}
let mut visited = HashSet::new();
visited.insert(key.as_str());
visited.insert(hash_path(path).key());
let mut flat = self.flatten_with_guard(root, &mut visited, 0);
if flat.replication_stats.as_ref().is_some_and(|stats| stats.is_empty()) {
flat.replication_stats = None;
@@ -1565,7 +1553,7 @@ impl DataUsageCache {
}
pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
let Some(e) = self.cache.get(&path.0) else {
let Some(e) = self.cache.get(&path.key()).cloned() else {
return;
};
@@ -1617,13 +1605,12 @@ impl DataUsageCache {
}
pub fn total_children_rec(&self, path: &str) -> usize {
let root = hash_path(path);
let mut visited = HashSet::new();
visited.insert(root.0.as_str());
visited.insert(hash_path(path).key());
self.total_children_rec_guard(path, &mut visited, 0)
}
fn total_children_rec_guard<'a>(&'a self, path: &str, visited: &mut HashSet<&'a str>, depth: usize) -> usize {
fn total_children_rec_guard(&self, path: &str, visited: &mut HashSet<String>, depth: usize) -> usize {
let Some(root) = self.find(path) else {
return 0;
};
@@ -1633,7 +1620,7 @@ impl DataUsageCache {
let mut n = 0;
for ch in root.children.iter() {
if visited.insert(ch.as_str()) {
if visited.insert(ch.clone()) {
n += 1 + self.total_children_rec_guard(ch, visited, depth + 1);
}
}
@@ -2428,69 +2428,6 @@ fn test_data_usage_cache_recursive_helpers_tolerate_cycles() {
assert!(!copied.cache.contains_key(&child_hash.key()));
}
#[test]
fn delete_recursive_preserves_depth_boundary_and_unrelated_entries() {
let mut cache = DataUsageCache::default();
for depth in 0..=MAX_DATA_USAGE_CACHE_DEPTH + 1 {
let mut entry = DataUsageEntry::default();
if depth <= MAX_DATA_USAGE_CACHE_DEPTH {
entry.children.insert(format!("node-{}", depth + 1));
}
cache.cache.insert(format!("node-{depth}"), entry);
}
cache.cache.insert("unrelated".to_string(), DataUsageEntry::default());
cache.delete_recursive(&DataUsageHash("node-0".to_string()));
assert_eq!(cache.cache.len(), 2, "delete only reachable entries within the depth limit");
assert!(cache.cache.contains_key(&format!("node-{}", MAX_DATA_USAGE_CACHE_DEPTH + 1)));
assert!(cache.cache.contains_key("unrelated"));
}
#[test]
fn scanner_summary_releases_wide_child_sets() {
let root = hash_path("bucket");
let mut cache = DataUsageCache::default();
cache.replace_hashed(
&root,
&None,
&DataUsageEntry {
size: 17,
objects: 1,
versions: 2,
delete_markers: 1,
failed_objects: 1,
all_tier_stats: Some(AllTierStats::new()),
..Default::default()
},
);
for index in 0..10_000 {
let child = hash_path(&format!("bucket/{index:05}"));
cache.replace_hashed(
&child,
&Some(root.clone()),
&DataUsageEntry {
size: 3,
objects: 1,
versions: 2,
..Default::default()
},
);
}
let summary = cache.size_recursive("bucket").expect("wide bucket summary");
assert_eq!((summary.size, summary.objects, summary.versions), (30_017, 10_001, 20_002));
assert_eq!((summary.delete_markers, summary.failed_objects), (1, 1));
assert_eq!(summary.children.capacity(), 0, "summaries must not retain the wide namespace allocation");
assert!(summary.all_tier_stats.is_some(), "preserve the root's optional statistics representation");
assert_eq!(cache.find("bucket").expect("unchanged root").children.len(), 10_000);
cache.cache.insert("unrelated".to_string(), DataUsageEntry::default());
cache.delete_recursive(&root);
assert_eq!(cache.cache.len(), 1, "wide subtree deletion must consume every child exactly once");
assert!(cache.cache.contains_key("unrelated"));
}
#[test]
fn test_data_usage_cache_flatten_does_not_count_root_twice_in_cycle() {
let root_hash = hash_path("bucket");
@@ -236,7 +236,6 @@ async fn profile_case(objects: usize, scenario: &str, samples: usize) {
let mut clone_ns = Vec::new();
let mut copy_ns = Vec::new();
let mut delete_ns = Vec::new();
let mut flatten_ns = Vec::new();
let mut encode_ns = Vec::new();
let mut save_ns = Vec::new();
@@ -258,14 +257,6 @@ async fn profile_case(objects: usize, scenario: &str, samples: usize) {
same_cache(&copied, &cache);
drop(copied);
let mut deleted = cache.clone();
let root = hash_path("bucket");
let started = WallInstant::now();
deleted.delete_recursive(black_box(&root));
delete_ns.push(elapsed_ns(started));
assert!(deleted.cache.is_empty(), "delete the root and every reachable entry");
drop(deleted);
let started = WallInstant::now();
let aggregate = black_box(cache.checked_flatten("bucket").expect("valid fixture tree"));
flatten_ns.push(elapsed_ns(started));
@@ -335,7 +326,6 @@ async fn profile_case(objects: usize, scenario: &str, samples: usize) {
"changed_entry_wire_bytes": changed_entry_wire_bytes, "save_body_bytes_per_sample": expected_wire.len() * 2,
"snapshot_complete": cache.info.snapshot_complete,
"clone": quantiles(clone_ns), "copy_with_children": quantiles(copy_ns), "checked_flatten": quantiles(flatten_ns),
"delete_recursive": quantiles(delete_ns),
"encode": quantiles(encode_ns), "save_inclusive": quantiles(save_ns), "memory_backend_ingest": quantiles(ingest_ns),
})
);
+5 -98
View File
@@ -28,7 +28,7 @@ use crate::storage_api::owner::{
EcstoreSetDisks, EcstoreStore, ecstore_is_reserved_or_invalid_bucket, ecstore_resolve_object_store_handle,
};
use futures::future::join_all;
use rustfs_data_usage::{PrefixUsageEntry, PrefixUsageQuery, PrefixUsageSummary};
use rustfs_data_usage::{PrefixUsageEntry, PrefixUsageSummary};
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime};
@@ -133,7 +133,6 @@ async fn compute_prefix_usage(
) -> BucketPrefixUsageResponse {
let sets: Vec<Arc<EcstoreSetDisks>> = store.all_set_disks();
let sets_total = sets.len();
let per_set_max_entries = if sets_total <= 1 { max_entries } else { usize::MAX };
let cache_name = format!("{bucket}/{DATA_USAGE_CACHE_NAME}");
let per_set = join_all(sets.into_iter().map(|set| {
@@ -176,25 +175,12 @@ async fn compute_prefix_usage(
return None;
}
let last_update = loaded.info.last_update;
// Fetch all sub-prefixes per set when sets_total > 1 so cross-set
// aggregation merges every set's contribution before truncating
// to the caller's max_entries bound.
let query = loaded.prefix_usage(bucket, prefix, per_set_max_entries);
let query = loaded.prefix_usage(bucket, prefix, max_entries);
Some((query, last_update))
}
}))
.await;
aggregate_set_prefix_usage(bucket, prefix, max_entries, sets_total, per_set.into_iter().flatten())
}
fn aggregate_set_prefix_usage(
bucket: &str,
prefix: &str,
max_entries: usize,
sets_total: usize,
per_set: impl IntoIterator<Item = (Option<PrefixUsageQuery>, Option<SystemTime>)>,
) -> BucketPrefixUsageResponse {
let mut usage = PrefixUsageSummary::default();
let mut sub_prefix_map: HashMap<String, PrefixUsageSummary> = HashMap::new();
let mut sets_reporting = 0usize;
@@ -204,7 +190,7 @@ fn aggregate_set_prefix_usage(
let mut truncated = false;
let mut last_update: Option<SystemTime> = None;
for (query, set_last_update) in per_set {
for (query, set_last_update) in per_set.into_iter().flatten() {
// last_update counts every set that has scanned the bucket, even
// when the prefix itself is absent on that set.
if let Some(set_last_update) = set_last_update
@@ -243,7 +229,8 @@ fn aggregate_set_prefix_usage(
.cmp(&left.usage.size)
.then_with(|| left.prefix.cmp(&right.prefix))
});
// Enforce the caller bound on the merged cross-set breakdown.
// Merged rows can exceed max_entries only when per-set truncation
// already flagged; enforce the caller bound on the merged view too.
if sub_prefixes.len() > max_entries {
truncated = true;
sub_prefixes.truncate(max_entries);
@@ -359,84 +346,4 @@ mod tests {
let map = guard.as_ref().expect("seeded");
assert!(map.len() <= CACHE_MAX_ENTRIES, "cache must stay bounded, got {}", map.len());
}
#[test]
fn multi_set_aggregation_merges_untruncated_set_breakdowns_before_applying_max_entries() {
use crate::data_usage_define::{DataUsageCache, DataUsageEntry};
let mut set0 = DataUsageCache::default();
set0.info.name = "bucket".to_string();
set0.replace(
"bucket",
"",
DataUsageEntry {
size: 190,
objects: 2,
versions: 2,
..Default::default()
},
);
set0.replace(
"bucket/a",
"bucket",
DataUsageEntry {
size: 100,
objects: 1,
versions: 1,
..Default::default()
},
);
set0.replace(
"bucket/b",
"bucket",
DataUsageEntry {
size: 90,
objects: 1,
versions: 1,
..Default::default()
},
);
let mut set1 = DataUsageCache::default();
set1.info.name = "bucket".to_string();
set1.replace(
"bucket",
"",
DataUsageEntry {
size: 190,
objects: 2,
versions: 2,
..Default::default()
},
);
set1.replace(
"bucket/c",
"bucket",
DataUsageEntry {
size: 100,
objects: 1,
versions: 1,
..Default::default()
},
);
set1.replace(
"bucket/b",
"bucket",
DataUsageEntry {
size: 90,
objects: 1,
versions: 1,
..Default::default()
},
);
let q0 = set0.prefix_usage("bucket", "", usize::MAX);
let q1 = set1.prefix_usage("bucket", "", usize::MAX);
let merged = super::aggregate_set_prefix_usage("bucket", "", 1, 2, vec![(q0, None), (q1, None)]);
assert!(merged.truncated);
assert_eq!(merged.sub_prefixes.len(), 1);
assert_eq!(merged.sub_prefixes[0].prefix, "b");
assert_eq!(merged.sub_prefixes[0].usage.size, 180);
assert_eq!(merged.sub_prefixes[0].usage.objects, 2);
}
}
+2 -2
View File
@@ -5440,8 +5440,8 @@ async fn scanner_legacy_usage_backup_survives_fencing_and_restart_after_real_met
.await
.expect_err("truncated primary must fail in the real object reader");
assert!(
error.to_string().contains("InlineData value out of range"),
"unexpected truncated-primary error: {error}"
matches!(error, EcstoreError::FileCorrupt),
"unexpected truncated-primary error: {error:?}"
);
assert_eq!(
read_config_with_revision(store.clone(), &backup_path)
+169 -180
View File
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use std::fs::FileType;
use std::io::ErrorKind;
@@ -21,10 +20,10 @@ use std::time::{Duration, Instant, SystemTime};
use crate::ReplTargetSizeSummary;
use crate::data_usage_define::{
DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageEntry, DataUsageHash, DataUsageHashMap,
DataUsageRawEnumerationCursor, DataUsageScanCheckpoint, DataUsageScanCheckpointReason, MAX_DATA_USAGE_CACHE_DEPTH,
PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry, SizeSummary, hash_path,
scan_frontier_covers_path,
DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCache, DataUsageCacheInfo, DataUsageCacheSource, DataUsageEntry, DataUsageHash,
DataUsageHashMap, DataUsageRawEnumerationCursor, DataUsageScanCheckpoint, DataUsageScanCheckpointReason,
MAX_DATA_USAGE_CACHE_DEPTH, PendingScannerHeal, PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry,
SizeSummary, hash_path, scan_frontier_covers_path,
};
use crate::error::ScannerError;
use crate::raw_page_index::{RawEnumerationPageIndex, RawEnumerationPageWriter};
@@ -752,36 +751,11 @@ struct ScannerCheckpointAncestor {
entry: DataUsageEntry,
}
struct ScannerUsageUpdates {
sender: mpsc::Sender<DataUsageEntry>,
cache: DataUsageCache,
last_update: SystemTime,
}
impl ScannerUsageUpdates {
fn new(sender: mpsc::Sender<DataUsageEntry>, bucket: String) -> Self {
Self {
sender,
cache: DataUsageCache {
info: DataUsageCacheInfo {
name: bucket,
..Default::default()
},
..Default::default()
},
last_update: SystemTime::UNIX_EPOCH,
}
}
fn is_due(&self) -> bool {
self.last_update.elapsed().unwrap_or(Duration::ZERO) >= Duration::from_secs(60)
}
}
pub struct FolderScanner {
root: String,
old_cache: DataUsageCache,
new_cache: DataUsageCache,
update_cache: DataUsageCache,
data_usage_scanner_debug: bool,
heal_object_select: u32,
@@ -799,10 +773,11 @@ pub struct FolderScanner {
disks: Vec<Arc<Disk>>,
disks_quorum: usize,
usage_updates: Option<ScannerUsageUpdates>,
updates: Option<mpsc::Sender<DataUsageEntry>>,
checkpoint_tx: Option<mpsc::Sender<DataUsageCache>>,
checkpoint_ancestors: Vec<ScannerCheckpointAncestor>,
checkpoint_depth: usize,
last_update: SystemTime,
checkpoint_objects: u64,
object_metadata_visits: metrics::Counter,
last_checkpoint_objects: u64,
@@ -823,6 +798,10 @@ pub struct FolderScanner {
resume_failed_paths: Vec<String>,
coverage_gap: bool,
raw_enumeration_progress: Vec<RawEnumerationProgress>,
pending_heal_sync_deferred: bool,
pending_heal_batch_dirty: bool,
#[cfg(test)]
pending_heal_sync_count: usize,
pending_size_reconciliation_keys: HashSet<String>,
pending_size_reconciliation_scopes: HashSet<String>,
pending_size_reconciliation_truncated: bool,
@@ -1057,9 +1036,8 @@ impl FolderScanner {
fn reuse_clean_root_child(&mut self, child: &CachedFolder, child_hash: &DataUsageHash, into: &mut DataUsageEntry) {
self.new_cache.copy_with_children(&self.old_cache, child_hash, &child.parent);
if let Some(updates) = &mut self.usage_updates {
updates.cache.copy_with_children(&self.old_cache, child_hash, &child.parent);
}
self.update_cache
.copy_with_children(&self.old_cache, child_hash, &child.parent);
into.add_child(child_hash);
}
@@ -1167,8 +1145,8 @@ impl FolderScanner {
}
/// Apply the per-object size-resolution ledger updates in one place. The
/// scanner cache is the durable boundary; progress updates contain only
/// usage totals and never own a copy of the reconciliation ledger.
/// scanner cache is the durable boundary; both working copies are updated
/// so an incremental publication cannot lose a debt or its resolution.
fn apply_size_reconciliation(&mut self, summary: &SizeSummary) {
let now = Self::now_secs();
self.pending_size_reconciliation_keys
@@ -1181,25 +1159,26 @@ impl FolderScanner {
);
self.pending_size_reconciliation_truncated |= summary.size_reconciliation_truncated;
let info = &mut self.new_cache.info;
for incoming in &summary.size_reconciliation {
if let Some(existing) = info.size_reconciliation.get_mut(&incoming.key) {
existing.reason = incoming.reason.clone();
existing.physical_size = incoming.physical_size;
existing.generation = incoming.generation.clone();
existing.version_id = incoming.version_id.clone();
existing.attempts = existing.attempts.saturating_add(1);
continue;
}
for info in [&mut self.new_cache.info, &mut self.update_cache.info] {
for incoming in &summary.size_reconciliation {
if let Some(existing) = info.size_reconciliation.get_mut(&incoming.key) {
existing.reason = incoming.reason.clone();
existing.physical_size = incoming.physical_size;
existing.generation = incoming.generation.clone();
existing.version_id = incoming.version_id.clone();
existing.attempts = existing.attempts.saturating_add(1);
continue;
}
if size_reconciliation_entry_bytes(incoming) > MAX_SIZE_RECONCILIATION_BYTES_PER_BUCKET {
continue;
}
if size_reconciliation_entry_bytes(incoming) > MAX_SIZE_RECONCILIATION_BYTES_PER_BUCKET {
continue;
}
let mut entry = incoming.clone();
entry.first_seen = now;
entry.attempts = 1;
info.size_reconciliation.insert(entry.key.clone(), entry);
let mut entry = incoming.clone();
entry.first_seen = now;
entry.attempts = 1;
info.size_reconciliation.insert(entry.key.clone(), entry);
}
}
}
@@ -1209,24 +1188,27 @@ impl FolderScanner {
let scopes = std::mem::take(&mut self.pending_size_reconciliation_scopes);
let truncated = std::mem::replace(&mut self.pending_size_reconciliation_truncated, false);
let info = &mut self.new_cache.info;
if !truncated {
info.size_reconciliation.retain(|key, entry| {
!scopes.contains(&size_reconciliation_scope_key(&entry.bucket, &entry.object)) || current_keys.contains(key)
});
for info in [&mut self.new_cache.info, &mut self.update_cache.info] {
if !truncated {
info.size_reconciliation.retain(|key, entry| {
!scopes.contains(&size_reconciliation_scope_key(&entry.bucket, &entry.object)) || current_keys.contains(key)
});
}
prune_size_reconciliation(info, now);
}
prune_size_reconciliation(info, now);
}
fn record_scan_resume_hint(&mut self, folder: &str) {
self.new_cache.info.scan_resume_after = Some(folder.to_string());
self.update_cache.info.scan_resume_after = Some(folder.to_string());
let checkpoint = DataUsageScanCheckpoint::new(folder.to_string(), DataUsageScanCheckpointReason::Unknown);
global_metrics().record_scanner_checkpoint_set(
checkpoint.version,
checkpoint.resume_after.clone(),
checkpoint.reason.as_str(),
);
self.new_cache.info.scan_checkpoint = Some(checkpoint);
self.new_cache.info.scan_checkpoint = Some(checkpoint.clone());
self.update_cache.info.scan_checkpoint = Some(checkpoint);
}
fn record_completed_child(&mut self, folder: &str) {
@@ -1529,7 +1511,8 @@ impl FolderScanner {
self.record_scan_resume_hint_if_not_ancestor(&child_hash.key());
parent_entry.add_child(child_hash);
self.new_cache.replace_hashed(child_hash, parent, &child_entry);
self.refresh_usage_subtree(child_hash, parent);
self.update_cache.delete_recursive(child_hash);
self.update_cache.copy_with_children(&self.new_cache, child_hash, parent);
self.send_update().await;
}
}
@@ -1603,7 +1586,7 @@ impl FolderScanner {
let candidate_type = pending_scanner_heal_candidate_type(kind);
let priority = request.priority;
let scan_mode = request.scan_mode.unwrap_or(self.scan_mode);
let result = match send_scanner_heal_request(candidate_type, request).await {
let result = match send_scanner_heal_request(candidate_type, request, scanner_disk_heal_scope(&self.local_disk)).await {
Ok(result) => result,
Err(err) => {
self.update_pending_scanner_heal_after_admission(
@@ -1672,75 +1655,29 @@ impl FolderScanner {
self.data_usage_scanner_debug = debug;
}
fn compact_completed_folder(&mut self, folder: &CachedFolder, hash: &DataUsageHash, entry: &DataUsageEntry) {
// Completed healthy subtrees can keep aggregate usage. Coverage gaps
// and failed-object debt still need child entries for precise resume
// and to avoid rereading healthy siblings on a retry.
let forward_sweep = self.old_cache.info.scan_progress.is_some();
if entry.compacted || self.new_cache.info.name == folder.name || (forward_sweep && self.coverage_gap) {
return;
}
let summary = if entry.children.is_empty() {
Cow::Borrowed(entry)
} else if let Some(flat) = self.new_cache.size_recursive(&hash.0) {
Cow::Owned(flat)
} else {
return;
};
if forward_sweep && summary.failed_objects != 0 {
return;
}
let should_compact = summary.objects < DATA_SCANNER_COMPACT_LEAST_OBJECT
|| entry.children.iter().all(|key| {
self.new_cache
.cache
.get(key)
.is_none_or(|child| child.children.is_empty() && child.objects <= 1)
});
if !should_compact {
return;
}
// Heal revisits also reach this path without a parent advancing
// coverage. Keep the endpoint addressable when its completed
// subtree is replaced by an aggregate.
if self.coverage_frontier.as_deref().is_some_and(|frontier| {
frontier
.strip_prefix(&folder.name)
.is_some_and(|suffix| suffix.starts_with('/'))
}) {
self.record_completed_child(&folder.name);
}
match summary {
Cow::Borrowed(_) => {
// The leaf is already cached; only its sampling mode changes.
if let Some(cached) = self.new_cache.cache.get_mut(&hash.0) {
cached.compacted = true;
}
}
Cow::Owned(mut flat) => {
flat.compacted = true;
self.new_cache.delete_recursive(hash);
self.new_cache.replace_hashed(hash, &folder.parent, &flat);
}
/// Send update if enough time has passed
/// Should be called on a regular basis when the new_cache contains more recent total than previously.
/// May or may not send an update upstream.
fn should_send_update(&self) -> bool {
if self.updates.is_none() {
return false;
}
let elapsed = self.last_update.elapsed().unwrap_or(Duration::from_secs(0));
elapsed >= Duration::from_secs(60)
}
fn refresh_usage_subtree(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
if let Some(updates) = &mut self.usage_updates {
updates.cache.delete_recursive(hash);
updates.cache.copy_with_children(&self.new_cache, hash, parent);
}
}
/// Send at most one aggregate per minute when a caller requested progress.
pub async fn send_update(&mut self) {
let Some(updates) = &mut self.usage_updates else { return };
if !updates.is_due() {
// Send at most an update every minute.
if !self.should_send_update() {
return;
}
if let Some(flat) = updates.cache.size_recursive(&self.new_cache.info.name) {
if let Err(e) = updates.sender.send(flat).await {
if let Some(flat) = self.update_cache.size_recursive(&self.new_cache.info.name)
&& let Some(ref updates) = self.updates
{
// Try to send without blocking
if let Err(e) = updates.send(flat).await {
error!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_FOLDER_STATE,
@@ -1752,17 +1689,17 @@ impl FolderScanner {
"Scanner folder update send failed"
);
}
updates.last_update = SystemTime::now();
self.last_update = SystemTime::now();
}
}
async fn send_update_for_entry(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>, entry: &DataUsageEntry) {
if let Some(updates) = &mut self.usage_updates
&& updates.is_due()
{
updates.cache.replace_hashed(hash, parent, entry);
self.send_update().await;
if !self.should_send_update() {
return;
}
self.update_cache.replace_hashed(hash, parent, entry);
self.send_update().await;
}
async fn record_missing_erasure_metadata(
@@ -1901,7 +1838,8 @@ impl FolderScanner {
wait_for_scanner_io(&ctx, self.sleeper.sleep_folder()).await?;
}
let mut queued_folders: Vec<QueuedFolder> = Vec::new();
let mut existing_folders: Vec<CachedFolder> = Vec::new();
let mut new_folders: Vec<CachedFolder> = Vec::new();
let mut found_object_metadata = false;
let mut object_count: u64 = 0;
let yield_every_objects = scanner_yield_every_n_objects();
@@ -2209,21 +2147,12 @@ impl FolderScanner {
self.reuse_clean_root_child(&this, &h, into);
} else if exists {
abandoned_children.remove(&h.key());
queued_folders.push(QueuedFolder {
folder: this,
source: FolderScanSource::Existing,
});
if let Some(updates) = &mut self.usage_updates {
updates
.cache
.copy_with_children(&self.old_cache, &h, &Some(this_hash.clone()));
}
existing_folders.push(this);
self.update_cache
.copy_with_children(&self.old_cache, &h, &Some(this_hash.clone()));
} else {
abandoned_children.remove(&h.key());
queued_folders.push(QueuedFolder {
folder: this,
source: FolderScanSource::New,
});
new_folders.push(this);
}
continue;
}
@@ -2249,6 +2178,7 @@ impl FolderScanner {
lifecycle: active_life_cycle.clone(),
object_lock: active_object_lock.clone(),
replication: active_replication.clone(),
heal_scope: None,
heal_enabled,
heal_bitrot: self.scan_mode == HealScanMode::Deep,
debug: self.data_usage_scanner_debug,
@@ -2445,17 +2375,17 @@ impl FolderScanner {
// frontier. Keep the emergency compaction limit for large trees.
let should_compact = (self.new_cache.info.name != folder.name
&& self.old_cache.info.scan_progress.is_none()
&& queued_folders.len() >= DATA_SCANNER_COMPACT_AT_FOLDERS)
|| queued_folders.len() >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS;
&& existing_folders.len() + new_folders.len() >= DATA_SCANNER_COMPACT_AT_FOLDERS)
|| existing_folders.len() + new_folders.len() >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS;
let total_folders = queued_folders.len();
let total_folders = existing_folders.len() + new_folders.len();
self.alert_excessive_folders(&folder.name, total_folders);
if !into.compacted && should_compact {
into.compacted = true;
for queued in &mut queued_folders {
queued.source = FolderScanSource::New;
}
new_folders.append(&mut existing_folders);
existing_folders.clear();
if self.data_usage_scanner_debug {
debug!(
@@ -2464,13 +2394,20 @@ impl FolderScanner {
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
folder = %folder.name,
entry_count = queued_folders.len(),
entry_count = new_folders.len(),
state = "preemptive_compaction",
"Scanner folder switched to compacted mode"
);
}
}
if !into.compacted {
for folder_item in &existing_folders {
let h = hash_path(&folder_item.name);
self.update_cache.copy_with_children(&self.old_cache, &h, &folder_item.parent);
}
}
let is_scan_root = folder.name == self.old_cache.info.name;
let scan_checkpoint = self.old_cache.info.scan_checkpoint.as_ref();
let checkpoint_resume_after = scan_checkpoint.and_then(|checkpoint| {
@@ -2494,6 +2431,15 @@ impl FolderScanner {
.and_then(|resume_after| folder_resume_match(&folder.name, resume_after))
.is_some_and(|resume_match| matches!(resume_match, FolderResumeMatch::Descendant));
let scan_resume_after = checkpoint_resume_after.or(self.old_cache.info.scan_resume_after.as_deref());
let mut queued_folders = Vec::with_capacity(new_folders.len() + existing_folders.len());
queued_folders.extend(new_folders.into_iter().map(|folder| QueuedFolder {
folder,
source: FolderScanSource::New,
}));
queued_folders.extend(existing_folders.into_iter().map(|folder| QueuedFolder {
folder,
source: FolderScanSource::Existing,
}));
let has_queued_folders = !queued_folders.is_empty();
let forward_sweep = self.old_cache.info.scan_progress.is_some();
let resume_order = if forward_sweep {
@@ -2534,21 +2480,19 @@ impl FolderScanner {
match queued_folder.source {
FolderScanSource::New => {
// Add new folders to the update tree so totals update for these.
if !into.compacted
&& let Some(updates) = &mut self.usage_updates
{
if !into.compacted {
let mut found_any = false;
let mut parent = this_hash.clone();
let update_root_hash = hash_path(&updates.cache.info.name);
let update_cache_name_hash = hash_path(&self.update_cache.info.name);
while parent != update_root_hash {
while parent != update_cache_name_hash {
let parent_key = parent.key();
let e = updates.cache.find(&parent_key);
let e = self.update_cache.find(&parent_key);
if e.is_none_or(|v| v.compacted) {
found_any = true;
break;
}
if let Some(next) = updates.cache.search_parent(&parent) {
if let Some(next) = self.update_cache.search_parent(&parent) {
parent = next;
} else {
found_any = true;
@@ -2557,8 +2501,7 @@ impl FolderScanner {
}
if !found_any {
// Add non-compacted empty entry.
updates
.cache
self.update_cache
.replace_hashed(&h, &Some(this_hash.clone()), &DataUsageEntry::default());
}
}
@@ -2632,17 +2575,13 @@ impl FolderScanner {
};
if outcome == FolderScanOutcome::Vanished {
self.new_cache.delete_recursive(&h);
if let Some(updates) = &mut self.usage_updates {
updates.cache.delete_recursive(&h);
}
self.update_cache.delete_recursive(&h);
into.children.remove(&h.key());
continue;
}
if outcome == FolderScanOutcome::ErasureData {
self.new_cache.delete_recursive(&h);
if let Some(updates) = &mut self.usage_updates {
updates.cache.delete_recursive(&h);
}
self.update_cache.delete_recursive(&h);
into.children.remove(&h.key());
// Earlier namespace siblings may have completed, but none
// can certify coverage of this unresolved parent object.
@@ -2662,13 +2601,23 @@ impl FolderScanner {
self.send_update_for_entry(&this_hash, &folder.parent, into).await;
tokio::task::yield_now().await;
} else {
tokio::task::yield_now().await;
into.add_child(&h);
self.record_completed_child(&folder_item.name);
tokio::task::yield_now().await;
self.maybe_send_checkpoint(&folder, &this_hash, into);
self.refresh_usage_subtree(&h, &folder_item.parent);
self.update_cache.delete_recursive(&h);
self.update_cache.copy_with_children(&self.new_cache, &h, &folder_item.parent);
self.send_update().await;
}
if queued_folder.source == FolderScanSource::New
&& !into.compacted
&& self.update_cache.find(&this_hash.key()).is_some_and(|v| !v.compacted)
{
self.update_cache.delete_recursive(&h);
self.update_cache
.copy_with_children(&self.new_cache, &h, &Some(this_hash.clone()));
}
}
// Scan for healing
@@ -3065,7 +3014,8 @@ impl FolderScanner {
into.add_child(&h);
// We scanned a folder, optionally send update.
self.refresh_usage_subtree(&h, &folder_item.parent);
self.update_cache.delete_recursive(&h);
self.update_cache.copy_with_children(&self.new_cache, &h, &folder_item.parent);
self.send_update().await;
self.maybe_send_checkpoint(&folder, &this_hash, into);
}
@@ -3079,7 +3029,36 @@ impl FolderScanner {
self.new_cache.replace_hashed(&this_hash, &folder.parent, into);
}
self.compact_completed_folder(&folder, &this_hash, into);
// Keep independently accounted children while the sweep cursor may
// reference them; the hard cardinality compaction below still applies.
if !into.compacted
&& self.old_cache.info.scan_progress.is_none()
&& self.new_cache.info.name != folder.name
&& let Some(mut flat) = self.new_cache.size_recursive(&this_hash.key())
{
flat.compacted = true;
let mut should_compact = false;
if flat.objects < DATA_SCANNER_COMPACT_LEAST_OBJECT {
should_compact = true;
} else {
// Compact if we only have objects as children...
should_compact = true;
for k in &into.children {
if let Some(v) = self.new_cache.cache.get(k)
&& (!v.children.is_empty() || v.objects > 1)
{
should_compact = false;
break;
}
}
}
if should_compact {
self.new_cache.delete_recursive(&this_hash);
self.new_cache.replace_hashed(&this_hash, &folder.parent, &flat);
}
}
// Compact if too many children...
if !into.compacted {
@@ -3092,13 +3071,12 @@ impl FolderScanner {
done_compact();
}
if !was_compacted
&& let Some(updates) = &mut self.usage_updates
&& updates.cache.cache.contains_key(&this_hash.key())
&& let Some(flat) = self.new_cache.size_recursive(&this_hash.key())
{
updates.cache.delete_recursive(&this_hash);
updates.cache.replace_hashed(&this_hash, &folder.parent, &flat);
if self.update_cache.cache.contains_key(&this_hash.key()) && !was_compacted {
// Replace if existed before.
if let Some(flat) = self.new_cache.size_recursive(&this_hash.key()) {
self.update_cache.delete_recursive(&this_hash);
self.update_cache.replace_hashed(&this_hash, &folder.parent, &flat);
}
}
self.finish_size_reconciliation_batch();
@@ -3196,7 +3174,10 @@ pub(crate) async fn scan_data_folder_scoped(
info: cache.info.clone(),
..Default::default()
};
let usage_updates = updates.map(|sender| ScannerUsageUpdates::new(sender, bucket_name.clone()));
let update_cache = DataUsageCache {
info: cache.info.clone(),
..Default::default()
};
let resume_failed_paths = if resume_frontier.is_some() {
failed_scan_paths(&cache)
} else {
@@ -3207,6 +3188,7 @@ pub(crate) async fn scan_data_folder_scoped(
root: base_path,
old_cache: cache,
new_cache,
update_cache,
data_usage_scanner_debug: false,
heal_object_select,
scan_mode,
@@ -3219,10 +3201,11 @@ pub(crate) async fn scan_data_folder_scoped(
sleeper,
disks,
disks_quorum,
usage_updates,
updates,
checkpoint_tx,
checkpoint_ancestors: Vec::new(),
checkpoint_depth: 0,
last_update: SystemTime::UNIX_EPOCH,
checkpoint_objects: 0,
object_metadata_visits,
last_checkpoint_objects: 0,
@@ -3238,7 +3221,11 @@ pub(crate) async fn scan_data_folder_scoped(
resume_frontier,
resume_failed_paths,
coverage_gap: false,
pending_heal_sync_deferred: false,
pending_heal_batch_dirty: false,
raw_enumeration_progress: Vec::new(),
#[cfg(test)]
pending_heal_sync_count: 0,
pending_size_reconciliation_keys: HashSet::new(),
pending_size_reconciliation_scopes: HashSet::new(),
pending_size_reconciliation_truncated: false,
@@ -3248,6 +3235,7 @@ pub(crate) async fn scan_data_folder_scoped(
let now = FolderScanner::now_secs();
prune_size_reconciliation(&mut scanner.new_cache.info, now);
prune_size_reconciliation(&mut scanner.update_cache.info, now);
// Check if context is cancelled
if ctx.is_cancelled() {
@@ -3418,6 +3406,7 @@ pub(crate) async fn scan_data_folder_scoped(
mod item_actions;
mod ledger;
pub(crate) use item_actions::scanner_disk_heal_scope;
use item_actions::*;
pub use item_actions::{GetSizeFn, ScannerItem};
use ledger::*;
@@ -474,10 +474,23 @@ pub(super) fn record_heal_candidate_admission(
}
}
pub(crate) fn scanner_disk_heal_scope(disk: &Disk) -> Option<DataUsageCacheSource> {
let location = disk.get_disk_location();
location.valid().then_some(())?;
Some(DataUsageCacheSource::new(location.pool_idx?, location.set_idx?))
}
pub(super) async fn send_scanner_heal_request(
candidate_type: &'static str,
request: HealChannelRequest,
mut request: HealChannelRequest,
scope: Option<DataUsageCacheSource>,
) -> Result<HealAdmissionResult, ScannerError> {
if request.object_prefix.as_deref().is_some_and(|object| !object.is_empty()) {
let scope = scope
.ok_or_else(|| ScannerError::Other("scanner object heal requires the scanning disk's pool and set".to_string()))?;
request.pool_index = Some(scope.pool_index);
request.set_index = Some(scope.set_index);
}
let priority = request.priority;
let trace_context = scanner_heal_candidate_trace_context(&request);
let candidate = crate::heal_completion::prepare(&request);
@@ -537,6 +550,8 @@ pub struct ScannerItem {
pub lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
pub object_lock: Option<Arc<ObjectLockConfiguration>>,
pub replication: Option<Arc<ReplicationConfig>>,
/// Actual scanning-disk location; refreshed by the disk reader before actions.
pub heal_scope: Option<DataUsageCacheSource>,
pub heal_enabled: bool,
pub heal_bitrot: bool,
pub debug: bool,
@@ -1161,6 +1176,7 @@ impl ScannerItem {
scan_mode,
HealChannelPriority::Low,
),
self.heal_scope,
)
.await;
@@ -1322,6 +1338,7 @@ mod tests {
lifecycle: None,
object_lock: None,
replication: None,
heal_scope: None,
heal_enabled: false,
heal_bitrot: false,
debug: false,

Some files were not shown because too many files have changed in this diff Show More