From fa49f0ee4f29a60740ebec00e3259106b597c409 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 19 Aug 2026 10:59:54 +0800 Subject: [PATCH 01/27] test(crypto): replace the one-file key scan with a repo-wide guard (#6246) --- .config/make/lint-fmt.mak | 5 + .config/make/pre-commit.mak | 6 +- .config/make/tests.mak | 1 + .github/workflows/ci-docs-only.yml | 3 + .github/workflows/ci.yml | 3 + crates/crypto/src/license_token.rs | 8 - scripts/README.md | 1 + scripts/check_embedded_secrets.sh | 279 +++++++++++++++++++++++++++++ 8 files changed, 295 insertions(+), 11 deletions(-) create mode 100755 scripts/check_embedded_secrets.sh diff --git a/.config/make/lint-fmt.mak b/.config/make/lint-fmt.mak index 46f659440..55ed1efc6 100644 --- a/.config/make/lint-fmt.mak +++ b/.config/make/lint-fmt.mak @@ -70,6 +70,11 @@ fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabi @echo "📣 Checking cryptographic capability wording guard..." ./scripts/check_fips_wording.sh +.PHONY: embedded-secrets-check +embedded-secrets-check: ## Check no private key material or credential literal is committed + @echo "🔑 Checking embedded secret material guard..." + ./scripts/check_embedded_secrets.sh + .PHONY: log-analyzer-rules-check log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source @echo "🩺 Checking log-analyzer rule anchors..." diff --git a/.config/make/pre-commit.mak b/.config/make/pre-commit.mak index 12eb9693e..b4ba093b0 100644 --- a/.config/make/pre-commit.mak +++ b/.config/make/pre-commit.mak @@ -19,13 +19,13 @@ planning-docs-check: ## Check that no planning-type documents are committed ./scripts/check_no_planning_docs.sh .PHONY: pre-commit -pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests +pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests @echo "✅ All pre-commit checks passed!" .PHONY: pre-pr -pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests +pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check doc-paths-check planning-docs-check log-analyzer-rules-check clippy-check test ## Run full pre-PR checks with clippy and tests @echo "✅ All pre-PR checks passed!" .PHONY: dev-check -dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks +dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks @echo "✅ Fast development checks passed!" diff --git a/.config/make/tests.mak b/.config/make/tests.mak index cdb8cd25e..937a22aca 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -34,6 +34,7 @@ script-tests: ## Run shell script tests ./scripts/test_exact_1mib_handoff_abba.sh ./scripts/test_pinned_paired_abba_bench.sh ./scripts/test_manual_transition_runbooks.sh + ./scripts/check_embedded_secrets.sh --self-test bash -n ./scripts/validate_object_data_cache_cold_stampede.sh python3 ./scripts/check_object_data_cache_follower_samples.py --self-test ./scripts/validate_object_data_cache_cold_stampede.sh --self-test diff --git a/.github/workflows/ci-docs-only.yml b/.github/workflows/ci-docs-only.yml index a156a61b9..5cd8ab733 100644 --- a/.github/workflows/ci-docs-only.yml +++ b/.github/workflows/ci-docs-only.yml @@ -120,6 +120,9 @@ jobs: - name: Check cryptographic capability wording run: ./scripts/check_fips_wording.sh + - name: Check no embedded secret material + run: ./scripts/check_embedded_secrets.sh + - name: Check no planning docs committed run: ./scripts/check_no_planning_docs.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 412145661..23a37d82b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,6 +155,9 @@ jobs: - name: Check cryptographic capability wording run: ./scripts/check_fips_wording.sh + - name: Check no embedded secret material + run: ./scripts/check_embedded_secrets.sh + - name: Check no planning docs committed run: ./scripts/check_no_planning_docs.sh diff --git a/crates/crypto/src/license_token.rs b/crates/crypto/src/license_token.rs index 8128605ec..cfec3a7ef 100644 --- a/crates/crypto/src/license_token.rs +++ b/crates/crypto/src/license_token.rs @@ -203,14 +203,6 @@ mod tests { assert!(result.is_err()); } - #[test] - fn test_source_does_not_embed_private_key() { - let source = include_str!("license_token.rs"); - let forbidden = ["BEGIN", "PRIVATE KEY"].join(" "); - - assert!(!source.contains(&forbidden)); - } - #[test] fn test_parse_signed_license_token_rejects_invalid_token() { let mut rng = rand::rng(); diff --git a/scripts/README.md b/scripts/README.md index 8f11d6931..3ba0b5499 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -28,6 +28,7 @@ their issue closes. | `check_architecture_migration_rules.sh` | ci-gate | Architecture-boundary anti-regression guard | ci.yml Quick Checks; `make pre-commit` | | `check_body_cache_whitelist.sh` | ci-gate | Keeps the app-layer body-cache eligibility gate fail-closed | ci.yml Quick Checks | | `check_doc_paths.sh` | ci-gate | Fails when instruction/architecture docs reference repo paths that no longer exist | `make pre-commit` / `pre-pr` | +| `check_embedded_secrets.sh` | ci-gate | Repo-wide scan blocking committed private key material and provider credential literals | ci.yml Quick Checks; `make pre-commit` / `pre-pr` | | `check_extension_schema_boundaries.sh` | ci-gate | Extension-schema crate boundary guard | ci.yml Quick Checks; `make pre-commit` | | `check_layer_dependencies.sh` | ci-gate | Crate-layering DAG guard (reads `layer-dependency-baseline.txt`) | ci.yml Quick Checks | | `check_logging_guardrails.sh` | ci-gate | Blocks legacy logging patterns from returning | `make pre-commit` / `pre-pr` | diff --git a/scripts/check_embedded_secrets.sh b/scripts/check_embedded_secrets.sh new file mode 100755 index 000000000..0634eb0f8 --- /dev/null +++ b/scripts/check_embedded_secrets.sh @@ -0,0 +1,279 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Guard: no private key material and no long-lived provider credential may +# exist as a literal anywhere in the repository — see AGENTS.md "Security +# Baseline" ("Never commit secrets, credentials, or key material") and +# .agents/skills/security-advisory-lessons ("Do not ship hard-coded shared +# tokens, HMAC secrets, private keys, or production test keys"). +# +# This replaces the unit test `test_source_does_not_embed_private_key` that +# used to live in crates/crypto/src/license_token.rs (rustfs/backlog#1884). +# That test read its own file with include_str! and asserted the file did not +# contain a PEM private-key header, protecting exactly one invariant: the RSA +# key that signs license tokens must never be checked in, because verification +# only ever needs the public key (crates/crypto/src/license_token.rs exposes +# `parse_signed_license_token`, and rustfs/src/license.rs reads the public key +# from RUSTFS_LICENSE_PUBLIC_KEY at runtime — no key material belongs in the +# tree at all). Its coverage was one file: moving the key one file sideways, +# even inside the same crate, passed silently, and renaming license_token.rs +# stopped the guard from compiling rather than reporting anything. +# +# This scan covers every tracked — and every not-yet-added, non-ignored — text +# file in the repository, so it is a strict superset of the retired assertion: +# the same needle, everywhere, plus the algorithm variants and the credential +# formats below. +# +# It deliberately does not exclude the paths .github/secret_scanning.yml tells +# GitHub push protection to ignore (crates/e2e_test, **/tests, **/benches, +# .docker, .vscode). Those exclusions exist because pasted test credentials are +# expected there, which is exactly where a real key is most likely to arrive +# unnoticed; this guard is the CI-side gate that still looks. +# +# Only literals are in scope. Key material injected at build time is out of +# scope on purpose: no build.rs in the workspace embeds key material and the +# license public key is read from the environment at startup, so an artifact +# scan would add a release build to a compile-free check job for no reachable +# failure mode today. Revisit if a build script ever bakes in key material. +# Binary files are skipped (`git grep -I`), and a key stored as bare base64 +# with its header stripped is not detected — the same two blind spots the +# retired test had. +# +# Every needle below is assembled around a variable so that the script's own +# text does not match the pattern it defines (the retired test used the same +# trick with ["BEGIN", "PRIVATE KEY"].join(" ")). That is what lets this script +# scan itself along with everything else instead of carving out a blind spot. +# +# `--self-test` builds throwaway fixture repositories and asserts every pattern +# family fires and every exemption holds; it is wired into `make script-tests`. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="${CHECK_EMBEDDED_SECRETS_ROOT:-$(cd "${SCRIPT_DIR}/.." && pwd)}" + +BEGIN_MARK="BEGIN" +KEY_MARK="Key" + +# The file the retired test pinned with include_str!. It stays listed so that +# renaming or moving it is reported here explicitly instead of quietly ending +# the license-key invariant, which is how the test failed. +PINNED_SOURCES=( + "crates/crypto/src/license_token.rs" +) + +# "|". The name is free of "|", so the first "|" splits. +# +# The private-key family matches the header phrase without requiring the PEM +# dashes, so a key pasted into a JSON/YAML string, a doc block, or a Rust +# string built without the delimiters is still caught. The credential family is +# format-anchored — fixed prefix plus fixed-width charset — so a match is a +# credential shape and not prose. +PATTERNS=( + "PEM private key header|${BEGIN_MARK}[[:space:]]+([A-Z0-9]+[[:space:]]+)*PRIVATE KEY" + "PuTTY private key file|PuTTY-User-${KEY_MARK}-File" + "AWS access key id|(A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}" + "GitHub token|gh[pousr]_[A-Za-z0-9]{36}" + "GitHub fine-grained token|github_pat_[A-Za-z0-9_]{22,}" + "Slack token|xox[abprs]-[A-Za-z0-9-]{10,}" + "Stripe live key|sk_live_[0-9a-zA-Z]{20,}" + "Google API key|AIza[0-9A-Za-z_-]{35}" + "SendGrid API key|SG\.[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}" + "npm access token|npm_[A-Za-z0-9]{36}" + "PyPI upload token|pypi-AgEIcHlwaS5vcmc[A-Za-z0-9_-]{50,}" +) + +# Exact strings that carry no secret wherever they appear. A hit is excused +# only if the line stops matching once these exact strings are removed, so a +# line holding both an example value and a real credential still fails, and +# editing an entry — swapping a placeholder body for real key material — makes +# the guard fire again. Entries that stop matching anything are reported as +# stale, so the list cannot decay into a blanket exclusion. +# +# 1-2: rustfs/src/admin/handlers/site_replication.rs negative fixtures for +# `validate_peer_connection_inner`, which must reject a private key +# submitted where a peer CA certificate is expected. Asserting on the +# rejection requires the header in the input; the key bodies are the +# literal word "secret". +# 3-4: AWS's own documented example access key id from the SigV4 test vectors, +# which this repository pairs with the equally documented example secret +# key across signer, IAM, madmin, and auth tests, plus the deliberate +# one-character variant rustfs/src/auth.rs uses to prove key comparison +# distinguishes near-identical ids. +AWS_EXAMPLE_STEM="AKIAIOSFODNN7EXAMPL" +NON_SECRET_LITERALS=( + "-----${BEGIN_MARK} PRIVATE KEY-----\\nsecret\\n-----END PRIVATE KEY-----" + "-----${BEGIN_MARK} RSA PRIVATE KEY-----\\nsecret\\n-----END RSA PRIVATE KEY-----" + "${AWS_EXAMPLE_STEM}E" + "${AWS_EXAMPLE_STEM}F" +) + +run_scan() { + cd "$ROOT_DIR" + + # Without this, a scan run outside a work tree would make every `git grep` + # fail and the guard would report success having read nothing. + if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + printf 'Embedded secret guard failed: %s is not a git work tree, so the scan cannot enumerate files\n' \ + "$ROOT_DIR" >&2 + return 1 + fi + + local literal_used=() + local i + for ((i = 0; i < ${#NON_SECRET_LITERALS[@]}; i++)); do + literal_used[i]="0" + done + + local status=0 + local source + for source in "${PINNED_SOURCES[@]}"; do + if [[ ! -f "$source" ]]; then + printf 'Embedded secret guard failed: %s is missing; update PINNED_SOURCES in scripts/check_embedded_secrets.sh after moving it\n' \ + "$source" >&2 + status=1 + fi + done + + local entry name pattern hits grep_status hit file rest line_no text trimmed sanitized + for entry in "${PATTERNS[@]}"; do + name="${entry%%|*}" + pattern="${entry#*|}" + + hits="" + grep_status=0 + hits="$(git grep --untracked -I -n -E -e "$pattern" -- .)" || grep_status=$? + if [[ "$grep_status" -gt 1 ]]; then + printf 'Embedded secret guard failed: git grep exited %s while scanning for %s\n' "$grep_status" "$name" >&2 + status=1 + continue + fi + + while IFS= read -r hit; do + [[ -z "$hit" ]] && continue + file="${hit%%:*}" + rest="${hit#*:}" + line_no="${rest%%:*}" + text="${rest#*:}" + trimmed="$(printf '%s' "$text" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + sanitized="$trimmed" + for ((i = 0; i < ${#NON_SECRET_LITERALS[@]}; i++)); do + if [[ "$sanitized" == *"${NON_SECRET_LITERALS[i]}"* ]]; then + sanitized="${sanitized//"${NON_SECRET_LITERALS[i]}"/}" + literal_used[i]="1" + fi + done + + if ! printf '%s' "$sanitized" | grep -q -E -e "$pattern"; then + continue + fi + + printf 'Embedded secret guard failed: %s at %s:%s\n %s\n' "$name" "$file" "$line_no" "$trimmed" >&2 + status=1 + done <<<"$hits" + done + + for ((i = 0; i < ${#NON_SECRET_LITERALS[@]}; i++)); do + if [[ "${literal_used[i]}" != "1" ]]; then + printf 'Embedded secret guard failed: stale exemption, nothing matches it any more: %s\n' \ + "${NON_SECRET_LITERALS[i]}" >&2 + status=1 + fi + done + + if [[ "$status" -ne 0 ]]; then + printf '\nRemove the key material or credential above, and rotate anything that was committed even briefly.\n' >&2 + printf 'A genuine non-secret match is excused by adding its exact text to NON_SECRET_LITERALS in scripts/check_embedded_secrets.sh with a reason.\n' >&2 + return 1 + fi + + printf 'Embedded secret guard passed (no private key material or provider credential literal in the tree).\n' + return 0 +} + +# One synthetic value per pattern family, assembled from a filler so this +# function does not match the patterns it exercises. +self_test_violation_lines() { + local fill="QWERTYUIOPASDFGHJKLZXCVBNM0123456789" + printf '%s\n' \ + "-----${BEGIN_MARK} PRIVATE KEY-----" \ + "PuTTY-User-${KEY_MARK}-File: ssh-rsa" \ + "AKIA${fill:0:16}" \ + "ghp_${fill:0:36}" \ + "github_pat_${fill:0:22}" \ + "xoxb-${fill:0:12}" \ + "sk_live_${fill:0:20}" \ + "AIza${fill:0:35}" \ + "SG.${fill:0:20}.${fill:0:20}" \ + "npm_${fill:0:36}" \ + "pypi-AgEIcHlwaS5vcmc${fill}${fill:0:14}" +} + +self_test_fixture() { + local dir="$1" i + mkdir -p "${dir}/crates/crypto/src" + : >"${dir}/crates/crypto/src/license_token.rs" + # Every exemption must appear, or the stale-exemption check fires and the + # fixture would fail for a reason the case under test is not about. + for ((i = 0; i < ${#NON_SECRET_LITERALS[@]}; i++)); do + printf 'excused %s\n' "${NON_SECRET_LITERALS[i]}" >>"${dir}/excused.txt" + done + git -C "$dir" init -q +} + +SELF_TEST_TMP="" + +self_test() { + SELF_TEST_TMP="$(mktemp -d)" + trap 'rm -rf "$SELF_TEST_TMP"' EXIT + + local failures=0 out scan_status + local clean="${SELF_TEST_TMP}/clean" dirty="${SELF_TEST_TMP}/dirty" renamed="${SELF_TEST_TMP}/renamed" + + self_test_fixture "$clean" + if out="$(CHECK_EMBEDDED_SECRETS_ROOT="$clean" "$0" 2>&1)"; then + printf 'self-test ok: clean fixture with every exemption present passes\n' + else + printf 'self-test FAILED: clean fixture should pass but reported:\n%s\n' "$out" >&2 + failures=$((failures + 1)) + fi + + self_test_fixture "$dirty" + self_test_violation_lines >"${dirty}/leaked.txt" + scan_status=0 + out="$(CHECK_EMBEDDED_SECRETS_ROOT="$dirty" "$0" 2>&1)" || scan_status=$? + if [[ "$scan_status" -eq 0 ]]; then + printf 'self-test FAILED: fixture holding one value per pattern family should fail\n' >&2 + failures=$((failures + 1)) + fi + local entry name + for entry in "${PATTERNS[@]}"; do + name="${entry%%|*}" + if ! printf '%s' "$out" | grep -q -F -- "$name"; then + printf 'self-test FAILED: pattern family "%s" did not fire on its own probe value\n' "$name" >&2 + failures=$((failures + 1)) + fi + done + [[ "$failures" -eq 0 ]] && printf 'self-test ok: all %s pattern families fire\n' "${#PATTERNS[@]}" + + self_test_fixture "$renamed" + rm -f "${renamed}/crates/crypto/src/license_token.rs" + if CHECK_EMBEDDED_SECRETS_ROOT="$renamed" "$0" >/dev/null 2>&1; then + printf 'self-test FAILED: a moved pinned source should be reported\n' >&2 + failures=$((failures + 1)) + else + printf 'self-test ok: moving a pinned source is reported\n' + fi + + if [[ "$failures" -ne 0 ]]; then + printf '%s self-test assertion(s) failed\n' "$failures" >&2 + return 1 + fi + printf 'Embedded secret guard self-test passed.\n' + return 0 +} + +if [[ "${1:-}" == "--self-test" ]]; then + self_test +else + run_scan +fi From 7b5389d2f903861796ed95df0d2b38e499290241 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 19 Aug 2026 11:00:20 +0800 Subject: [PATCH 02/27] test(io-schedule): drop 24 no-op serial markers from the IO scheduler unit tests (#6243) --- rustfs/src/storage/concurrency/io_schedule.rs | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/rustfs/src/storage/concurrency/io_schedule.rs b/rustfs/src/storage/concurrency/io_schedule.rs index 8d202708a..9ec717623 100644 --- a/rustfs/src/storage/concurrency/io_schedule.rs +++ b/rustfs/src/storage/concurrency/io_schedule.rs @@ -1766,11 +1766,9 @@ mod tests { }; use rustfs_io_core::io_profile::{AccessPattern, StorageMedia}; use rustfs_io_metrics::bandwidth::{BandwidthSnapshot, BandwidthTier}; - use serial_test::serial; use std::time::Duration; #[tokio::test] - #[serial] async fn test_io_priority_queue_basic() { let config = IoPriorityQueueConfig::default(); let queue = IoPriorityQueue::new(config); @@ -1789,7 +1787,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_priority_queue_dequeue_order() { let config = IoPriorityQueueConfig::default(); let queue = IoPriorityQueue::new(config); @@ -1817,7 +1814,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_priority_queue_status() { let config = IoPriorityQueueConfig::default(); let queue = IoPriorityQueue::new(config); @@ -1835,7 +1831,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_priority_queue_starvation_prevention() { let config = IoPriorityQueueConfig { starvation_threshold_secs: 1, @@ -1859,7 +1854,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_priority_from_size() { // High priority: < 1MB assert_eq!(IoPriority::from_size(100 * 1024), IoPriority::High); @@ -1875,7 +1869,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_load_level_from_wait_duration() { use std::time::Duration; @@ -1893,7 +1886,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_scheduler_config_default() { let config = IoSchedulerConfig::default(); @@ -1907,7 +1899,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_scheduler_config_to_core_config() { let config = IoSchedulerConfig::default(); let core = config.to_core_config(); @@ -1923,7 +1914,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_priority_queue_config_to_core_config() { let config = IoPriorityQueueConfig::default(); let core = config.to_core_config(); @@ -1935,7 +1925,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_io_priority_queue_config_from_scheduler_config() { let scheduler_config = IoSchedulerConfig { queue_high_capacity: 128, @@ -1958,7 +1947,6 @@ mod tests { // ============================================ #[tokio::test] - #[serial] async fn test_multi_factor_strategy_nvme_sequential_low_load() { // NVMe + Sequential + Low load = maximum buffer size let context = IoSchedulingContext { @@ -1985,7 +1973,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_hdd_random_high_load() { // HDD + Random + High load = conservative buffer size let context = IoSchedulingContext { @@ -2012,7 +1999,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_ssd_mixed_medium_load() { // SSD + Mixed + Medium load = moderate buffer let context = IoSchedulingContext { @@ -2040,7 +2026,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_critical_load_disables_features() { // Any media + Critical load = minimal features let context = IoSchedulingContext { @@ -2065,7 +2050,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_buffer_cap_enforcement() { // Test that storage media caps are enforced let context = IoSchedulingContext { @@ -2090,7 +2074,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_applies_sequential_hint_when_pattern_unknown() { let context = IoSchedulingContext { file_size: 2 * 1024 * 1024 * 1024, // 2GiB @@ -2115,7 +2098,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_bandwidth_low_reduces_buffer() { // Low bandwidth should reduce buffer let context = IoSchedulingContext { @@ -2139,7 +2121,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_high_concurrency_reduction() { // High concurrency should reduce buffer let context = IoSchedulingContext { @@ -2162,7 +2143,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_sequential_boost() { // Sequential reads should get boost let sequential_context = IoSchedulingContext { @@ -2204,7 +2184,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_unknown_media_conservative() { // Unknown media should be conservative let context = IoSchedulingContext { @@ -2230,7 +2209,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_priority_classification() { // Test priority classification based on file size let small_context = IoSchedulingContext { @@ -2277,7 +2255,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_readahead_decision_matrix() { // Test readahead enable/disable logic let configs = vec![ @@ -2363,7 +2340,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_buffer_multiplier_stages() { // Test that all multiplier stages are applied let context = IoSchedulingContext { @@ -2398,7 +2374,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multi_factor_strategy_compatibility_path() { // Test that compatibility path (from_wait_duration) still works let wait_duration = Duration::from_millis(50); From 728efcec8993489dbef781b29ef8979ef30ac4f2 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 19 Aug 2026 11:01:20 +0800 Subject: [PATCH 03/27] test(ecstore): assert the error conversions, and stop the census over-reporting (#6241) --- crates/ecstore/src/disk/error.rs | 27 +++++++++++--- crates/io-core/src/io_profile.rs | 55 ++++++++++++++++++----------- crates/notify/src/runtime_facade.rs | 12 ++++++- crates/s3-types/src/event_name.rs | 13 +++++-- scripts/find_assertless_tests.py | 46 +++++++++++++++++++++--- 5 files changed, 121 insertions(+), 32 deletions(-) diff --git a/crates/ecstore/src/disk/error.rs b/crates/ecstore/src/disk/error.rs index 51fb04daa..9c2f1706c 100644 --- a/crates/ecstore/src/disk/error.rs +++ b/crates/ecstore/src/disk/error.rs @@ -853,13 +853,32 @@ mod tests { #[test] fn test_error_conversions() { - // Test From implementations + // A plain io::Error carries no typed payload to recover, so it lands in + // `Io` rather than being guessed at from its kind — `NotFound` here must + // not silently become `FileNotFound`, which quorum aggregation counts as + // a different error (rustfs/backlog#1836). let io_error = std::io::Error::new(std::io::ErrorKind::NotFound, "test"); - let _disk_error: DiskError = io_error.into(); + let disk_error: DiskError = io_error.into(); + match &disk_error { + DiskError::Io(inner) => assert_eq!(inner.kind(), std::io::ErrorKind::NotFound), + other => panic!("a plain io::Error must stay typed as Io, got {other:?}"), + } - let json_str = r#"{"invalid": json}"#; // Invalid JSON + // A typed DiskError boxed through io::Error round-trips back to itself + // instead of degrading to `Io`. + let boxed: std::io::Error = std::io::Error::other(DiskError::VolumeNotFound); + assert_eq!(DiskError::from(boxed), DiskError::VolumeNotFound); + + // serde_json errors have no dedicated variant and fold into `other`, + // keeping the original message. + let json_str = r#"{"invalid": json}"#; let json_error = serde_json::from_str::(json_str).unwrap_err(); - let _disk_error: DiskError = json_error.into(); + let json_message = json_error.to_string(); + let disk_error: DiskError = json_error.into(); + assert!( + disk_error.to_string().contains(&json_message), + "the json error message must survive the conversion: {disk_error}" + ); } #[test] diff --git a/crates/io-core/src/io_profile.rs b/crates/io-core/src/io_profile.rs index 86cd79448..46b894c49 100644 --- a/crates/io-core/src/io_profile.rs +++ b/crates/io-core/src/io_profile.rs @@ -436,30 +436,45 @@ mod tests { assert_eq!(unknown_profile.sequential_boost_multiplier, 1.0); } - #[cfg(target_os = "linux")] + // What platform probing returns depends on the machine, so these pin the two + // rules that do not: the override wins over probing, and probing that is + // switched off reports Unknown rather than guessing (rustfs/backlog#1836). #[test] - fn test_linux_storage_detection_exists() { - // This test just verifies the detection function exists and doesn't panic - // The actual result depends on the system it's running on - let result = detect_storage_media(true, ""); - // We should get some result (not panic) - match result { - StorageMedia::Nvme | StorageMedia::Ssd | StorageMedia::Hdd | StorageMedia::Unknown => { - // All valid results - } + fn storage_media_override_wins_over_platform_detection() { + for (override_value, expected) in [ + ("nvme", StorageMedia::Nvme), + ("ssd", StorageMedia::Ssd), + ("hdd", StorageMedia::Hdd), + ] { + assert_eq!(detect_storage_media(true, override_value), expected); + assert_eq!( + detect_storage_media(false, override_value), + expected, + "an override must be honoured even with detection disabled" + ); } } - #[cfg(target_os = "macos")] #[test] - fn test_macos_storage_detection_exists() { - // This test just verifies the detection function exists and doesn't panic - let result = detect_storage_media(true, ""); - // We should get some result (not panic) - match result { - StorageMedia::Nvme | StorageMedia::Ssd | StorageMedia::Hdd | StorageMedia::Unknown => { - // All valid results - } - } + fn disabled_detection_reports_unknown_instead_of_guessing() { + assert_eq!(detect_storage_media(false, ""), StorageMedia::Unknown); + assert_eq!( + detect_storage_media(false, "not-a-medium"), + StorageMedia::Unknown, + "an unparseable override falls through to the disabled path" + ); + } + + #[test] + fn enabled_detection_returns_a_medium_for_this_platform() { + // Whatever this machine reports, it must be one of the known variants and + // it must be stable across calls — a probe that flapped would make the + // scheduler's profile depend on when it asked. + let first = detect_storage_media(true, ""); + assert!(matches!( + first, + StorageMedia::Nvme | StorageMedia::Ssd | StorageMedia::Hdd | StorageMedia::Unknown + )); + assert_eq!(detect_storage_media(true, ""), first); } } diff --git a/crates/notify/src/runtime_facade.rs b/crates/notify/src/runtime_facade.rs index 683a01ba6..f692880c5 100644 --- a/crates/notify/src/runtime_facade.rs +++ b/crates/notify/src/runtime_facade.rs @@ -527,9 +527,19 @@ mod tests { } #[tokio::test] - async fn runtime_facade_stops_empty_replay_workers() { + async fn stopping_replay_workers_is_a_no_op_when_there_are_none() { let (facade, _, _) = build_facade(); + facade.stop_replay_workers().await; + + // The stop path takes the worker list and hands it to the adapter, so an + // empty facade must come back with the list still empty and dispatch + // released rather than left paused (rustfs/backlog#1836). + assert!(facade.replay_workers.read().await.is_empty()); + + // Calling it twice must stay harmless: shutdown paths do exactly that. + facade.stop_replay_workers().await; + assert!(facade.replay_workers.read().await.is_empty()); } #[tokio::test] diff --git a/crates/s3-types/src/event_name.rs b/crates/s3-types/src/event_name.rs index 659b88b46..dee252dbd 100644 --- a/crates/s3-types/src/event_name.rs +++ b/crates/s3-types/src/event_name.rs @@ -873,9 +873,16 @@ mod tests { /// now return a finite, non-panicking mask. #[test] fn test_mask_never_recurses_for_any_variant() { - for ev in ALL_EVENT_NAMES { - // Must terminate (no infinite recursion / stack overflow). - let _ = ev.mask(); + // Terminating is the point — a regression here overflows the stack rather + // than failing an assertion — but the masks are collected and checked so + // the loop cannot be optimised into nothing and so a variant that starts + // returning an empty mask is caught too (rustfs/backlog#1836). + let masks: Vec = ALL_EVENT_NAMES.iter().map(|ev| ev.mask()).collect(); + + assert_eq!(masks.len(), ALL_EVENT_NAMES.len()); + for (ev, mask) in ALL_EVENT_NAMES.iter().zip(&masks) { + assert_ne!(*mask, 0, "{ev:?} must carry at least one bit"); + assert_eq!(ev.mask(), *mask, "{ev:?} must return the same mask every call"); } } diff --git a/scripts/find_assertless_tests.py b/scripts/find_assertless_tests.py index dc73447de..094d051af 100755 --- a/scripts/find_assertless_tests.py +++ b/scripts/find_assertless_tests.py @@ -49,19 +49,47 @@ import sys from pathlib import Path VERIFY_SIGNALS = re.compile( - r"assert!|assert_eq!|assert_ne!|debug_assert|panic!\(|\.expect\(|\.unwrap\(|" + r"assert[a-z0-9_]*!|debug_assert|panic!\(|\.expect\(|\.unwrap\(|" r"unreachable!|matches!\(|insta::|proptest!|\.await\?|\)\?|\?;|should_panic" ) DELEGATION = re.compile( - r"\b(?:assert|verify|check|expect|ensure|run)_[a-z0-9_]*\s*\(|" - r"\b[a-z0-9_]+_(?:case|cases|harness|roundtrip|round_trip)\s*\(" + r"\b(?:assert|verify|check|expect|ensure|run)_[a-z0-9_]*(?:::<[^>]*>)?\s*\(|" + r"\b[a-z0-9_]+_(?:case|cases|harness|roundtrip|round_trip)(?:::<[^>]*>)?\s*\(" ) + +# A body whose whole content is one call delegates by construction, whatever the +# callee is named: `run(DurabilityMode::Strict).await` and +# `aborting_encode_drops_blocked_producer(EncodePipeline::Vec).await` both hand +# every assertion to a shared harness. +SINGLE_CALL_BODY = re.compile( + r"\A\s*[a-zA-Z_][a-zA-Z0-9_:]*(?:::<[^>]*>)?\s*\([^;]*\)\s*(?:\.await\s*)?;?\s*\Z", + re.S, +) + +# A nested `fn` that is only bound and discarded is a signature guard: the type +# system is the assertion, exactly like the `fn _name()` form below. +SIGNATURE_GUARD = re.compile(r"\bfn\s+[a-zA-Z0-9_]+\s*(?:<[^>]*>)?\s*\([^;]*\)[^;]*\{", re.S) +DISCARDED_BINDING = re.compile(r"\blet\s+_\s*=\s*[a-zA-Z_][a-zA-Z0-9_]*\s*;") +# `let _ = Type::::method;` — a path item referenced but never called can only +# be a signature guard; the call form (`let _ = x.foo();`) is excluded by the +# absence of parens before the semicolon. +DISCARDED_PATH_ITEM = re.compile(r"\blet\s+_\s*=\s*[a-zA-Z_][a-zA-Z0-9_]*(?:::(?:<[^>]*>|[a-zA-Z_][a-zA-Z0-9_]*))+\s*;") COMPILE_TIME_CHECK = re.compile(r"\bfn\s+_[a-zA-Z0-9_]*\s*(?:<[^>]*>)?\s*\(") TEST_ATTR = re.compile(r"#\[(?:tokio::)?test[\](]") TEST_CASE_ATTR = re.compile(r"#\[test_case") FN_LINE = re.compile(r"^\s*(?:pub\s+)?(?:async\s+)?fn\s+([a-zA-Z0-9_]+)") + +def extract_body(text: str) -> str: + """Return what is between the outermost braces of a scanned function.""" + start = text.find("{") + end = text.rfind("}") + if start == -1 or end <= start: + return text + return text[start + 1 : end] + + def scan_file(path: Path): try: lines = path.read_text(encoding="utf-8").split("\n") @@ -105,7 +133,17 @@ def scan_file(path: Path): break k += 1 text = "\n".join(body) - if not VERIFY_SIGNALS.search(text) and not DELEGATION.search(text) and not COMPILE_TIME_CHECK.search(text): + # The attribute block carries verification too: `#[should_panic(expected + # = "...")]` makes the panic message the assertion. + attr_text = "\n".join(attrs) + inner = extract_body(text) + delegates = ( + DELEGATION.search(text) + or SINGLE_CALL_BODY.match(inner) + or (SIGNATURE_GUARD.search(inner) and DISCARDED_BINDING.search(inner)) + or DISCARDED_PATH_ITEM.search(inner) + ) + if not VERIFY_SIGNALS.search(text) and not VERIFY_SIGNALS.search(attr_text) and not delegates and not COMPILE_TIME_CHECK.search(text): print(f"{path}:{j + 1}: {name}") i = k + 1 From aa6b9001f159b4a40f7c839ac137e96c101e25d0 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 19 Aug 2026 11:01:38 +0800 Subject: [PATCH 04/27] test(e2e): drop 79 no-op serial markers from six more suites (#6239) --- crates/e2e_test/src/admin_auth_test.rs | 5 ----- crates/e2e_test/src/admin_iam_crud_test.rs | 4 ---- .../src/admin_timeout_regression_test.rs | 2 -- crates/e2e_test/src/anonymous_access_test.rs | 5 ----- crates/e2e_test/src/api_rate_limit_test.rs | 4 ---- .../src/archive_download_integrity_test.rs | 14 -------------- crates/e2e_test/src/bucket_logging_test.rs | 4 ---- crates/e2e_test/src/bucket_policy_check_test.rs | 2 -- .../src/bucket_stats_regression_test.rs | 4 ---- crates/e2e_test/src/checksum_upload_test.rs | 8 -------- crates/e2e_test/src/cluster_concurrency_test.rs | 3 --- .../src/cluster_multidrive_pool_test.rs | 3 --- crates/e2e_test/src/compression_test.rs | 7 ------- crates/e2e_test/src/connection_cap_test.rs | 3 --- crates/e2e_test/src/console_smoke_test.rs | 2 -- crates/e2e_test/src/content_encoding_test.rs | 4 ---- .../e2e_test/src/copy_object_checksum_test.rs | 8 -------- .../e2e_test/src/copy_object_metadata_test.rs | 5 ----- crates/e2e_test/src/copy_object_tagging_test.rs | 3 --- .../src/copy_object_version_restore_test.rs | 3 --- .../src/copy_source_invalid_date_test.rs | 2 -- .../e2e_test/src/create_bucket_region_test.rs | 3 --- crates/e2e_test/src/data_usage_test.rs | 3 --- .../delete_marker_migration_semantics_test.rs | 3 --- .../src/delete_object_no_content_length_test.rs | 2 -- .../src/delete_objects_versioning_test.rs | 3 --- crates/e2e_test/src/delete_regression_test.rs | 6 ------ .../src/distributed_startup_regression_test.rs | 4 ---- .../src/existing_object_tag_policy_test.rs | 5 ----- .../src/get_codec_streaming_compat_test.rs | 2 -- crates/e2e_test/src/group_delete_test.rs | 4 ---- .../src/head_object_consistency_test.rs | 2 -- crates/e2e_test/src/head_object_range_test.rs | 2 -- .../src/heal_erasure_disk_rebuild_test.rs | 5 ----- .../src/internode_rpc_signature_e2e_test.rs | 6 ------ .../src/kms/bucket_default_encryption_test.rs | 6 ------ .../src/kms/configured_roundtrip_test.rs | 3 --- .../src/kms/copy_object_self_copy_sse_test.rs | 4 ---- .../kms/copy_object_version_restore_sse_test.rs | 2 -- .../src/kms/encryption_metadata_test.rs | 4 ---- .../kms_authorization_negative_matrix_test.rs | 3 --- .../e2e_test/src/kms/kms_comprehensive_test.rs | 6 ------ crates/e2e_test/src/kms/kms_edge_cases_test.rs | 7 ------- .../e2e_test/src/kms/kms_fault_recovery_test.rs | 5 ----- crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs | 3 --- crates/e2e_test/src/kms/kms_local_test.rs | 5 ----- .../src/kms/multipart_encryption_test.rs | 6 ------ crates/e2e_test/src/kms/test_runner.rs | 3 --- crates/e2e_test/src/leading_slash_key_test.rs | 3 --- .../e2e_test/src/lifecycle_regression_test.rs | 5 ----- .../src/list_buckets_double_slash_test.rs | 4 ---- .../src/list_buckets_iam_filter_test.rs | 2 -- ...t_object_versions_metadata_extension_test.rs | 2 -- .../src/list_object_versions_regression_test.rs | 3 --- .../src/list_objects_duplicates_test.rs | 4 ---- .../list_objects_v2_metadata_extension_test.rs | 2 -- .../src/list_objects_v2_pagination_test.rs | 13 ------------- crates/e2e_test/src/listing_regression_test.rs | 5 ----- .../e2e_test/src/mc_mirror_small_bucket_test.rs | 2 -- .../e2e_test/src/namespace_lock_quorum_test.rs | 3 --- crates/e2e_test/src/negative_sigv4_test.rs | 7 ------- .../src/notification_startup_regression_test.rs | 3 --- .../e2e_test/src/notification_webhook_test.rs | 4 ---- crates/e2e_test/src/object_lambda_test.rs | 17 ----------------- .../src/overwrite_cleanup_regression_test.rs | 2 -- crates/e2e_test/src/presigned_negative_test.rs | 8 -------- crates/e2e_test/src/protocols/test_runner.rs | 2 -- crates/e2e_test/src/quota_test.rs | 15 --------------- .../e2e_test/src/reliant/conditional_writes.rs | 5 ----- .../src/reliant/get_deleted_object_test.rs | 5 ----- .../head_deleted_object_versioning_test.rs | 2 -- .../src/reliant/head_tls_bodyless_test.rs | 2 -- crates/e2e_test/src/reliant/sql.rs | 8 -------- .../e2e_test/src/replication_extension_test.rs | 7 ------- crates/e2e_test/src/security_boundary_test.rs | 5 ----- .../src/server_startup_failfast_test.rs | 2 -- .../e2e_test/src/snowball_auto_extract_test.rs | 7 ------- crates/e2e_test/src/special_chars_test.rs | 15 --------------- .../src/stale_multipart_cleanup_cluster_test.rs | 2 -- crates/e2e_test/src/sts_query_compat_test.rs | 7 ------- .../src/tier_transition_regression_test.rs | 4 ---- crates/e2e_test/src/tls_hot_reload_test.rs | 2 -- .../e2e_test/src/version_id_regression_test.rs | 11 ----------- 83 files changed, 392 deletions(-) diff --git a/crates/e2e_test/src/admin_auth_test.rs b/crates/e2e_test/src/admin_auth_test.rs index 3cdd92c75..96e7878c4 100644 --- a/crates/e2e_test/src/admin_auth_test.rs +++ b/crates/e2e_test/src/admin_auth_test.rs @@ -38,7 +38,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; - use serial_test::serial; use std::error::Error; use std::io::Read; use std::process::{Command, Stdio}; @@ -162,7 +161,6 @@ mod tests { /// A fully authenticated but non-admin credential must be rejected with /// `403 AccessDenied` on an admin API, while the root credential succeeds. #[tokio::test(flavor = "multi_thread")] - #[serial] async fn non_admin_credential_denied_on_admin_api() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -198,7 +196,6 @@ mod tests { } #[tokio::test(flavor = "multi_thread")] - #[serial] async fn non_admin_credential_denied_on_manual_transition_run() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -326,7 +323,6 @@ mod tests { /// credential is accepted and the old one is rejected, on both the S3 data /// plane and the admin plane. #[tokio::test(flavor = "multi_thread")] - #[serial] async fn root_credential_rotation_takes_effect() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -389,7 +385,6 @@ mod tests { /// runtime. We capture the child's stdout/stderr directly (the shared /// harness inherits stdio) and poll for the warning until it appears. #[tokio::test(flavor = "multi_thread")] - #[serial] async fn default_credentials_emit_startup_warning() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/admin_iam_crud_test.rs b/crates/e2e_test/src/admin_iam_crud_test.rs index 508edf810..84b50709c 100644 --- a/crates/e2e_test/src/admin_iam_crud_test.rs +++ b/crates/e2e_test/src/admin_iam_crud_test.rs @@ -33,7 +33,6 @@ use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::{Client, Config}; use reqwest::StatusCode; -use serial_test::serial; use std::error::Error; use tokio::time::{Duration, sleep}; @@ -178,7 +177,6 @@ async fn assert_admin_status( } #[tokio::test] -#[serial] async fn test_update_service_account_enforces_owner_and_parent_scope() -> TestResult { init_logging(); @@ -348,7 +346,6 @@ async fn test_update_service_account_enforces_owner_and_parent_scope() -> TestRe /// Full user -> policy -> service-account lifecycle, proving each management /// call takes effect on the data plane, not just that the endpoint answers 200. #[tokio::test] -#[serial] async fn test_admin_user_policy_service_account_crud_lifecycle() -> TestResult { init_logging(); @@ -573,7 +570,6 @@ async fn test_admin_user_policy_service_account_crud_lifecycle() -> TestResult { /// non-admin credential with 403 AccessDenied (sec-4 assertion pattern; the /// gate implementation itself is owned by sec-4 / admin_auth_test). #[tokio::test] -#[serial] async fn test_admin_iam_endpoints_deny_non_admin_credential() -> TestResult { init_logging(); diff --git a/crates/e2e_test/src/admin_timeout_regression_test.rs b/crates/e2e_test/src/admin_timeout_regression_test.rs index c561e5d76..94a91f03e 100644 --- a/crates/e2e_test/src/admin_timeout_regression_test.rs +++ b/crates/e2e_test/src/admin_timeout_regression_test.rs @@ -21,7 +21,6 @@ use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; use serde::Deserialize; -use serial_test::serial; use std::error::Error; use std::process::Command; use tokio::time::{Duration, sleep, timeout}; @@ -100,7 +99,6 @@ fn offline_server_count(info: &InfoMessage) -> usize { } #[tokio::test(flavor = "multi_thread")] -#[serial] async fn test_single_admin_timeout_does_not_immediately_mark_peer_offline() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/anonymous_access_test.rs b/crates/e2e_test/src/anonymous_access_test.rs index 0891ac0a9..68a4f4095 100644 --- a/crates/e2e_test/src/anonymous_access_test.rs +++ b/crates/e2e_test/src/anonymous_access_test.rs @@ -18,7 +18,6 @@ use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; use aws_sdk_s3::types::PublicAccessBlockConfiguration; -use serial_test::serial; use tracing::info; async fn setup_public_bucket( @@ -73,7 +72,6 @@ async fn anonymous_get_object( /// Issue #2036: Anonymous GetObject should succeed when bucket policy allows it /// and no PublicAccessBlock configuration exists (ConfigNotFound). #[tokio::test] -#[serial] async fn test_anonymous_access_allowed_when_public_access_block_missing() -> Result<(), Box> { init_logging(); @@ -100,7 +98,6 @@ async fn test_anonymous_access_allowed_when_public_access_block_missing() -> Res /// Anonymous GetObject should be denied when RestrictPublicBuckets is true. #[tokio::test] -#[serial] async fn test_anonymous_access_denied_when_restrict_public_buckets_enabled() -> Result<(), Box> { init_logging(); @@ -137,7 +134,6 @@ async fn test_anonymous_access_denied_when_restrict_public_buckets_enabled() /// Anonymous GetObject should succeed when PublicAccessBlock exists but /// RestrictPublicBuckets is explicitly false. #[tokio::test] -#[serial] async fn test_anonymous_access_allowed_when_restrict_public_buckets_disabled() -> Result<(), Box> { init_logging(); @@ -176,7 +172,6 @@ async fn test_anonymous_access_allowed_when_restrict_public_buckets_disabled() /// reaches authorization through a fallback branch, and that branch has to apply the /// same public-access gate as a direct grant. #[tokio::test] -#[serial] async fn ghsa_x298_anonymous_list_object_versions_denied_when_restrict_public_buckets_enabled() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/api_rate_limit_test.rs b/crates/e2e_test/src/api_rate_limit_test.rs index 28f80517d..e7a3e682d 100644 --- a/crates/e2e_test/src/api_rate_limit_test.rs +++ b/crates/e2e_test/src/api_rate_limit_test.rs @@ -18,13 +18,11 @@ //! completely inert with default configuration. use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; -use serial_test::serial; use tracing::info; type TestResult = Result<(), Box>; #[tokio::test] -#[serial] async fn api_rate_limit_enforces_429_with_retry_after_when_enabled() -> TestResult { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -87,7 +85,6 @@ async fn api_rate_limit_enforces_429_with_retry_after_when_enabled() -> TestResu } #[tokio::test] -#[serial] async fn api_rate_limit_bucket_dimension_throttles_per_bucket() -> TestResult { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -133,7 +130,6 @@ async fn api_rate_limit_bucket_dimension_throttles_per_bucket() -> TestResult { } #[tokio::test] -#[serial] async fn api_rate_limit_stays_inert_by_default() -> TestResult { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/archive_download_integrity_test.rs b/crates/e2e_test/src/archive_download_integrity_test.rs index ac23f5d70..e043c6105 100644 --- a/crates/e2e_test/src/archive_download_integrity_test.rs +++ b/crates/e2e_test/src/archive_download_integrity_test.rs @@ -24,7 +24,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::{pre_sign_v4, sign_v4}; use s3s::Body; - use serial_test::serial; use sha2::{Digest, Sha256}; use std::error::Error; use std::io::{Cursor, Write}; @@ -339,7 +338,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_put_allows_content_encoding_by_default() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -367,7 +365,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_put_rejects_content_encoding_when_strict_mode_enabled() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -391,7 +388,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_put_with_aws_chunked_does_not_persist_content_encoding_by_default() -> Result<(), Box> { init_logging(); @@ -427,7 +423,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_put_with_aws_chunked_and_effective_encoding_roundtrips_by_default() -> Result<(), Box> { init_logging(); @@ -463,7 +458,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_put_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -498,7 +492,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_put_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled() -> Result<(), Box> { init_logging(); @@ -529,7 +522,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_download_roundtrip_with_http_compression_enabled() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -591,7 +583,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_multipart_roundtrip_preserves_bytes() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -687,7 +678,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_multipart_get_ignores_empty_conditional_etag_headers() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -723,7 +713,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_roundtrips_by_default() -> Result<(), Box> { init_logging(); @@ -753,7 +742,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_multipart_with_aws_chunked_allowed_when_strict_mode_enabled() -> Result<(), Box> { init_logging(); @@ -783,7 +771,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_archive_multipart_with_aws_chunked_and_effective_encoding_rejects_when_strict_mode_enabled() -> Result<(), Box> { init_logging(); @@ -816,7 +803,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_presigned_get_and_reverse_proxy_preserve_multipart_bytes() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/bucket_logging_test.rs b/crates/e2e_test/src/bucket_logging_test.rs index 79d2e6843..e60483097 100644 --- a/crates/e2e_test/src/bucket_logging_test.rs +++ b/crates/e2e_test/src/bucket_logging_test.rs @@ -24,11 +24,9 @@ mod tests { }; use http::Method; use http::header::CONTENT_TYPE; - use serial_test::serial; use tracing::info; #[tokio::test] - #[serial] async fn test_dummy_bucket_compatibility_endpoints() { init_logging(); info!("Starting test: dummy-compat bucket APIs should match S3-compatible behavior"); @@ -236,7 +234,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_dummy_bucket_compatibility_endpoints_no_such_bucket() { init_logging(); info!("Starting test: dummy-compat bucket APIs should return NoSuchBucket for missing bucket"); @@ -392,7 +389,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_dummy_bucket_endpoints_http_contracts() { init_logging(); info!("Starting test: dummy-compat bucket API HTTP contracts"); diff --git a/crates/e2e_test/src/bucket_policy_check_test.rs b/crates/e2e_test/src/bucket_policy_check_test.rs index 9b958f343..e71e5fe97 100644 --- a/crates/e2e_test/src/bucket_policy_check_test.rs +++ b/crates/e2e_test/src/bucket_policy_check_test.rs @@ -18,7 +18,6 @@ use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::{Client, Config}; -use serial_test::serial; use tracing::info; async fn create_user( @@ -51,7 +50,6 @@ fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: } #[tokio::test] -#[serial] async fn test_bucket_policy_authenticated_user() -> Result<(), Box> { init_logging(); if !crate::common::awscurl_available() { diff --git a/crates/e2e_test/src/bucket_stats_regression_test.rs b/crates/e2e_test/src/bucket_stats_regression_test.rs index 3df98f1ef..7754c0639 100644 --- a/crates/e2e_test/src/bucket_stats_regression_test.rs +++ b/crates/e2e_test/src/bucket_stats_regression_test.rs @@ -35,7 +35,6 @@ mod tests { use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; use rustfs_data_usage::DataUsageInfo; - use serial_test::serial; use std::error::Error; use tokio::time::{Duration, sleep}; use tracing::info; @@ -59,7 +58,6 @@ mod tests { /// 3. Query admin data usage API /// 4. Verify object count > 0 #[tokio::test] - #[serial] async fn test_bucket_object_count_updates_after_put() -> TestResult { init_logging(); info!("RT-09: bucket object count updates after PUT"); @@ -126,7 +124,6 @@ mod tests { /// Regression pattern: stats remain unchanged after objects are deleted /// (rustfs#5615). #[tokio::test] - #[serial] async fn test_bucket_object_count_updates_after_delete() -> TestResult { init_logging(); info!("RT-09b: bucket object count updates after DELETE"); @@ -220,7 +217,6 @@ mod tests { /// Regression pattern: DataUsageInfo undercounts versioned bucket versions /// and delete markers (rustfs#3898). #[tokio::test] - #[serial] async fn test_versioned_bucket_stats_count_all_versions() -> TestResult { init_logging(); info!("RT-09c: versioned bucket stats count all versions"); diff --git a/crates/e2e_test/src/checksum_upload_test.rs b/crates/e2e_test/src/checksum_upload_test.rs index efa4a81b8..814d18d5a 100644 --- a/crates/e2e_test/src/checksum_upload_test.rs +++ b/crates/e2e_test/src/checksum_upload_test.rs @@ -26,7 +26,6 @@ mod tests { use base64::Engine; use md5::{Digest as Md5Digest, Md5}; use rustfs_rio::{Checksum, ChecksumType as RioChecksumType}; - use serial_test::serial; use sha2::Sha256; use tracing::info; @@ -90,7 +89,6 @@ mod tests { /// PutObject with Content-MD5: upload succeeds and GetObject returns same content. #[tokio::test] - #[serial] async fn test_put_object_with_content_md5() { init_logging(); info!("TEST: PutObject with Content-MD5"); @@ -126,7 +124,6 @@ mod tests { /// PutObject with x-amz-checksum-sha256: upload succeeds and GetObject returns same content. #[tokio::test] - #[serial] async fn test_put_object_with_checksum_sha256() { init_logging(); info!("TEST: PutObject with x-amz-checksum-sha256"); @@ -164,7 +161,6 @@ mod tests { /// PutObject with a SHA256 checksum that does NOT match the body must be /// rejected (BadDigest / checksum mismatch), NOT accepted with HTTP 200. #[tokio::test] - #[serial] async fn test_put_object_rejects_mismatched_sha256() { init_logging(); info!("TEST: PutObject rejects mismatched x-amz-checksum-sha256 (issue #4341)"); @@ -212,7 +208,6 @@ mod tests { /// After PutObject with a correct SHA256 checksum, HeadObject with /// ChecksumMode=ENABLED must return that stored base64 SHA256 digest. #[tokio::test] - #[serial] async fn test_head_object_returns_stored_sha256() { init_logging(); info!("TEST: HeadObject returns stored SHA256 with ChecksumMode=ENABLED (issue #4341)"); @@ -258,7 +253,6 @@ mod tests { /// Multipart upload with checksum: CreateMultipartUpload, UploadPart(s) with checksum_sha256, CompleteMultipartUpload; then GetObject verifies content. /// Uses part size >= 5MB (server minimum) for two parts. #[tokio::test] - #[serial] async fn test_multipart_upload_with_checksum() { init_logging(); info!("TEST: MultipartUpload with checksum (checksum_sha256 on parts)"); @@ -356,7 +350,6 @@ mod tests { /// Regression test for issue #2282: /// CRC64NVME full-object checksum should match between direct PutObject and multipart upload. #[tokio::test] - #[serial] async fn test_crc64nvme_matches_between_put_object_and_multipart_upload() { init_logging(); info!("TEST: CRC64NVME matches between direct PutObject and multipart upload"); @@ -492,7 +485,6 @@ mod tests { /// value is rejected with BadDigest and nothing is stored. Full HEAD/GET header /// echo round-trip is additionally exercised by the boto3+awscrt e2e. #[tokio::test] - #[serial] async fn test_additional_checksums_verify_on_write() { init_logging(); info!("TEST: additional checksums (XXHash3/64/128, SHA-512, MD5) verify-on-write"); diff --git a/crates/e2e_test/src/cluster_concurrency_test.rs b/crates/e2e_test/src/cluster_concurrency_test.rs index 7702dfac3..cc2f2cddc 100644 --- a/crates/e2e_test/src/cluster_concurrency_test.rs +++ b/crates/e2e_test/src/cluster_concurrency_test.rs @@ -16,7 +16,6 @@ use crate::common::RustFSTestClusterEnvironment; use aws_sdk_s3::Client; use aws_sdk_s3::error::SdkError; use bytes::Bytes; -use serial_test::serial; use std::sync::Arc; use tokio::sync::Barrier; use tracing::{info, warn}; @@ -135,7 +134,6 @@ async fn run_race_iteration( } #[tokio::test] -#[serial] async fn test_conditional_put_race_cluster() -> Result<(), Box> { crate::common::init_logging(); info!("Starting conditional PUT race test with auto cluster"); @@ -192,7 +190,6 @@ async fn test_conditional_put_race_cluster() -> Result<(), Box Result<(), Box> { crate::common::init_logging(); info!("Starting basic conditional PUT test with auto cluster"); diff --git a/crates/e2e_test/src/cluster_multidrive_pool_test.rs b/crates/e2e_test/src/cluster_multidrive_pool_test.rs index 3f9193984..3f88e5af4 100644 --- a/crates/e2e_test/src/cluster_multidrive_pool_test.rs +++ b/crates/e2e_test/src/cluster_multidrive_pool_test.rs @@ -31,7 +31,6 @@ //! (toxiproxy / socket proxy) and 5GiB large-object budgets. use crate::common::{ClusterTopology, RustFSTestClusterEnvironment}; -use serial_test::serial; type TestResult = Result<(), Box>; @@ -58,7 +57,6 @@ async fn put_get_roundtrip(cluster: &RustFSTestClusterEnvironment, key: &str, pa /// 4 nodes x 2 drives, single pool: the multi-drive layout boots and round-trips. #[tokio::test] -#[serial] async fn cluster_multidrive_single_pool_smoke() -> TestResult { crate::common::init_logging(); @@ -81,7 +79,6 @@ async fn cluster_multidrive_single_pool_smoke() -> TestResult { /// Two single-node pools, 2 drives each: the multi-pool layout boots and /// round-trips. Every pool is a distinct erasure pool (`pool_idx` 0 and 1). #[tokio::test] -#[serial] async fn cluster_two_pool_smoke() -> TestResult { crate::common::init_logging(); diff --git a/crates/e2e_test/src/compression_test.rs b/crates/e2e_test/src/compression_test.rs index 775decfa6..7014f5f18 100644 --- a/crates/e2e_test/src/compression_test.rs +++ b/crates/e2e_test/src/compression_test.rs @@ -3,7 +3,6 @@ use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; -use serial_test::serial; use std::fs; use std::path::PathBuf; use std::process::Command; @@ -102,7 +101,6 @@ async fn start_rustfs_with_compression(env: &mut RustFSTestEnvironment) -> Resul } #[tokio::test] -#[serial] async fn test_compression_roundtrip() -> Result<(), Box> { init_logging(); info!("Starting compression roundtrip test"); @@ -230,7 +228,6 @@ async fn fetch_range( /// (rustfs/rustfs#5957: multipart uploads previously bypassed disk compression /// entirely). #[tokio::test] -#[serial] async fn test_compression_multipart_roundtrip() -> Result<(), Box> { init_logging(); info!("Starting multipart compression roundtrip test"); @@ -349,7 +346,6 @@ const MPU_HIGH_RATIO_BUCKET: &str = "compression-mpu-high-ratio-bucket"; /// reproduced the mid-payload Pending truncation (rustfs/rustfs#5957). Every GET shape must return /// the exact original bytes, and the stored size must show the data really was compressed. #[tokio::test] -#[serial] async fn test_compression_multipart_high_ratio_binary_roundtrip() -> Result<(), Box> { init_logging(); info!("Starting multipart high-ratio binary compression roundtrip test"); @@ -446,7 +442,6 @@ const MPU_COPY_RANGE_LEN: usize = 5 * 1024 * 1024; /// range must be decompressed on read and re-compressed into the destination part, so the final /// object has to match "source prefix + uploaded tail" byte for byte. #[tokio::test] -#[serial] async fn test_compression_multipart_upload_part_copy_roundtrip() -> Result<(), Box> { init_logging(); info!("Starting multipart upload-part-copy compression roundtrip test"); @@ -570,7 +565,6 @@ const MPU_THREE_PARTS_TAIL_SIZE: usize = 512 * 1024; /// Three-part upload with uneven part sizes: each partNumber GET must map back to exactly one /// compressed part stream, and a suffix range must resolve inside the trailing part. #[tokio::test] -#[serial] async fn test_compression_multipart_three_parts_part_number_gets() -> Result<(), Box> { init_logging(); info!("Starting three-part multipart compression partNumber test"); @@ -689,7 +683,6 @@ async fn start_rustfs_with_compression_and_sse( /// shape must still return the original plaintext bytes. Physical size must shrink because the /// compression runs before encryption. #[tokio::test] -#[serial] async fn test_compression_multipart_sse_s3_roundtrip() -> Result<(), Box> { use aws_sdk_s3::types::ServerSideEncryption; diff --git a/crates/e2e_test/src/connection_cap_test.rs b/crates/e2e_test/src/connection_cap_test.rs index bb835a712..b59584164 100644 --- a/crates/e2e_test/src/connection_cap_test.rs +++ b/crates/e2e_test/src/connection_cap_test.rs @@ -18,7 +18,6 @@ //! concurrency — a queued connection is served only after a held one closes. use crate::common::{RustFSTestEnvironment, init_logging}; -use serial_test::serial; use std::time::Duration; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; @@ -57,7 +56,6 @@ async fn read_response_head(stream: &mut TcpStream, dur: Duration) -> Option TestResult { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -89,7 +87,6 @@ async fn open_and_stall(addr: &str) -> std::io::Result { } #[tokio::test] -#[serial] async fn connection_cap_blocks_excess_connections_until_permits_free() -> TestResult { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/console_smoke_test.rs b/crates/e2e_test/src/console_smoke_test.rs index cca86b7e3..431eee313 100644 --- a/crates/e2e_test/src/console_smoke_test.rs +++ b/crates/e2e_test/src/console_smoke_test.rs @@ -33,7 +33,6 @@ //! serve the unauthenticated console endpoints at all. use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; -use serial_test::serial; use std::error::Error; use tokio::time::{Duration, sleep}; @@ -58,7 +57,6 @@ async fn wait_for_console_ready(console_base: &str) -> Result TestResult { init_logging(); diff --git a/crates/e2e_test/src/content_encoding_test.rs b/crates/e2e_test/src/content_encoding_test.rs index f99117424..abd0bb1ba 100644 --- a/crates/e2e_test/src/content_encoding_test.rs +++ b/crates/e2e_test/src/content_encoding_test.rs @@ -22,12 +22,10 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use tracing::info; /// Verify Content-Encoding header roundtrips through PUT, GET, and HEAD operations #[tokio::test] - #[serial] async fn test_content_encoding_roundtrip() { init_logging(); info!("Starting Content-Encoding roundtrip test"); @@ -105,7 +103,6 @@ mod tests { /// Issue #1857: Content-Encoding "aws-chunked" is used by SigV4 streaming clients and must /// not be stored or returned. Upload with aws-chunked and verify GET/HEAD do not return it. #[tokio::test] - #[serial] async fn test_content_encoding_aws_chunked_not_returned_issue_1857() { init_logging(); info!("Issue #1857: aws-chunked must not be persisted or returned"); @@ -161,7 +158,6 @@ mod tests { /// Issue #2475 / Route A: when aws-chunked is combined with an effective object encoding, /// only the effective encoding should roundtrip through GET/HEAD. #[tokio::test] - #[serial] async fn test_content_encoding_aws_chunked_with_effective_encoding_roundtrip() { init_logging(); info!("aws-chunked,gzip should persist only gzip"); diff --git a/crates/e2e_test/src/copy_object_checksum_test.rs b/crates/e2e_test/src/copy_object_checksum_test.rs index bdd6251a9..b4ac11bbb 100644 --- a/crates/e2e_test/src/copy_object_checksum_test.rs +++ b/crates/e2e_test/src/copy_object_checksum_test.rs @@ -30,7 +30,6 @@ mod tests { use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; use rustfs_rio::{Checksum, ChecksumType as RioChecksumType}; - use serial_test::serial; use sha2::{Digest, Sha256}; use tracing::info; @@ -114,7 +113,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_copy_supports_all_checksum_algorithms() { init_logging(); @@ -196,7 +194,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_copy_without_algorithm_preserves_every_supported_source_checksum() { init_logging(); @@ -262,7 +259,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_copy_without_algorithm_preserves_composite_checksum_type() { init_logging(); @@ -352,7 +348,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_copy_rejects_unknown_algorithm_without_destination_mutation() { init_logging(); @@ -453,7 +448,6 @@ mod tests { /// bytes, return it in `CopyObjectResult.ChecksumSHA256`, and persist it so a checksum-mode /// HEAD on the destination returns the identical value. #[tokio::test] - #[serial] async fn test_copy_with_checksum_algorithm_returns_and_persists_sha256() { init_logging(); info!("Issue #4996: CopyObject with ChecksumAlgorithm=SHA256 must return and persist the checksum"); @@ -523,7 +517,6 @@ mod tests { /// No algorithm requested: when the source object already carries a checksum, the copy must /// preserve it on the destination (AWS default), visible via a checksum-mode HEAD. #[tokio::test] - #[serial] async fn test_copy_without_algorithm_preserves_source_checksum() { init_logging(); info!("Issue #4996: CopyObject without ChecksumAlgorithm must preserve the source object's checksum"); @@ -603,7 +596,6 @@ mod tests { /// checksum-not-inherited path, and exercises the CRC32 code path (a different branch of /// ChecksumType::from_string than SHA256). #[tokio::test] - #[serial] async fn test_copy_requested_algorithm_overrides_source_checksum() { init_logging(); info!("Issue #4996: a requested CopyObject checksum algorithm must override the source object's algorithm"); diff --git a/crates/e2e_test/src/copy_object_metadata_test.rs b/crates/e2e_test/src/copy_object_metadata_test.rs index f56fe7f41..289c2abc2 100644 --- a/crates/e2e_test/src/copy_object_metadata_test.rs +++ b/crates/e2e_test/src/copy_object_metadata_test.rs @@ -22,11 +22,9 @@ mod tests { use aws_sdk_s3::types::{ BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, MetadataDirective, StorageClass, VersioningConfiguration, }; - use serial_test::serial; use tracing::info; #[tokio::test] - #[serial] async fn copy_object_standard_metadata_copy_replace_and_clear() { init_logging(); info!("Issue #2789: self-copy metadata replacement must preserve object data"); @@ -300,7 +298,6 @@ mod tests { } #[tokio::test] - #[serial] async fn copy_object_replace_accepts_each_standard_field_independently() { init_logging(); let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); @@ -416,7 +413,6 @@ mod tests { } #[tokio::test] - #[serial] async fn copy_object_replace_handles_versioned_multipart_source() { init_logging(); let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); @@ -530,7 +526,6 @@ mod tests { } #[tokio::test] - #[serial] async fn invalid_replacement_metadata_does_not_mutate_destination() { init_logging(); let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment"); diff --git a/crates/e2e_test/src/copy_object_tagging_test.rs b/crates/e2e_test/src/copy_object_tagging_test.rs index 23861a705..ad1103658 100644 --- a/crates/e2e_test/src/copy_object_tagging_test.rs +++ b/crates/e2e_test/src/copy_object_tagging_test.rs @@ -21,7 +21,6 @@ mod tests { use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, MetadataDirective, TaggingDirective, VersioningConfiguration}; - use serial_test::serial; use std::collections::BTreeMap; async fn object_tags(client: &Client, bucket: &str, key: &str) -> BTreeMap { @@ -39,7 +38,6 @@ mod tests { } #[tokio::test] - #[serial] async fn copy_object_applies_copy_replace_and_empty_tagging_directives() { init_logging(); let mut env = RustFSTestEnvironment::new() @@ -305,7 +303,6 @@ mod tests { } #[tokio::test] - #[serial] async fn copy_object_tag_replacement_honors_request_tag_policy_denial() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/copy_object_version_restore_test.rs b/crates/e2e_test/src/copy_object_version_restore_test.rs index 25958bc35..e99da5843 100644 --- a/crates/e2e_test/src/copy_object_version_restore_test.rs +++ b/crates/e2e_test/src/copy_object_version_restore_test.rs @@ -21,11 +21,9 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; - use serial_test::serial; use tracing::info; #[tokio::test] - #[serial] async fn test_self_copy_of_historical_version_restores_data_and_metadata() { init_logging(); info!("Issue #4238: self-copy of a historical version must be allowed and preserve metadata"); @@ -165,7 +163,6 @@ mod tests { /// version copied via `x-amz-copy-source-version-id` (SDK `CopySourceVersionId`), kept distinct /// from the newly created destination `x-amz-version-id`. #[tokio::test] - #[serial] async fn test_copy_of_non_latest_source_version_returns_copy_source_version_id() { init_logging(); info!("Issue #4976: versioned CopyObject must return x-amz-copy-source-version-id for the exact source version"); diff --git a/crates/e2e_test/src/copy_source_invalid_date_test.rs b/crates/e2e_test/src/copy_source_invalid_date_test.rs index 1513dddce..09f15a31b 100644 --- a/crates/e2e_test/src/copy_source_invalid_date_test.rs +++ b/crates/e2e_test/src/copy_source_invalid_date_test.rs @@ -47,7 +47,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; - use serial_test::serial; use std::error::Error; /// Signed raw `PUT` copy request with an explicit copy-source conditional @@ -84,7 +83,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_copy_source_if_unmodified_since_valid_and_invalid() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/create_bucket_region_test.rs b/crates/e2e_test/src/create_bucket_region_test.rs index 44f3ea13b..7f19b2297 100644 --- a/crates/e2e_test/src/create_bucket_region_test.rs +++ b/crates/e2e_test/src/create_bucket_region_test.rs @@ -25,13 +25,11 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::types::{BucketLocationConstraint, CreateBucketConfiguration}; - use serial_test::serial; use std::error::Error; /// `CreateBucket` with a `LocationConstraint` body must pass SigV4 validation /// and create the bucket, mirroring `minio-go` `MakeBucket(bucket, "us-east-1")`. #[tokio::test] - #[serial] async fn test_create_bucket_with_us_east_1_location_constraint() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -62,7 +60,6 @@ mod tests { /// A plain `CreateBucket` (no body) must also succeed; guards against a /// regression where an empty body would be hashed incorrectly during SigV4. #[tokio::test] - #[serial] async fn test_create_bucket_without_location_constraint() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/data_usage_test.rs b/crates/e2e_test/src/data_usage_test.rs index d99d0750a..2dfd56eb5 100644 --- a/crates/e2e_test/src/data_usage_test.rs +++ b/crates/e2e_test/src/data_usage_test.rs @@ -15,7 +15,6 @@ use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; use rustfs_data_usage::DataUsageInfo; -use serial_test::serial; use tokio::time::{Duration, sleep}; use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestEnvironment, TEST_BUCKET, awscurl_get, init_logging}; @@ -60,7 +59,6 @@ where /// Regression test for data usage accuracy (issue #1012). /// Launches rustfs, writes 1000 objects, then asserts admin data usage reports the full count. #[tokio::test(flavor = "multi_thread")] -#[serial] #[ignore = "Starts a rustfs server and requires awscurl; enable when running full E2E"] async fn data_usage_reports_all_objects() -> Result<(), Box> { init_logging(); @@ -118,7 +116,6 @@ async fn data_usage_reports_all_objects() -> Result<(), Box Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/delete_marker_migration_semantics_test.rs b/crates/e2e_test/src/delete_marker_migration_semantics_test.rs index 5a51c053a..8ae3b75c3 100644 --- a/crates/e2e_test/src/delete_marker_migration_semantics_test.rs +++ b/crates/e2e_test/src/delete_marker_migration_semantics_test.rs @@ -18,7 +18,6 @@ mod tests { use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; - use serial_test::serial; async fn create_versioned_bucket(client: &Client, bucket: &str) { client @@ -72,7 +71,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_versioning_only_delete_marker_has_minio_compatible_visibility_for_migration_proof() { init_logging(); let mut env = RustFSTestEnvironment::new().await.expect("create test environment"); @@ -113,7 +111,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_versioning_delete_marker_plus_history_remains_visible_for_migration_proof() { init_logging(); let mut env = RustFSTestEnvironment::new().await.expect("create test environment"); diff --git a/crates/e2e_test/src/delete_object_no_content_length_test.rs b/crates/e2e_test/src/delete_object_no_content_length_test.rs index 76bee5e3d..6f74ebf3a 100644 --- a/crates/e2e_test/src/delete_object_no_content_length_test.rs +++ b/crates/e2e_test/src/delete_object_no_content_length_test.rs @@ -24,7 +24,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; - use serial_test::serial; use std::error::Error; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; @@ -92,7 +91,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_delete_object_version_without_content_length_succeeds() -> Result<(), Box> { init_logging(); info!("🧪 TEST: signed DELETE Object?versionId succeeds without Content-Length"); diff --git a/crates/e2e_test/src/delete_objects_versioning_test.rs b/crates/e2e_test/src/delete_objects_versioning_test.rs index 41f872370..acab0d320 100644 --- a/crates/e2e_test/src/delete_objects_versioning_test.rs +++ b/crates/e2e_test/src/delete_objects_versioning_test.rs @@ -29,7 +29,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::Client; use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration}; - use serial_test::serial; use tracing::info; fn create_s3_client(env: &RustFSTestEnvironment) -> Client { @@ -42,7 +41,6 @@ mod tests { /// a versioned bucket, calling `list_object_versions` **immediately** (with /// no sleep) returns the newly-created DeleteMarker with `is_latest = true`. #[tokio::test] - #[serial] async fn test_delete_objects_delete_marker_immediately_visible() { init_logging(); info!("🧪 TEST: DeleteMarker from delete_objects is immediately visible via list_object_versions"); @@ -190,7 +188,6 @@ mod tests { /// a single `delete_objects` call all have their delete markers visible /// immediately afterwards. #[tokio::test] - #[serial] async fn test_delete_objects_multiple_keys_delete_markers_immediately_visible() { init_logging(); info!("🧪 TEST: Multiple delete markers from delete_objects are immediately visible"); diff --git a/crates/e2e_test/src/delete_regression_test.rs b/crates/e2e_test/src/delete_regression_test.rs index 1f61a55c2..3bcc6fa5c 100644 --- a/crates/e2e_test/src/delete_regression_test.rs +++ b/crates/e2e_test/src/delete_regression_test.rs @@ -33,7 +33,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration}; - use serial_test::serial; use std::error::Error; use tracing::info; @@ -51,7 +50,6 @@ mod tests { /// 4. Verify the object is NOT in LIST /// 5. Verify HEAD returns 404 #[tokio::test] - #[serial] async fn test_delete_removes_object_from_list() -> TestResult { init_logging(); info!("RT-05: delete removes object from list"); @@ -132,7 +130,6 @@ mod tests { /// Regression pattern: batch delete returns success but some objects /// remain in LIST. #[tokio::test] - #[serial] async fn test_batch_delete_removes_all_objects() -> TestResult { init_logging(); info!("RT-05c: batch delete removes all objects"); @@ -212,7 +209,6 @@ mod tests { /// Covers the pattern where permanent deletion of a specific version /// fails with FileAccessDenied (rustfs#4978). #[tokio::test] - #[serial] async fn test_versioned_permanent_delete() -> TestResult { init_logging(); info!("RT-05d: versioned permanent delete"); @@ -283,7 +279,6 @@ mod tests { /// Covers the pattern where creating a delete marker and then listing /// versions shows incorrect state (rustfs#760). #[tokio::test] - #[serial] async fn test_versioned_delete_marker_and_list_consistency() -> TestResult { init_logging(); info!("RT-05e: versioned delete marker and list consistency"); @@ -379,7 +374,6 @@ mod tests { /// Regression pattern: after delete, the object data files remain on disk /// (rustfs#5029: Node Does Not Remove Files After Reconnect). #[tokio::test] - #[serial] async fn test_delete_removes_object_head_returns_404() -> TestResult { init_logging(); info!("RT-05f: delete → HEAD 404 consistency"); diff --git a/crates/e2e_test/src/distributed_startup_regression_test.rs b/crates/e2e_test/src/distributed_startup_regression_test.rs index b7387b50b..ff2da755b 100644 --- a/crates/e2e_test/src/distributed_startup_regression_test.rs +++ b/crates/e2e_test/src/distributed_startup_regression_test.rs @@ -32,7 +32,6 @@ mod tests { use crate::common::{RustFSTestClusterEnvironment, init_logging}; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use std::error::Error; use tokio::time::{Duration, sleep}; use tracing::info; @@ -50,7 +49,6 @@ mod tests { /// 3. Verify all nodes report healthy /// 4. Verify S3 operations work through any node #[tokio::test] - #[serial] async fn test_four_node_cluster_startup_and_health() -> TestResult { init_logging(); info!("RT-10: 4-node cluster startup and health"); @@ -103,7 +101,6 @@ mod tests { /// Regression pattern: after a node restart, it cannot rejoin the cluster /// or enters a faulty state (rustfs#2601). #[tokio::test] - #[serial] async fn test_cluster_survives_node_restart() -> TestResult { init_logging(); info!("RT-10b: cluster survives node restart"); @@ -168,7 +165,6 @@ mod tests { /// Regression pattern: bucket metadata is not replicated to all nodes, /// causing NoSuchBucket errors on some nodes (rustfs#3191). #[tokio::test] - #[serial] async fn test_bucket_visible_from_all_nodes() -> TestResult { init_logging(); info!("RT-10c: bucket visible from all nodes"); diff --git a/crates/e2e_test/src/existing_object_tag_policy_test.rs b/crates/e2e_test/src/existing_object_tag_policy_test.rs index 9a26e95db..17c34f166 100644 --- a/crates/e2e_test/src/existing_object_tag_policy_test.rs +++ b/crates/e2e_test/src/existing_object_tag_policy_test.rs @@ -23,7 +23,6 @@ use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging}; use aws_sdk_s3::{Client, Config}; -use serial_test::serial; use tracing::info; use uuid::Uuid; @@ -174,7 +173,6 @@ async fn cleanup_bucket_and_object(admin: &Client, bucket: &str, key: &str) { /// IAM identity policy: GetObject allowed only when `s3:ExistingObjectTag/security` == `public`. #[tokio::test] -#[serial] async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box> { init_logging(); if !awscurl_available() { @@ -233,7 +231,6 @@ async fn test_e2e_iam_policy_existing_object_tag_get_object() -> Result<(), Box< /// Bucket policy: same `ExistingObjectTag` condition; user has no canned IAM policy attached. #[tokio::test] -#[serial] async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), Box> { init_logging(); if !awscurl_available() { @@ -295,7 +292,6 @@ async fn test_e2e_bucket_policy_existing_object_tag_get_object() -> Result<(), B /// STS `AssumeRole` with inline `Policy` (session policy): GetObject only when `ExistingObjectTag/security` is `public`. #[tokio::test] -#[serial] async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result<(), Box> { init_logging(); if !awscurl_available() { @@ -372,7 +368,6 @@ async fn test_e2e_sts_assume_role_session_policy_existing_object_tag() -> Result /// STS inline session policy: DeleteObjects must evaluate `s3:DeleteObject` per requested object key. #[tokio::test] -#[serial] async fn test_e2e_sts_session_policy_delete_objects_object_prefix_only() -> Result<(), Box> { init_logging(); if !awscurl_available() { diff --git a/crates/e2e_test/src/get_codec_streaming_compat_test.rs b/crates/e2e_test/src/get_codec_streaming_compat_test.rs index eb758aa7f..4fdc8326f 100644 --- a/crates/e2e_test/src/get_codec_streaming_compat_test.rs +++ b/crates/e2e_test/src/get_codec_streaming_compat_test.rs @@ -66,7 +66,6 @@ mod tests { use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; - use serial_test::serial; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; use std::error::Error; @@ -277,7 +276,6 @@ mod tests { } #[tokio::test] - #[serial] async fn codec_streaming_matches_legacy_duplex_body_and_headers() -> TestResult { init_logging(); diff --git a/crates/e2e_test/src/group_delete_test.rs b/crates/e2e_test/src/group_delete_test.rs index 147feb89d..38d053f99 100644 --- a/crates/e2e_test/src/group_delete_test.rs +++ b/crates/e2e_test/src/group_delete_test.rs @@ -17,7 +17,6 @@ use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_put, init_logging}; use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::{Client, Config}; -use serial_test::serial; use tracing::info; fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client { @@ -84,7 +83,6 @@ async fn update_group_members_rejects_invalid_new_group_names() -> Result<(), Bo /// Test that deleting a group with members fails, and deleting an empty group succeeds. #[tokio::test(flavor = "multi_thread")] -#[serial] #[ignore = "requires awscurl and spawns a real RustFS server"] async fn test_delete_group_requires_empty_membership() -> Result<(), Box> { init_logging(); @@ -144,7 +142,6 @@ async fn test_delete_group_requires_empty_membership() -> Result<(), Box Result<(), Box> { init_logging(); @@ -212,7 +209,6 @@ async fn test_user_with_only_group_gets_group_policies() -> Result<(), Box Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/head_object_consistency_test.rs b/crates/e2e_test/src/head_object_consistency_test.rs index 93e7b8a4e..a7499224b 100644 --- a/crates/e2e_test/src/head_object_consistency_test.rs +++ b/crates/e2e_test/src/head_object_consistency_test.rs @@ -16,7 +16,6 @@ use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; use aws_sdk_s3::presigning::PresigningConfig; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; -use serial_test::serial; use std::time::Duration; use tracing::info; @@ -29,7 +28,6 @@ fn list_contains_key(output: &aws_sdk_s3::operation::list_objects_v2::ListObject } #[tokio::test] -#[serial] async fn head_object_consistency_after_write_and_multipart_and_presigned_head() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/head_object_range_test.rs b/crates/e2e_test/src/head_object_range_test.rs index 5cc4b7a12..577440020 100644 --- a/crates/e2e_test/src/head_object_range_test.rs +++ b/crates/e2e_test/src/head_object_range_test.rs @@ -1,6 +1,5 @@ use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::primitives::ByteStream; -use serial_test::serial; use tracing::info; const RANGE_HEAD_BUCKET: &str = "range-head-test-bucket"; @@ -8,7 +7,6 @@ const RANGE_HEAD_KEY: &str = "range-head-object.bin"; const ACCEPT_RANGES_BYTES: &str = "bytes"; #[tokio::test] -#[serial] async fn head_object_advertises_accept_ranges() -> Result<(), Box> { init_logging(); info!("Starting HeadObject Accept-Ranges regression test"); diff --git a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs index 749f21d7f..dac1eef5e 100644 --- a/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs +++ b/crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs @@ -19,7 +19,6 @@ mod tests { use crate::chaos::signed_admin_post; use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging}; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use std::collections::HashSet; use std::error::Error; use std::path::{Path, PathBuf}; @@ -63,7 +62,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_auto_heal_rebuilds_runtime_wiped_disk_without_restart() { init_logging(); info!("Issue #1533: auto heal should rebuild a runtime-wiped disk in a 4-disk single-node erasure set without restart"); @@ -182,7 +180,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_admin_deep_heal_rebuilds_cleared_disk_in_single_node_erasure_set() { init_logging(); info!("Discussion #2964: admin deep heal should rebuild a wiped disk in a 4-disk single-node erasure set"); @@ -332,7 +329,6 @@ mod tests { } #[tokio::test(flavor = "multi_thread")] - #[serial] async fn test_cluster_root_heal_rebuilds_replaced_remote_disk() -> Result<(), Box> { init_logging(); info!("Root recursive heal should rebuild data on a remote node after its disk is replaced and the node rejoins"); @@ -444,7 +440,6 @@ mod tests { /// topology early-return or the merge hard-fail) turns the down-window /// response into a 500 and fails this test. #[tokio::test] - #[serial] async fn test_background_heal_status_degrades_while_peer_down_and_recovers_after_rejoin() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs b/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs index c7632d013..bb751fe55 100644 --- a/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs +++ b/crates/e2e_test/src/internode_rpc_signature_e2e_test.rs @@ -101,7 +101,6 @@ use rustfs_config::{ }; use rustfs_protos::canonical_make_volume_request_body; use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse, PingRequest, PingResponse}; -use serial_test::serial; use sha2::{Digest, Sha256}; use std::error::Error; use tonic::{Code, Request, Response, Status}; @@ -397,7 +396,6 @@ fn assert_rejected(result: Result, expected: Code, e /// Grouped into one server start because each case is independent and spawning /// a `rustfs` process per assertion would dominate the runtime. #[tokio::test] -#[serial] async fn internode_rpc_signature_default_posture_e2e() -> TestResult { init_logging(); align_rpc_secret_with_server(); @@ -424,7 +422,6 @@ async fn internode_rpc_signature_default_posture_e2e() -> TestResult { /// epoch is learned from a real response, then the same server is restarted in place to prove its /// replacement epoch rejects the captured request even though the nonce cache is necessarily new. #[tokio::test] -#[serial] async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> TestResult { init_logging(); align_rpc_secret_with_server(); @@ -497,7 +494,6 @@ async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> Te /// A mutating v2 request cannot use that lane; once the epoch proof is returned, the first v3 /// mutation succeeds. This protects a server restart without reopening a general downgrade path. #[tokio::test] -#[serial] async fn replay_scope_strict_requires_v3_after_ping_bootstrap_e2e() -> TestResult { init_logging(); align_rpc_secret_with_server(); @@ -704,7 +700,6 @@ async fn legacy_only_signature_is_accepted_in_default_posture(url: &str) { /// /// The paired v2 positive control rules out "strict simply breaks everything". #[tokio::test] -#[serial] async fn signature_strict_rejects_legacy_only_downgrade() -> TestResult { init_logging(); align_rpc_secret_with_server(); @@ -741,7 +736,6 @@ async fn signature_strict_rejects_legacy_only_downgrade() -> TestResult { /// takes the still-open legacy lane), which is what pins the rejection to the /// handler's digest gate; the cited message confirms which check spoke. #[tokio::test] -#[serial] async fn body_digest_strict_rejects_digestless_mutation() -> TestResult { init_logging(); align_rpc_secret_with_server(); diff --git a/crates/e2e_test/src/kms/bucket_default_encryption_test.rs b/crates/e2e_test/src/kms/bucket_default_encryption_test.rs index 965b33d02..fecba2b89 100644 --- a/crates/e2e_test/src/kms/bucket_default_encryption_test.rs +++ b/crates/e2e_test/src/kms/bucket_default_encryption_test.rs @@ -27,12 +27,10 @@ use aws_sdk_s3::types::{ ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, }; use rustfs_rio::{Checksum, ChecksumType}; -use serial_test::serial; use tracing::{debug, info, warn}; /// Test 1: When bucket is configured with default SSE-S3 encryption, put_object should automatically apply encryption #[tokio::test] -#[serial] async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box> { init_logging(); info!("Testing bucket default SSE-S3 encryption impact on put_object"); @@ -155,7 +153,6 @@ async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box Result<(), Box> { init_logging(); info!("Testing bucket default SSE-KMS encryption impact on put_object"); @@ -275,7 +272,6 @@ async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box Result<(), Box> { init_logging(); info!("Testing bucket default encryption impact on create_multipart_upload"); @@ -473,7 +469,6 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box Result<(), Box> { init_logging(); info!("Testing explicitly specified encryption parameters override bucket default configuration"); @@ -569,7 +564,6 @@ async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box Result<(), Box> { init_logging(); info!("Testing SSE-KMS without explicit key ID populates default key"); diff --git a/crates/e2e_test/src/kms/configured_roundtrip_test.rs b/crates/e2e_test/src/kms/configured_roundtrip_test.rs index a666ef8d7..512105d4f 100644 --- a/crates/e2e_test/src/kms/configured_roundtrip_test.rs +++ b/crates/e2e_test/src/kms/configured_roundtrip_test.rs @@ -20,7 +20,6 @@ use super::common::{ }; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, ServerSideEncryption, VersioningConfiguration}; -use serial_test::serial; use std::error::Error; use uuid::Uuid; @@ -386,7 +385,6 @@ async fn assert_versioned_sse_kms_roundtrip_and_cleanup( } #[tokio::test] -#[serial] async fn test_configured_local_kms_admin_and_versioned_cleanup() -> TestResult { let mut env = LocalKMSTestEnvironment::new().await?; env.base_env.start_rustfs_server(Vec::new()).await?; @@ -434,7 +432,6 @@ async fn test_configured_local_kms_admin_and_versioned_cleanup() -> TestResult { } #[tokio::test] -#[serial] #[ignore = "requires a Vault binary"] async fn test_configured_vault_kms_admin_and_versioned_cleanup() -> TestResult { let mut env = VaultTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs b/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs index 85abf316e..1cf19a565 100644 --- a/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs +++ b/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs @@ -32,11 +32,9 @@ use aws_sdk_s3::types::{ MetadataDirective, ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, }; -use serial_test::serial; use tracing::info; #[tokio::test] -#[serial] async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() { init_logging(); info!("same-key CopyObject with REPLACE metadata must not re-key an SSE-S3 object"); @@ -136,7 +134,6 @@ async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() { } #[tokio::test] -#[serial] async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() { init_logging(); info!("same-key CopyObject that drops SSE must rewrite the data, not orphan the ciphertext"); @@ -233,7 +230,6 @@ async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() { } #[tokio::test] -#[serial] async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decryptable() { init_logging(); info!("bucket default encryption must also keep a same-key copy off the metadata-only path"); diff --git a/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs b/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs index 34dd682d7..3241a217d 100644 --- a/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs +++ b/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs @@ -25,11 +25,9 @@ use super::common::{LocalKMSTestEnvironment, create_key_with_specific_id}; use crate::common::init_logging; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, ServerSideEncryption, VersioningConfiguration}; -use serial_test::serial; use tracing::info; #[tokio::test] -#[serial] async fn test_self_copy_of_historical_sse_s3_version_is_readable() { init_logging(); info!("Issue #4238 (SSE): restoring an encrypted historical version must stay decryptable"); diff --git a/crates/e2e_test/src/kms/encryption_metadata_test.rs b/crates/e2e_test/src/kms/encryption_metadata_test.rs index e1e082df8..a316668f6 100644 --- a/crates/e2e_test/src/kms/encryption_metadata_test.rs +++ b/crates/e2e_test/src/kms/encryption_metadata_test.rs @@ -22,7 +22,6 @@ use aws_sdk_s3::types::{ CompletedMultipartUpload, CompletedPart, ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule, }; -use serial_test::serial; use std::collections::{HashMap, VecDeque}; use tracing::info; @@ -82,7 +81,6 @@ pub(super) fn assert_storage_encrypted(storage_root: &std::path::Path, bucket: & } #[tokio::test] -#[serial] async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box> { init_logging(); info!("Validating SSE-S3 managed encryption metadata exposure"); @@ -143,7 +141,6 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box Result<(), Box> { init_logging(); info!("Validating SSE-KMS managed encryption metadata (including copy)"); @@ -247,7 +244,6 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(), } #[tokio::test] -#[serial] async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box> { init_logging(); info!("Validating ciphertext persistence for multipart SSE-KMS uploads"); diff --git a/crates/e2e_test/src/kms/kms_authorization_negative_matrix_test.rs b/crates/e2e_test/src/kms/kms_authorization_negative_matrix_test.rs index bc6ef5b24..a6fd6cb43 100644 --- a/crates/e2e_test/src/kms/kms_authorization_negative_matrix_test.rs +++ b/crates/e2e_test/src/kms/kms_authorization_negative_matrix_test.rs @@ -35,7 +35,6 @@ use aws_sdk_s3::config::{Config, Credentials, Region}; use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::ServerSideEncryption; -use serial_test::serial; use std::time::Duration; use tracing::info; @@ -209,7 +208,6 @@ fn disable_body(key_id: &str) -> String { /// Data-path matrix: SSE-KMS writes and reads are authorized against the resolved key. #[tokio::test] -#[serial] async fn sse_kms_per_key_authorization_negative_matrix() -> TestResult { init_logging(); @@ -355,7 +353,6 @@ async fn sse_kms_per_key_authorization_negative_matrix() -> TestResult { /// Runs without the SSE enforcement switch: admin scoping is unconditional, and /// leaving the switch off proves the two planes are independent. #[tokio::test] -#[serial] async fn kms_admin_per_key_authorization_negative_matrix() -> TestResult { init_logging(); diff --git a/crates/e2e_test/src/kms/kms_comprehensive_test.rs b/crates/e2e_test/src/kms/kms_comprehensive_test.rs index b4a61a7f6..584ae64eb 100644 --- a/crates/e2e_test/src/kms/kms_comprehensive_test.rs +++ b/crates/e2e_test/src/kms/kms_comprehensive_test.rs @@ -24,13 +24,11 @@ use super::common::{ test_sse_kms_encryption, test_sse_s3_encryption, }; use crate::common::{TEST_BUCKET, init_logging}; -use serial_test::serial; use tokio::time::{Duration, sleep}; use tracing::info; /// Comprehensive test: Full KMS workflow with all encryption types #[tokio::test] -#[serial] async fn test_comprehensive_kms_full_workflow() -> Result<(), Box> { init_logging(); info!("🏁 Start the KMS full-featured synthesis test"); @@ -99,7 +97,6 @@ async fn test_mixed_encryption_workload( /// Comprehensive stress test: Large dataset with multiple encryption types #[tokio::test] -#[serial] async fn test_comprehensive_stress_test() -> Result<(), Box> { init_logging(); info!("💪 Start the KMS stress test"); @@ -134,7 +131,6 @@ async fn test_comprehensive_stress_test() -> Result<(), Box Result<(), Box> { init_logging(); info!("🔐 Begin the comprehensive test of encryption key isolation"); @@ -206,7 +202,6 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box Result<(), Box> { init_logging(); info!("⚡ Started comprehensive testing of concurrent encryption operations"); @@ -252,7 +247,6 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box Result<(), Box> { init_logging(); info!("📊 Start KMS performance benchmarking"); diff --git a/crates/e2e_test/src/kms/kms_edge_cases_test.rs b/crates/e2e_test/src/kms/kms_edge_cases_test.rs index fe4dcb66d..9d9d1a765 100644 --- a/crates/e2e_test/src/kms/kms_edge_cases_test.rs +++ b/crates/e2e_test/src/kms/kms_edge_cases_test.rs @@ -26,7 +26,6 @@ use crate::common::{TEST_BUCKET, init_logging}; use aws_sdk_s3::types::ServerSideEncryption; use base64::Engine; use md5::{Digest as Md5Digest, Md5}; -use serial_test::serial; use std::sync::Arc; use tokio::sync::Semaphore; use tracing::{info, warn}; @@ -39,7 +38,6 @@ fn md5_hex(input: impl AsRef<[u8]>) -> String { /// Test encryption of zero-byte files (empty files) #[tokio::test] -#[serial] async fn test_kms_zero_byte_file_encryption() -> Result<(), Box> { init_logging(); info!("🧪 Testing KMS encryption with zero-byte files"); @@ -113,7 +111,6 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box Result<(), Box> { init_logging(); info!("🧪 Testing KMS encryption with single-byte files"); @@ -206,7 +203,6 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box Result<(), Box> { init_logging(); info!("🧪 Testing KMS multipart upload boundary conditions"); @@ -282,7 +278,6 @@ async fn test_kms_multipart_boundary_conditions() -> Result<(), Box Result<(), Box> { init_logging(); info!("🧪 Testing KMS invalid key scenarios and error handling"); @@ -370,7 +365,6 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box Result<(), Box> { init_logging(); info!("🧪 Testing KMS concurrent encryption operations"); @@ -478,7 +472,6 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box Result<(), Box> { init_logging(); info!("🧪 Testing KMS key validation and security properties"); diff --git a/crates/e2e_test/src/kms/kms_fault_recovery_test.rs b/crates/e2e_test/src/kms/kms_fault_recovery_test.rs index 2325281ee..fa6c08170 100644 --- a/crates/e2e_test/src/kms/kms_fault_recovery_test.rs +++ b/crates/e2e_test/src/kms/kms_fault_recovery_test.rs @@ -24,7 +24,6 @@ use super::common::LocalKMSTestEnvironment; use crate::common::{TEST_BUCKET, init_logging}; use aws_sdk_s3::types::ServerSideEncryption; -use serial_test::serial; use std::fs; use std::time::Duration; use tokio::time::sleep; @@ -32,7 +31,6 @@ use tracing::{info, warn}; /// Test KMS behavior when key directory is temporarily unavailable #[tokio::test] -#[serial] async fn test_kms_key_directory_unavailable() -> Result<(), Box> { init_logging(); info!("🧪 Testing KMS behavior with unavailable key directory"); @@ -123,7 +121,6 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box Result<(), Box> { init_logging(); info!("🧪 Testing KMS behavior with corrupted key files"); @@ -215,7 +212,6 @@ async fn test_kms_corrupted_key_files() -> Result<(), Box Result<(), Box> { init_logging(); info!("🧪 Testing KMS multipart upload interruption and recovery"); @@ -399,7 +395,6 @@ async fn test_kms_multipart_upload_interruption() -> Result<(), Box Result<(), Box> { init_logging(); info!("🧪 Testing KMS behavior under resource constraints"); diff --git a/crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs b/crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs index 173e87767..3058f4be9 100644 --- a/crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs +++ b/crates/e2e_test/src/kms/kms_ilm_sse_kms_test.rs @@ -51,7 +51,6 @@ use aws_sdk_s3::types::{ TransitionStorageClass, }; use serde::Deserialize; -use serial_test::serial; use std::time::{Duration as StdDuration, Instant}; use tracing::info; @@ -424,7 +423,6 @@ async fn wait_for_restore_complete(client: &Client, bucket: &str, key: &str, dea /// filter as the cause of the deletion and proves the encrypted bucket stays /// readable end to end after the scanner has run. #[tokio::test] -#[serial] async fn ilm_expiration_on_sse_kms_bucket_under_enforcement() -> TestResult { init_logging(); @@ -485,7 +483,6 @@ async fn ilm_expiration_on_sse_kms_bucket_under_enforcement() -> TestResult { /// (the mechanism `reliant/tiering.rs` established), so the test does not /// depend on scanner scheduling; the 1s scanner cycle stays on as a backstop. #[tokio::test] -#[serial] async fn ilm_transition_on_sse_kms_bucket_under_enforcement_reads_back() -> TestResult { init_logging(); diff --git a/crates/e2e_test/src/kms/kms_local_test.rs b/crates/e2e_test/src/kms/kms_local_test.rs index 82b44a154..ccda86989 100644 --- a/crates/e2e_test/src/kms/kms_local_test.rs +++ b/crates/e2e_test/src/kms/kms_local_test.rs @@ -24,11 +24,9 @@ use super::common::{ test_kms_key_management, test_sse_c_encryption, }; use crate::common::{TEST_BUCKET, init_logging}; -use serial_test::serial; use tracing::{error, info}; #[tokio::test] -#[serial] async fn test_local_kms_end_to_end() -> Result<(), Box> { init_logging(); if skip_if_kms_admin_tool_unavailable("test_local_kms_end_to_end") { @@ -114,7 +112,6 @@ async fn test_local_kms_end_to_end() -> Result<(), Box Result<(), Box> { init_logging(); info!("🧪 Step 1: Test the basic single-file encryption function"); @@ -85,7 +83,6 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box Result<(), Box> { init_logging(); info!("🧪 Step 2: Test unencrypted shard uploads"); @@ -184,7 +181,6 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo /// Step 3: Test Shard Upload + SSE-S3 Encryption (Focus Test) #[tokio::test] -#[serial] async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box> { init_logging(); info!("🧪 Step 3: Test Shard Upload + SSE-S3 Encryption"); @@ -308,7 +304,6 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box Result<(), Box> { init_logging(); info!("🧪 Step 4: test large-file multipart encryption"); @@ -434,7 +429,6 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box Result<(), Box> { init_logging(); info!("🧪 Step 5: test multipart uploads for every encryption mode"); diff --git a/crates/e2e_test/src/kms/test_runner.rs b/crates/e2e_test/src/kms/test_runner.rs index d5e831b79..558c14631 100644 --- a/crates/e2e_test/src/kms/test_runner.rs +++ b/crates/e2e_test/src/kms/test_runner.rs @@ -19,7 +19,6 @@ //! filtering, and comprehensive reporting capabilities. use crate::common::init_logging; -use serial_test::serial; use std::time::Instant; use tokio::time::{Duration, sleep}; use tracing::{debug, error, info, warn}; @@ -458,7 +457,6 @@ impl KMSTestSuite { /// Quick test suite for critical tests only #[tokio::test] -#[serial] async fn test_kms_critical_suite() -> Result<(), Box> { let config = TestSuiteConfig { categories: vec![TestCategory::CoreFunctionality, TestCategory::MultipartEncryption], @@ -481,7 +479,6 @@ async fn test_kms_critical_suite() -> Result<(), Box Result<(), Box> { let suite = KMSTestSuite::new(); let results = suite.run_test_suite().await; diff --git a/crates/e2e_test/src/leading_slash_key_test.rs b/crates/e2e_test/src/leading_slash_key_test.rs index 4de4812df..841c4f060 100644 --- a/crates/e2e_test/src/leading_slash_key_test.rs +++ b/crates/e2e_test/src/leading_slash_key_test.rs @@ -24,7 +24,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use std::error::Error; use tracing::info; @@ -36,7 +35,6 @@ mod tests { /// PUT with a leading-slash key must succeed and the object must be /// readable under the normalized key (leading slash stripped). #[tokio::test] - #[serial] async fn test_put_object_with_leading_slash_key() -> Result<(), Box> { init_logging(); info!("Starting test: PUT object with leading slash in key (Issue #2427)"); @@ -94,7 +92,6 @@ mod tests { /// Duplicate and repeated slashes after a leading slash collapse MinIO-style. #[tokio::test] - #[serial] async fn test_put_object_with_duplicate_slashes_normalized() -> Result<(), Box> { init_logging(); info!("Starting test: duplicate slash normalization (Issue #2427)"); diff --git a/crates/e2e_test/src/lifecycle_regression_test.rs b/crates/e2e_test/src/lifecycle_regression_test.rs index e2e1c039e..3a3615dd4 100644 --- a/crates/e2e_test/src/lifecycle_regression_test.rs +++ b/crates/e2e_test/src/lifecycle_regression_test.rs @@ -36,7 +36,6 @@ mod tests { BucketLifecycleConfiguration, BucketVersioningStatus, ExpirationStatus, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter, NoncurrentVersionExpiration, VersioningConfiguration, }; - use serial_test::serial; use std::error::Error; use tracing::info; @@ -80,7 +79,6 @@ mod tests { /// /// This tests the rule persistence path (rustfs#4963: 3 days → 0 days). #[tokio::test] - #[serial] async fn test_lifecycle_expiration_rule_persists_correctly() -> TestResult { init_logging(); info!("RT-03: lifecycle expiration rule persists correctly"); @@ -148,7 +146,6 @@ mod tests { /// Covers the pattern where noncurrent version expiration rules are /// accepted but old versions are never cleaned up. #[tokio::test] - #[serial] async fn test_lifecycle_noncurrent_version_expiration_rule_persists() -> TestResult { init_logging(); info!("RT-03b: noncurrent version expiration rule persists"); @@ -233,7 +230,6 @@ mod tests { /// after restart. Transition rules require a configured remote tier /// (tested in reliant/tiering.rs), so this test uses expiration only. #[tokio::test] - #[serial] async fn test_lifecycle_prefix_rule_persists() -> TestResult { init_logging(); info!("RT-04: lifecycle prefix rule persists"); @@ -294,7 +290,6 @@ mod tests { /// Regression pattern: DELETE on a versioned object fails or does not /// create a delete marker, or the delete marker is not visible in LIST. #[tokio::test] - #[serial] async fn test_delete_marker_creation_and_visibility() -> TestResult { init_logging(); info!("RT-05b: delete marker creation and visibility"); diff --git a/crates/e2e_test/src/list_buckets_double_slash_test.rs b/crates/e2e_test/src/list_buckets_double_slash_test.rs index ddbb68307..592b252b4 100644 --- a/crates/e2e_test/src/list_buckets_double_slash_test.rs +++ b/crates/e2e_test/src/list_buckets_double_slash_test.rs @@ -27,7 +27,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; - use serial_test::serial; use std::error::Error; /// Sends a SigV4-signed `GET` where the signature is computed over `sign_path` @@ -67,7 +66,6 @@ mod tests { /// `GET /` (path-style service call) returns `ListBuckets`. #[tokio::test] - #[serial] async fn test_list_buckets_single_slash() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -86,7 +84,6 @@ mod tests { /// compat layer rewrites `//` to `/` before `s3s` parses/verifies the request, /// so both routing and signature verification operate on `/`. #[tokio::test] - #[serial] async fn test_list_buckets_double_slash_browser_compat() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -105,7 +102,6 @@ mod tests { /// (`GET //bucket`) must be left untouched by the compat layer — it is not a /// `ListBuckets` request and s3s continues to reject the empty bucket name. #[tokio::test] - #[serial] async fn test_double_slash_rewrite_is_narrowly_scoped() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/list_buckets_iam_filter_test.rs b/crates/e2e_test/src/list_buckets_iam_filter_test.rs index ca748be1a..ac15e0713 100644 --- a/crates/e2e_test/src/list_buckets_iam_filter_test.rs +++ b/crates/e2e_test/src/list_buckets_iam_filter_test.rs @@ -15,7 +15,6 @@ use crate::common::{RustFSTestEnvironment, admin_ok, build_test_s3_config, build_test_sts_client, init_logging}; use aws_sdk_s3::Client; use aws_sdk_s3::error::ProvideErrorMetadata; -use serial_test::serial; use tokio::time::{Duration, Instant}; fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client { @@ -76,7 +75,6 @@ async fn create_service_account( } #[tokio::test] -#[serial] async fn list_buckets_filters_with_iam_bucket_resources() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/list_object_versions_metadata_extension_test.rs b/crates/e2e_test/src/list_object_versions_metadata_extension_test.rs index 150c45090..1e5fd0977 100644 --- a/crates/e2e_test/src/list_object_versions_metadata_extension_test.rs +++ b/crates/e2e_test/src/list_object_versions_metadata_extension_test.rs @@ -24,7 +24,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; - use serial_test::serial; use std::error::Error; use tracing::info; @@ -54,7 +53,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_list_object_versions_metadata_extension_returns_metadata_tags_and_internal() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/list_object_versions_regression_test.rs b/crates/e2e_test/src/list_object_versions_regression_test.rs index b7365a8de..0c29dc110 100644 --- a/crates/e2e_test/src/list_object_versions_regression_test.rs +++ b/crates/e2e_test/src/list_object_versions_regression_test.rs @@ -21,7 +21,6 @@ mod tests { use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; - use serial_test::serial; use tracing::info; fn create_s3_client(env: &RustFSTestEnvironment) -> Client { @@ -29,7 +28,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_list_object_versions_immediately_returns_latest_put_after_delete_marker() { init_logging(); info!("🧪 TEST: ListObjectVersions returns the newest version immediately after put -> delete -> put"); @@ -182,7 +180,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_list_object_versions_prefix_with_marker_object_returns_children() { init_logging(); info!("🧪 TEST: ListObjectVersions returns prefix children when a marker object also exists"); diff --git a/crates/e2e_test/src/list_objects_duplicates_test.rs b/crates/e2e_test/src/list_objects_duplicates_test.rs index 2d7a74351..9b8349dcc 100644 --- a/crates/e2e_test/src/list_objects_duplicates_test.rs +++ b/crates/e2e_test/src/list_objects_duplicates_test.rs @@ -17,7 +17,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use tracing::info; /// Helper function to create an S3 client for testing @@ -60,7 +59,6 @@ mod tests { /// The bug was that "folder/" (the object) and "folder/" (derived prefix) were both added to CommonPrefixes /// when delimiter was "/" because the deduplication check was explicitly skipped for "/" delimiter. #[tokio::test] - #[serial] async fn test_list_objects_v2_unique_common_prefixes() { init_logging(); info!("Starting test: ListObjectsV2 should return unique CommonPrefixes"); @@ -140,7 +138,6 @@ mod tests { /// When both "marker/subdir/" and "marker/subdir/file.txt" exist, listing with /// Prefix="marker/" must not duplicate "marker/subdir/file.txt" in Contents. #[tokio::test] - #[serial] async fn test_list_objects_v2_unique_contents_with_explicit_directory_markers() { init_logging(); info!("Starting test: ListObjectsV2 should return unique keys with explicit directory markers"); @@ -208,7 +205,6 @@ mod tests { /// and never produce the prefix entry `a/`. Delimiter="/" listings then /// returned Contents `a` but silently dropped CommonPrefix `a/`. #[tokio::test] - #[serial] async fn test_list_objects_v2_object_and_same_named_prefix_coexist() { init_logging(); info!("Starting test: ListObjectsV2 should return both object `a` and CommonPrefix `a/`"); diff --git a/crates/e2e_test/src/list_objects_v2_metadata_extension_test.rs b/crates/e2e_test/src/list_objects_v2_metadata_extension_test.rs index 72be61b6c..2cad0cc68 100644 --- a/crates/e2e_test/src/list_objects_v2_metadata_extension_test.rs +++ b/crates/e2e_test/src/list_objects_v2_metadata_extension_test.rs @@ -23,7 +23,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; - use serial_test::serial; use std::error::Error; use tracing::info; @@ -53,7 +52,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_list_objects_v2_metadata_extension_returns_metadata_tags_and_internal() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/list_objects_v2_pagination_test.rs b/crates/e2e_test/src/list_objects_v2_pagination_test.rs index c970713d8..70b00467b 100644 --- a/crates/e2e_test/src/list_objects_v2_pagination_test.rs +++ b/crates/e2e_test/src/list_objects_v2_pagination_test.rs @@ -30,7 +30,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use std::collections::HashSet; use tracing::info; @@ -61,7 +60,6 @@ mod tests { /// Test for Issue #2775: continuation forwarding must not /// skip a child directory when the prefix component repeats in the key. #[tokio::test] - #[serial] async fn test_list_objects_v2_repeated_prefix_continuation() { init_logging(); info!("Starting test: ListObjectsV2 repeated-prefix continuation"); @@ -187,7 +185,6 @@ mod tests { /// This is the core bug from issue #1596: the server was returning /// IsTruncated=true even when all objects fit within the requested max_keys. #[tokio::test] - #[serial] async fn test_list_objects_v2_not_truncated_when_all_objects_returned() { init_logging(); info!("Starting test: ListObjectsV2 should not be truncated when all objects fit within max_keys"); @@ -252,7 +249,6 @@ mod tests { /// 2. NextContinuationToken is returned (not NextMarker) /// 3. Using ContinuationToken fetches the remaining objects #[tokio::test] - #[serial] async fn test_list_objects_v2_pagination_with_continuation_token() { init_logging(); info!("Starting test: ListObjectsV2 pagination with continuation token"); @@ -394,7 +390,6 @@ mod tests { /// Edge case: when max_keys exactly equals the number of objects, /// IsTruncated should be false. #[tokio::test] - #[serial] async fn test_list_objects_v2_max_keys_equals_object_count() { init_logging(); info!("Starting test: ListObjectsV2 with max_keys equal to object count"); @@ -455,7 +450,6 @@ mod tests { /// /// Edge case: IsTruncated should be false for empty bucket. #[tokio::test] - #[serial] async fn test_list_objects_v2_empty_bucket() { init_logging(); info!("Starting test: ListObjectsV2 with empty bucket"); @@ -495,7 +489,6 @@ mod tests { /// Test ListObjectsV2 caps max_keys above the service limit and still paginates. #[tokio::test] - #[serial] async fn test_list_objects_v2_max_keys_above_limit_returns_token() { init_logging(); info!("Starting test: ListObjectsV2 with max_keys above limit"); @@ -563,7 +556,6 @@ mod tests { /// S3 semantics: when max_keys is 0, the response should include no objects /// and IsTruncated should be false. #[tokio::test] - #[serial] async fn test_list_objects_v2_max_keys_zero() { init_logging(); info!("Starting test: ListObjectsV2 with max_keys=0"); @@ -620,7 +612,6 @@ mod tests { /// With max_keys=1000, all 5 visible results (3 prefixes + 2 objects) fit in one /// page, so IsTruncated must be false even though raw entry count is much larger. #[tokio::test] - #[serial] async fn test_list_objects_v2_delimiter_collapsed_prefix_no_false_truncation() { init_logging(); info!("Starting test: ListObjectsV2 delimiter collapsed-prefix no false truncation"); @@ -744,7 +735,6 @@ mod tests { /// Each page returns up to 50 CommonPrefixes. The server must correctly set /// IsTruncated and provide a valid continuation token across all pages. #[tokio::test] - #[serial] async fn test_list_objects_v2_delimiter_small_page_traverses_all() { init_logging(); info!("Starting test: ListObjectsV2 delimiter small page traverses all keys"); @@ -867,7 +857,6 @@ mod tests { /// but after delimiter collapse only 10 CommonPrefixes are visible (10 < 1000). /// IsTruncated must be false since there are no additional visible results. #[tokio::test] - #[serial] async fn test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below() { init_logging(); info!("Starting test: ListObjectsV2 raw > MaxKeys but visible < MaxKeys after collapse"); @@ -970,7 +959,6 @@ mod tests { /// This complements test_list_objects_v2_max_keys_above_limit_returns_token which /// tests the non-delimiter case. #[tokio::test] - #[serial] async fn test_list_objects_v2_maxkeys_above_limit_with_delimiter() { init_logging(); info!("Starting test: ListObjectsV2 MaxKeys above limit with delimiter"); @@ -1041,7 +1029,6 @@ mod tests { /// the next page: with keys `a`, `a.txt`, `zz` and max_keys=1, page 2 /// returned `zz` and `a.txt` was never listed. #[tokio::test] - #[serial] async fn test_list_objects_v2_continuation_keeps_keys_after_marker_stem() { init_logging(); info!("Starting test: continuation must not skip keys sorting below the cursor tag"); diff --git a/crates/e2e_test/src/listing_regression_test.rs b/crates/e2e_test/src/listing_regression_test.rs index 914b3aab3..ee2569f29 100644 --- a/crates/e2e_test/src/listing_regression_test.rs +++ b/crates/e2e_test/src/listing_regression_test.rs @@ -31,7 +31,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use std::collections::HashSet; use std::error::Error; use tracing::info; @@ -49,7 +48,6 @@ mod tests { /// 3. Verify all 100 keys are returned exactly once /// 4. Verify no duplicates or skipped keys #[tokio::test] - #[serial] async fn test_list_objects_v2_completeness_100_objects() -> TestResult { init_logging(); info!("RT-06: listing completeness with 100 objects"); @@ -133,7 +131,6 @@ mod tests { /// Regression pattern: prefix filter returns empty or includes wrong keys /// (rustfs#5051: empty results for shallow prefixes). #[tokio::test] - #[serial] async fn test_list_objects_v2_prefix_filter_correctness() -> TestResult { init_logging(); info!("RT-06b: prefix filter correctness"); @@ -233,7 +230,6 @@ mod tests { /// Regression pattern: delimiter handling produces incorrect CommonPrefixes /// or misses objects at the delimiter boundary. #[tokio::test] - #[serial] async fn test_list_objects_v2_delimiter_common_prefixes() -> TestResult { init_logging(); info!("RT-06c: delimiter and CommonPrefixes"); @@ -290,7 +286,6 @@ mod tests { /// Regression pattern: IsTruncated=false when there are more objects /// (rustfs#4810: walk_dir timeout truncation with false IsTruncated). #[tokio::test] - #[serial] async fn test_list_objects_v2_is_truncated_correctness() -> TestResult { init_logging(); info!("RT-06d: IsTruncated correctness"); diff --git a/crates/e2e_test/src/mc_mirror_small_bucket_test.rs b/crates/e2e_test/src/mc_mirror_small_bucket_test.rs index 6654a8663..2a9c1a507 100644 --- a/crates/e2e_test/src/mc_mirror_small_bucket_test.rs +++ b/crates/e2e_test/src/mc_mirror_small_bucket_test.rs @@ -13,7 +13,6 @@ // limitations under the License. use crate::common::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, RustFSTestEnvironment}; -use serial_test::serial; use std::path::Path; use std::process::Command; use std::time::Duration; @@ -73,7 +72,6 @@ fn count_files(root: &Path) -> usize { } #[tokio::test] -#[serial] async fn test_mc_mirror_small_bucket_completes_without_list_timeout() -> TestResult { crate::common::init_logging(); info!("Starting issue #3107 mc mirror regression test"); diff --git a/crates/e2e_test/src/namespace_lock_quorum_test.rs b/crates/e2e_test/src/namespace_lock_quorum_test.rs index 9b9e1d53f..0af609ecc 100644 --- a/crates/e2e_test/src/namespace_lock_quorum_test.rs +++ b/crates/e2e_test/src/namespace_lock_quorum_test.rs @@ -16,7 +16,6 @@ use crate::common::RustFSTestClusterEnvironment; use aws_sdk_s3::Client; use aws_sdk_s3::error::SdkError; use bytes::Bytes; -use serial_test::serial; use std::sync::Arc; use tokio::sync::Barrier; use tracing::{info, warn}; @@ -51,7 +50,6 @@ fn format_s3_error(err: SdkError TestResult { crate::common::init_logging(); info!("Starting namespace lock quorum regression test with auto cluster"); @@ -128,7 +126,6 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum() /// `StorageError::other(...)` → `StorageError::Io(...)`, which fell through to /// `S3ErrorCode::InternalError` (500) in the error mapping. #[tokio::test] -#[serial] async fn test_concurrent_put_same_key_never_returns_500() -> TestResult { crate::common::init_logging(); info!("Starting concurrent PUT 500 regression test"); diff --git a/crates/e2e_test/src/negative_sigv4_test.rs b/crates/e2e_test/src/negative_sigv4_test.rs index f0539e0a0..a6a440383 100644 --- a/crates/e2e_test/src/negative_sigv4_test.rs +++ b/crates/e2e_test/src/negative_sigv4_test.rs @@ -37,7 +37,6 @@ use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; use aws_sdk_s3::primitives::ByteStream; use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::request_signature_v4::{SIGN_V4_ALGORITHM, get_scope, get_signature, get_signing_key}; -use serial_test::serial; use std::fmt::Write as _; use time::macros::format_description; use time::{Duration, OffsetDateTime}; @@ -183,7 +182,6 @@ async fn setup(env: &mut RustFSTestEnvironment) -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -214,7 +212,6 @@ async fn valid_header_sigv4_request_succeeds() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -254,7 +251,6 @@ async fn tampered_signature_returns_signature_does_not_match() -> Result<(), Box /// (b) A valid AccessKeyId paired with the wrong secret key must be rejected /// with SignatureDoesNotMatch / 403. #[tokio::test] -#[serial] async fn wrong_secret_key_returns_signature_does_not_match() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -279,7 +275,6 @@ async fn wrong_secret_key_returns_signature_does_not_match() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -320,7 +315,6 @@ async fn tampered_payload_is_rejected() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -344,7 +338,6 @@ async fn skewed_date_returns_request_time_too_skewed() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/notification_startup_regression_test.rs b/crates/e2e_test/src/notification_startup_regression_test.rs index f554d3cfd..0d0feed98 100644 --- a/crates/e2e_test/src/notification_startup_regression_test.rs +++ b/crates/e2e_test/src/notification_startup_regression_test.rs @@ -32,7 +32,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; - use serial_test::serial; use std::error::Error; use tracing::info; @@ -47,7 +46,6 @@ mod tests { /// starts successfully with notification enabled and can serve S3 requests. /// A full webhook delivery test is in notification_webhook_test.rs. #[tokio::test] - #[serial] async fn test_notification_enabled_server_starts_cleanly() -> TestResult { init_logging(); info!("RT-01: notification enabled server starts cleanly"); @@ -92,7 +90,6 @@ mod tests { /// 3. Restart server /// 4. Verify notification config still exists #[tokio::test] - #[serial] async fn test_notification_config_survives_restart() -> TestResult { init_logging(); info!("RT-02: notification config survives restart"); diff --git a/crates/e2e_test/src/notification_webhook_test.rs b/crates/e2e_test/src/notification_webhook_test.rs index e63cea554..1eec17343 100644 --- a/crates/e2e_test/src/notification_webhook_test.rs +++ b/crates/e2e_test/src/notification_webhook_test.rs @@ -47,7 +47,6 @@ use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS; use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER}; use s3s::Body; use serde_json::Value; -use serial_test::serial; use std::error::Error; use std::io::Cursor; use std::path::Path; @@ -625,7 +624,6 @@ fn assert_generated_request_id_correlation(record: &Value, request_id: &str) { /// RUSTFS_NOTIFY_ENABLE, an HTTPS webhook using a configured CA must become /// online and receive a real S3 event POST. #[tokio::test] -#[serial] async fn test_https_webhook_target_delivers_event_with_notify_env_enabled() -> TestResult { init_logging(); @@ -680,7 +678,6 @@ async fn test_https_webhook_target_delivers_event_with_notify_env_enabled() -> T /// PUT / multipart-complete / DELETE each deliver one event with correct fields, /// and the prefix/suffix filter drops non-matching keys. #[tokio::test] -#[serial] async fn test_webhook_event_delivery_and_filtering() -> TestResult { init_logging(); @@ -900,7 +897,6 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult { /// An event queued while the target endpoint rejects delivery survives on the /// durable store and is redelivered once the endpoint comes back. #[tokio::test] -#[serial] async fn test_webhook_redelivers_event_after_target_recovers() -> TestResult { init_logging(); diff --git a/crates/e2e_test/src/object_lambda_test.rs b/crates/e2e_test/src/object_lambda_test.rs index 69d2e46ef..66f259e6d 100644 --- a/crates/e2e_test/src/object_lambda_test.rs +++ b/crates/e2e_test/src/object_lambda_test.rs @@ -20,7 +20,6 @@ use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::{pre_sign_v4, sign_v4}; use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS; use s3s::Body; -use serial_test::serial; use std::collections::HashMap; use std::error::Error; use time::OffsetDateTime; @@ -548,7 +547,6 @@ async fn read_listen_notification_event( } #[tokio::test] -#[serial] async fn test_notification_target_persists_across_restart_and_delete() -> Result<(), Box> { init_logging(); @@ -608,7 +606,6 @@ async fn test_notification_target_persists_across_restart_and_delete() -> Result } #[tokio::test] -#[serial] async fn test_notification_target_with_path_is_online_via_transport_probe() -> Result<(), Box> { init_logging(); @@ -641,7 +638,6 @@ async fn test_notification_target_with_path_is_online_via_transport_probe() -> R } #[tokio::test] -#[serial] async fn test_get_object_lambda_accepts_presigned_requests() -> Result<(), Box> { init_logging(); @@ -682,7 +678,6 @@ async fn test_get_object_lambda_accepts_presigned_requests() -> Result<(), Box Result<(), Box> { init_logging(); @@ -722,7 +717,6 @@ async fn test_get_object_lambda_accepts_named_webhook_target_arn() -> Result<(), } #[tokio::test] -#[serial] async fn test_get_object_lambda_invokes_runtime_webhook_target() -> Result<(), Box> { init_logging(); @@ -790,7 +784,6 @@ async fn test_get_object_lambda_invokes_runtime_webhook_target() -> Result<(), B } #[tokio::test] -#[serial] async fn test_get_object_lambda_passthroughs_non_success_webhook_response() -> Result<(), Box> { init_logging(); @@ -850,7 +843,6 @@ async fn test_get_object_lambda_passthroughs_non_success_webhook_response() -> R } #[tokio::test] -#[serial] async fn test_get_object_lambda_rejects_success_response_without_auth_headers() -> Result<(), Box> { init_logging(); @@ -896,7 +888,6 @@ async fn test_get_object_lambda_rejects_success_response_without_auth_headers() } #[tokio::test] -#[serial] async fn test_get_object_lambda_rejects_success_response_with_mismatched_auth_headers() -> Result<(), Box> { init_logging(); @@ -943,7 +934,6 @@ async fn test_get_object_lambda_rejects_success_response_with_mismatched_auth_he } #[tokio::test] -#[serial] async fn test_get_object_lambda_rejects_unsupported_target_type() -> Result<(), Box> { init_logging(); @@ -980,7 +970,6 @@ async fn test_get_object_lambda_rejects_unsupported_target_type() -> Result<(), } #[tokio::test] -#[serial] async fn test_get_object_lambda_rejects_unconfigured_target() -> Result<(), Box> { init_logging(); @@ -1017,7 +1006,6 @@ async fn test_get_object_lambda_rejects_unconfigured_target() -> Result<(), Box< } #[tokio::test] -#[serial] async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box> { init_logging(); @@ -1063,7 +1051,6 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box Result<(), Box> { init_logging(); @@ -1106,7 +1093,6 @@ async fn test_configure_object_lambda_target_rejects_invalid_endpoint() -> Resul } #[tokio::test] -#[serial] async fn test_configure_object_lambda_notify_webhook_rejects_response_header_timeout_key() -> Result<(), Box> { init_logging(); @@ -1140,7 +1126,6 @@ async fn test_configure_object_lambda_notify_webhook_rejects_response_header_tim } #[tokio::test] -#[serial] async fn test_listen_notification_emits_after_put_object() -> Result<(), Box> { init_logging(); @@ -1184,7 +1169,6 @@ async fn test_listen_notification_emits_after_put_object() -> Result<(), Box Result<(), Box> { init_logging(); @@ -1219,7 +1203,6 @@ async fn test_listen_notification_emits_on_empty_bucket_when_notify_disabled() - } #[tokio::test] -#[serial] async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/overwrite_cleanup_regression_test.rs b/crates/e2e_test/src/overwrite_cleanup_regression_test.rs index f41113b12..7be7c26ea 100644 --- a/crates/e2e_test/src/overwrite_cleanup_regression_test.rs +++ b/crates/e2e_test/src/overwrite_cleanup_regression_test.rs @@ -13,7 +13,6 @@ // limitations under the License. use aws_sdk_s3::primitives::ByteStream; -use serial_test::serial; use std::path::{Path, PathBuf}; use uuid::Uuid; @@ -24,7 +23,6 @@ const TEST_OBJECT: &str = "large-object.bin"; const PAYLOAD_SIZE: usize = 512 * 1024; #[tokio::test(flavor = "multi_thread")] -#[serial] async fn unversioned_overwrite_removes_previous_physical_data_dir() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/presigned_negative_test.rs b/crates/e2e_test/src/presigned_negative_test.rs index 767cc7ca5..31df39fd0 100644 --- a/crates/e2e_test/src/presigned_negative_test.rs +++ b/crates/e2e_test/src/presigned_negative_test.rs @@ -43,7 +43,6 @@ use aws_sdk_s3::presigning::{PresignedRequest, PresigningConfig}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::{Client, Config}; use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; -use serial_test::serial; use std::time::{Duration, SystemTime}; use tracing::info; @@ -157,7 +156,6 @@ async fn setup(env: &mut RustFSTestEnvironment) -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -182,7 +180,6 @@ async fn valid_presigned_get_succeeds() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -211,7 +208,6 @@ async fn valid_presigned_put_succeeds() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -236,7 +232,6 @@ async fn expired_presigned_get_is_rejected() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -262,7 +257,6 @@ async fn tampered_signature_returns_signature_does_not_match() -> Result<(), Box /// (c) A presigned URL generated with the WRONG secret (but the real access key /// id) must be rejected with 403 / SignatureDoesNotMatch. #[tokio::test] -#[serial] async fn wrong_secret_key_returns_signature_does_not_match() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -290,7 +284,6 @@ async fn wrong_secret_key_returns_signature_does_not_match() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -325,7 +318,6 @@ async fn tampered_target_key_returns_signature_does_not_match() -> Result<(), Bo /// (e / acceptance 4 negative half) Tampering the signature of a presigned PUT /// must be rejected with 403 / SignatureDoesNotMatch — the write must not land. #[tokio::test] -#[serial] async fn tampered_presigned_put_returns_signature_does_not_match() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; diff --git a/crates/e2e_test/src/protocols/test_runner.rs b/crates/e2e_test/src/protocols/test_runner.rs index 0a578804d..7150ab2cd 100644 --- a/crates/e2e_test/src/protocols/test_runner.rs +++ b/crates/e2e_test/src/protocols/test_runner.rs @@ -22,7 +22,6 @@ use crate::protocols::sftp_compliance::{ }; use crate::protocols::sftp_core::{test_sftp_core_operations, test_sftp_idle_timeout_disconnects}; use crate::protocols::webdav_core::test_webdav_core_operations; -use serial_test::serial; use std::time::Instant; use tokio::time::{Duration, sleep}; use tracing::{error, info}; @@ -229,7 +228,6 @@ fn all_protocol_tests() -> Vec { /// Test suite #[tokio::test] -#[serial] async fn test_protocol_core_suite() -> Result<(), Box> { let suite = ProtocolTestSuite::new(); let results = suite.run_test_suite().await; diff --git a/crates/e2e_test/src/quota_test.rs b/crates/e2e_test/src/quota_test.rs index 87eee5f37..e7c095674 100644 --- a/crates/e2e_test/src/quota_test.rs +++ b/crates/e2e_test/src/quota_test.rs @@ -15,7 +15,6 @@ use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_post, awscurl_put, init_logging}; use aws_sdk_s3::Client; use http::{Method, StatusCode}; -use serial_test::serial; use tokio::time::{Duration, sleep, timeout}; use tracing::{debug, info}; @@ -255,7 +254,6 @@ mod integration_tests { use aws_sdk_s3::error::ProvideErrorMetadata; #[tokio::test] - #[serial] async fn test_quota_basic_operations() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -300,7 +298,6 @@ mod integration_tests { /// with 400 UnexpectedContent, and an over-quota aws-chunked PUT must still get the quota /// rejection. #[tokio::test] - #[serial] async fn test_quota_admission_aws_chunked_declared_encoding() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -352,7 +349,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_update_and_clear() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -388,7 +384,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_delete_operations() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -425,7 +420,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_usage_tracking() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -464,7 +458,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_statistics() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -498,7 +491,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_check_api() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -539,7 +531,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_multiple_buckets() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -580,7 +571,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_error_handling() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -616,7 +606,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_http_endpoints() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -682,7 +671,6 @@ mod integration_tests { /// Test that a normal user with `readwrite` policy can read quota but cannot set/clear quota. #[tokio::test] - #[serial] async fn test_quota_normal_user_permissions() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -738,7 +726,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_copy_operations() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -784,7 +771,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_batch_delete() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { @@ -843,7 +829,6 @@ mod integration_tests { } #[tokio::test] - #[serial] async fn test_quota_multipart_upload() -> Result<(), Box> { init_logging(); if skip_without_awscurl() { diff --git a/crates/e2e_test/src/reliant/conditional_writes.rs b/crates/e2e_test/src/reliant/conditional_writes.rs index 843ecc4e0..166ce66f5 100644 --- a/crates/e2e_test/src/reliant/conditional_writes.rs +++ b/crates/e2e_test/src/reliant/conditional_writes.rs @@ -6,7 +6,6 @@ use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::error::SdkError; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; use bytes::Bytes; -use serial_test::serial; use std::error::Error; const ENDPOINT: &str = "http://localhost:9000"; @@ -89,7 +88,6 @@ fn generate_test_key(prefix: &str) -> String { } #[tokio::test] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_conditional_put_okay() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -132,7 +130,6 @@ async fn test_conditional_put_okay() -> Result<(), Box> { } #[tokio::test] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_conditional_put_failed() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -195,7 +192,6 @@ async fn test_conditional_put_failed() -> Result<(), Box> } #[tokio::test] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_conditional_put_when_object_does_not_exist() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -240,7 +236,6 @@ async fn test_conditional_put_when_object_does_not_exist() -> Result<(), Box Result<(), Box> { let client = create_aws_s3_client().await?; diff --git a/crates/e2e_test/src/reliant/get_deleted_object_test.rs b/crates/e2e_test/src/reliant/get_deleted_object_test.rs index b34159ec1..2b87a11c6 100644 --- a/crates/e2e_test/src/reliant/get_deleted_object_test.rs +++ b/crates/e2e_test/src/reliant/get_deleted_object_test.rs @@ -24,7 +24,6 @@ use aws_sdk_s3::Client; use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::error::SdkError; use bytes::Bytes; -use serial_test::serial; use std::error::Error; use tracing::info; @@ -70,7 +69,6 @@ async fn setup_test_bucket(client: &Client) -> Result<(), Box> { } #[tokio::test] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_get_deleted_object_returns_nosuchkey() -> Result<(), Box> { // Initialize logging @@ -144,7 +142,6 @@ async fn test_get_deleted_object_returns_nosuchkey() -> Result<(), Box Result<(), Box> { let _ = tracing_subscriber::fmt() @@ -196,7 +193,6 @@ async fn test_head_deleted_object_returns_nosuchkey() -> Result<(), Box Result<(), Box> { let _ = tracing_subscriber::fmt() @@ -233,7 +229,6 @@ async fn test_get_nonexistent_object_returns_nosuchkey() -> Result<(), Box Result<(), Box> { let _ = tracing_subscriber::fmt() diff --git a/crates/e2e_test/src/reliant/head_deleted_object_versioning_test.rs b/crates/e2e_test/src/reliant/head_deleted_object_versioning_test.rs index a4d471754..b89579bde 100644 --- a/crates/e2e_test/src/reliant/head_deleted_object_versioning_test.rs +++ b/crates/e2e_test/src/reliant/head_deleted_object_versioning_test.rs @@ -25,7 +25,6 @@ use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::error::SdkError; use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration}; use bytes::Bytes; -use serial_test::serial; use std::error::Error; use tracing::info; @@ -85,7 +84,6 @@ async fn setup_test_bucket(client: &Client) -> Result<(), Box> { /// Test that HeadObject on a deleted object returns NoSuchKey when versioning is enabled #[tokio::test] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_head_deleted_object_versioning_returns_nosuchkey() -> Result<(), Box> { let _ = tracing_subscriber::fmt() diff --git a/crates/e2e_test/src/reliant/head_tls_bodyless_test.rs b/crates/e2e_test/src/reliant/head_tls_bodyless_test.rs index 6ef214a17..a535e431e 100644 --- a/crates/e2e_test/src/reliant/head_tls_bodyless_test.rs +++ b/crates/e2e_test/src/reliant/head_tls_bodyless_test.rs @@ -30,7 +30,6 @@ use reqwest::{Certificate, Client, Response, StatusCode}; use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; -use serial_test::serial; use std::error::Error; use std::path::Path; use std::process::Command; @@ -157,7 +156,6 @@ async fn start_tls_rustfs_server(env: &mut RustFSTestEnvironment, tls_dir: &Path } #[tokio::test] -#[serial] async fn test_head_missing_object_over_tls_http2_is_bodyless() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/reliant/sql.rs b/crates/e2e_test/src/reliant/sql.rs index 35ef8bcae..6f63ddab0 100644 --- a/crates/e2e_test/src/reliant/sql.rs +++ b/crates/e2e_test/src/reliant/sql.rs @@ -20,7 +20,6 @@ use aws_sdk_s3::types::{ CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput, JsonType, OutputSerialization, }; use bytes::Bytes; -use serial_test::serial; use std::error::Error; const ENDPOINT: &str = "http://localhost:9000"; @@ -118,7 +117,6 @@ async fn process_select_response( } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_select_object_content_csv_basic() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -160,7 +158,6 @@ async fn test_select_object_content_csv_basic() -> Result<(), Box> { } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_select_object_content_csv_aggregation() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -206,7 +203,6 @@ async fn test_select_object_content_csv_aggregation() -> Result<(), Box Result<(), Box> { let client = create_aws_s3_client().await?; @@ -248,7 +244,6 @@ async fn test_select_object_content_json_basic() -> Result<(), Box> { } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_select_object_content_csv_limit() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -291,7 +286,6 @@ async fn test_select_object_content_csv_limit() -> Result<(), Box> { } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_select_object_content_csv_order_by() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -337,7 +331,6 @@ async fn test_select_object_content_csv_order_by() -> Result<(), Box> } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[serial] #[ignore = "requires running RustFS server at localhost:9000"] async fn test_select_object_content_error_handling() -> Result<(), Box> { let client = create_aws_s3_client().await?; @@ -373,7 +366,6 @@ async fn test_select_object_content_error_handling() -> Result<(), Box Result<(), Box> { let client = create_aws_s3_client().await?; diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index b9645deba..d0282c83a 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -60,7 +60,6 @@ use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; use s3s::header::X_AMZ_REPLICATION_STATUS; -use serial_test::serial; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; use std::convert::Infallible; @@ -4630,7 +4629,6 @@ async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult { /// fails without any PUT reaching the target (capability cache, proven from /// the target journal); plaintext objects still replicate COMPLETED. #[tokio::test] -#[serial] async fn test_ssec_replication_fails_closed_when_target_drops_passthrough_headers() -> TestResult { init_logging(); @@ -4740,7 +4738,6 @@ async fn test_ssec_replication_fails_closed_when_target_drops_passthrough_header /// a capability limit, and a plaintext-only deployment against a MinIO-like /// target must not turn red. #[tokio::test] -#[serial] async fn test_replication_check_flags_ssec_passthrough_dropping_target() -> TestResult { init_logging(); @@ -4831,7 +4828,6 @@ async fn test_replication_check_flags_ssec_passthrough_dropping_target() -> Test /// HEAD-back audit against the recovered RustFS target — and the replica must /// be readable with the customer key. #[tokio::test] -#[serial] async fn test_bucket_replication_sse_c_heals_after_target_outage() -> TestResult { init_logging(); @@ -4924,7 +4920,6 @@ async fn test_bucket_replication_sse_c_heals_after_target_outage() -> TestResult /// the existing-object resync (`replicate_all` transport, N2-audited), land as /// a REPLICA, and read back with the customer key. #[tokio::test] -#[serial] async fn test_bucket_replication_sse_c_existing_object_resync() -> TestResult { init_logging(); @@ -8819,7 +8814,6 @@ async fn start_read_proxy_lab( /// headers are forwarded verbatim, and an inbound request that was itself /// proxied is answered locally (404) without touching the target. #[tokio::test] -#[serial] async fn test_get_and_head_proxy_unreplicated_object_to_replication_target() -> TestResult { init_logging(); @@ -9002,7 +8996,6 @@ async fn test_get_and_head_proxy_unreplicated_object_to_replication_target() -> /// proxied to the replication target with the anti-loop marker, mirroring /// MinIO `proxyGetTaggingToRepTarget`. #[tokio::test] -#[serial] async fn test_get_object_tagging_proxies_unreplicated_object_to_replication_target() -> TestResult { init_logging(); diff --git a/crates/e2e_test/src/security_boundary_test.rs b/crates/e2e_test/src/security_boundary_test.rs index 1f5fdda83..0a62ef0f5 100644 --- a/crates/e2e_test/src/security_boundary_test.rs +++ b/crates/e2e_test/src/security_boundary_test.rs @@ -25,7 +25,6 @@ use crate::common::{RustFSTestEnvironment, awscurl_available, awscurl_put, init_ use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart, Tag, Tagging}; -use serial_test::serial; use std::error::Error; use tracing::info; @@ -36,7 +35,6 @@ use tracing::info; /// far beyond that limit and assert the server rejects it with the specific /// error, rather than accepting an arbitrarily large control-plane body. #[tokio::test] -#[serial] async fn test_large_xml_body_rejection() -> Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -92,7 +90,6 @@ async fn test_large_xml_body_rejection() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -149,7 +146,6 @@ async fn test_excessive_multipart_parts() -> Result<(), Box Result<(), Box> { init_logging(); let mut env = RustFSTestEnvironment::new().await?; @@ -232,7 +228,6 @@ async fn test_concurrent_object_operations() -> Result<(), Box Result<(), Box> { init_logging(); if !awscurl_available() { diff --git a/crates/e2e_test/src/server_startup_failfast_test.rs b/crates/e2e_test/src/server_startup_failfast_test.rs index 784a08e56..410779405 100644 --- a/crates/e2e_test/src/server_startup_failfast_test.rs +++ b/crates/e2e_test/src/server_startup_failfast_test.rs @@ -23,7 +23,6 @@ #[cfg(test)] mod tests { use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path}; - use serial_test::serial; use std::net::TcpListener; use std::time::{Duration, Instant}; @@ -31,7 +30,6 @@ mod tests { /// while :9001 is occupied: the server exits at startup, and the harness /// must surface that promptly rather than waiting out the 60s timeout. #[tokio::test] - #[serial] async fn test_start_fails_fast_when_server_exits_during_startup() { init_logging(); diff --git a/crates/e2e_test/src/snowball_auto_extract_test.rs b/crates/e2e_test/src/snowball_auto_extract_test.rs index aad32e7cc..2077af321 100644 --- a/crates/e2e_test/src/snowball_auto_extract_test.rs +++ b/crates/e2e_test/src/snowball_auto_extract_test.rs @@ -17,7 +17,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; - use serial_test::serial; use std::error::Error; use std::io::Cursor; @@ -101,7 +100,6 @@ mod tests { } #[tokio::test] - #[serial] async fn snowball_auto_extract_supports_minio_prefix_and_directory_markers() -> Result<(), Box> { init_logging(); @@ -138,7 +136,6 @@ mod tests { } #[tokio::test] - #[serial] async fn snowball_auto_extract_supports_standard_headers_with_combined_extract_options() -> Result<(), Box> { init_logging(); @@ -229,7 +226,6 @@ mod tests { } #[tokio::test] - #[serial] async fn snowball_auto_extract_ignores_directories_when_requested() -> Result<(), Box> { init_logging(); @@ -268,7 +264,6 @@ mod tests { } #[tokio::test] - #[serial] async fn snowball_auto_extract_ignores_invalid_entries_when_requested() -> Result<(), Box> { init_logging(); @@ -304,7 +299,6 @@ mod tests { } #[tokio::test] - #[serial] async fn snowball_auto_extract_rejects_parent_dir_entry_without_cross_bucket_write() -> Result<(), Box> { init_logging(); @@ -347,7 +341,6 @@ mod tests { } #[tokio::test] - #[serial] async fn snowball_auto_extract_prefers_exact_minio_prefix_over_suffix_fallback() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/special_chars_test.rs b/crates/e2e_test/src/special_chars_test.rs index 8989b02f3..a4df48e78 100644 --- a/crates/e2e_test/src/special_chars_test.rs +++ b/crates/e2e_test/src/special_chars_test.rs @@ -34,7 +34,6 @@ mod tests { use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::sign_v4; use s3s::Body; - use serial_test::serial; use std::error::Error; use tracing::{debug, info}; @@ -93,7 +92,6 @@ mod tests { /// mc cp README.md "local/dummy/a%20f+/b/c/3/README.md" /// ``` #[tokio::test] - #[serial] async fn test_object_with_space_in_path() { init_logging(); info!("Starting test: object with space in path"); @@ -175,7 +173,6 @@ mod tests { /// /test/data/org_main-org/dashboards/ES+net/LHC+Data+Challenge/firefly-details.json /// ``` #[tokio::test] - #[serial] async fn test_object_with_plus_in_path() { init_logging(); info!("Starting test: object with plus sign in path"); @@ -245,7 +242,6 @@ mod tests { /// Test with mixed special characters #[tokio::test] - #[serial] async fn test_object_with_mixed_special_chars() { init_logging(); info!("Starting test: object with mixed special characters"); @@ -305,7 +301,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_signed_get_missing_object_with_trailing_equals_returns_no_such_key() -> Result<(), Box> { init_logging(); @@ -334,7 +329,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_signed_get_existing_object_with_trailing_equals_returns_content() -> Result<(), Box> { init_logging(); @@ -373,7 +367,6 @@ mod tests { /// Test DELETE operation with special characters #[tokio::test] - #[serial] async fn test_delete_object_with_special_chars() { init_logging(); info!("Starting test: DELETE object with special characters"); @@ -421,7 +414,6 @@ mod tests { /// Test exact scenario from the issue #[tokio::test] - #[serial] async fn test_issue_scenario_exact() { init_logging(); info!("Starting test: Exact scenario from GitHub issue"); @@ -494,7 +486,6 @@ mod tests { /// Test HEAD object with special characters #[tokio::test] - #[serial] async fn test_head_object_with_special_chars() { init_logging(); info!("Starting test: HEAD object with special characters"); @@ -538,7 +529,6 @@ mod tests { /// Test COPY object with special characters in both source and destination #[tokio::test] - #[serial] async fn test_copy_object_with_special_chars() { init_logging(); info!("Starting test: COPY object with special characters"); @@ -597,7 +587,6 @@ mod tests { /// Test Unicode characters in object keys #[tokio::test] - #[serial] async fn test_unicode_characters_in_path() { init_logging(); info!("Starting test: Unicode characters in object paths"); @@ -661,7 +650,6 @@ mod tests { /// Test special characters in different parts of the path #[tokio::test] - #[serial] async fn test_special_chars_in_different_path_positions() { init_logging(); info!("Starting test: Special characters in different path positions"); @@ -719,7 +707,6 @@ mod tests { /// Test that control characters are properly rejected #[tokio::test] - #[serial] async fn test_control_characters_rejected() { init_logging(); info!("Starting test: Control characters should be rejected"); @@ -769,7 +756,6 @@ mod tests { /// Test LIST with various special character prefixes #[tokio::test] - #[serial] async fn test_list_with_special_char_prefixes() { init_logging(); info!("Starting test: LIST with special character prefixes"); @@ -838,7 +824,6 @@ mod tests { /// Test delimiter-based listing with special characters #[tokio::test] - #[serial] async fn test_list_with_delimiter_and_special_chars() { init_logging(); info!("Starting test: LIST with delimiter and special characters"); diff --git a/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs b/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs index 618b79b61..c5fc45e17 100644 --- a/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs +++ b/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs @@ -16,7 +16,6 @@ use crate::common::{RustFSTestClusterEnvironment, init_logging}; use aws_sdk_s3::error::SdkError; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::CompletedMultipartUpload; -use serial_test::serial; use tokio::time::{Duration, sleep}; use tracing::info; use uuid::Uuid; @@ -101,7 +100,6 @@ async fn wait_for_cleanup_on_all_nodes( } #[tokio::test] -#[serial] async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster() -> Result<(), Box> { init_logging(); diff --git a/crates/e2e_test/src/sts_query_compat_test.rs b/crates/e2e_test/src/sts_query_compat_test.rs index 517cc1c82..4f96c2e5c 100644 --- a/crates/e2e_test/src/sts_query_compat_test.rs +++ b/crates/e2e_test/src/sts_query_compat_test.rs @@ -25,7 +25,6 @@ use hyper::server::conn::http1; use hyper::service::service_fn; use hyper_util::rt::TokioIo; use serde_json::Value; -use serial_test::serial; use std::collections::BTreeSet; use std::convert::Infallible; use std::error::Error; @@ -350,7 +349,6 @@ impl Drop for OpaMock { } #[tokio::test] -#[serial] async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult { init_logging(); @@ -487,7 +485,6 @@ async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult { } #[tokio::test] -#[serial] async fn test_sts_assume_role_opa_contract() -> TestResult { init_logging(); @@ -559,7 +556,6 @@ async fn test_sts_assume_role_opa_contract() -> TestResult { } #[tokio::test] -#[serial] async fn test_list_buckets_opa_contract() -> TestResult { init_logging(); @@ -645,7 +641,6 @@ async fn test_list_buckets_opa_contract() -> TestResult { } #[tokio::test] -#[serial] async fn test_sts_and_list_buckets_fail_closed_while_opa_is_initializing() -> TestResult { init_logging(); @@ -662,7 +657,6 @@ async fn test_sts_and_list_buckets_fail_closed_while_opa_is_initializing() -> Te } #[tokio::test] -#[serial] async fn test_sts_and_list_buckets_fail_closed_after_opa_validation_failure() -> TestResult { init_logging(); @@ -679,7 +673,6 @@ async fn test_sts_and_list_buckets_fail_closed_after_opa_validation_failure() -> } #[tokio::test] -#[serial] async fn test_sts_query_rate_limit_error_is_aws_sdk_compatible() -> TestResult { init_logging(); diff --git a/crates/e2e_test/src/tier_transition_regression_test.rs b/crates/e2e_test/src/tier_transition_regression_test.rs index 5349e3084..26c2922e1 100644 --- a/crates/e2e_test/src/tier_transition_regression_test.rs +++ b/crates/e2e_test/src/tier_transition_regression_test.rs @@ -30,7 +30,6 @@ mod tests { use crate::common::{RustFSTestEnvironment, admin_ok, init_logging}; use serde_json::Value; - use serial_test::serial; use std::error::Error; use tracing::info; @@ -42,7 +41,6 @@ mod tests { /// validates that an expiration-only rule (the persistence path) survives /// a server restart. #[tokio::test] - #[serial] async fn test_lifecycle_rule_persists_after_restart() -> TestResult { init_logging(); info!("RT-13: lifecycle rule persists after restart"); @@ -105,7 +103,6 @@ mod tests { /// Regression pattern: tier add/verify/delete API fails or the tier /// configuration is not persisted (rustfs#5218). #[tokio::test] - #[serial] async fn test_admin_tier_list_endpoint_returns_json() -> TestResult { init_logging(); info!("RT-13b: admin tier list endpoint returns JSON"); @@ -135,7 +132,6 @@ mod tests { /// is not persisted (rustfs#5013), causing the scanner to not run or /// use stale settings. #[tokio::test] - #[serial] async fn test_scanner_config_persists_after_restart() -> TestResult { init_logging(); info!("RT-13c: scanner config persists after restart"); diff --git a/crates/e2e_test/src/tls_hot_reload_test.rs b/crates/e2e_test/src/tls_hot_reload_test.rs index 59c5894ce..b8f752627 100644 --- a/crates/e2e_test/src/tls_hot_reload_test.rs +++ b/crates/e2e_test/src/tls_hot_reload_test.rs @@ -34,7 +34,6 @@ use rcgen::generate_simple_self_signed; use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; use rustls::{ClientConfig, ClientConnection, DigitallySignedStruct, Error as RustlsError, SignatureScheme, StreamOwned}; -use serial_test::serial; use sha2::{Digest, Sha256}; use std::error::Error; use std::io::{Read, Write}; @@ -242,7 +241,6 @@ async fn roundtrip_and_return(mut session: TlsSession) -> Result TestResult { init_logging(); // Install the process-wide rustls crypto provider (idempotent). diff --git a/crates/e2e_test/src/version_id_regression_test.rs b/crates/e2e_test/src/version_id_regression_test.rs index 1ff2e34ef..75d19e5db 100644 --- a/crates/e2e_test/src/version_id_regression_test.rs +++ b/crates/e2e_test/src/version_id_regression_test.rs @@ -27,7 +27,6 @@ mod tests { use aws_sdk_s3::Client; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration}; - use serial_test::serial; use tracing::info; fn create_s3_client(env: &RustFSTestEnvironment) -> Client { @@ -86,7 +85,6 @@ mod tests { /// Test 1: PutObject should return version_id when versioning is enabled /// This directly addresses the Veeam issue from #1066 #[tokio::test] - #[serial] async fn test_put_object_returns_version_id_with_versioning() { init_logging(); info!("🧪 TEST: PutObject returns version_id with versioning enabled"); @@ -130,7 +128,6 @@ mod tests { /// Test 2: CopyObject should return version_id when versioning is enabled #[tokio::test] - #[serial] async fn test_copy_object_returns_version_id_with_versioning() { init_logging(); info!("🧪 TEST: CopyObject returns version_id with versioning enabled"); @@ -185,7 +182,6 @@ mod tests { /// Test 3: CompleteMultipartUpload should return version_id when versioning is enabled #[tokio::test] - #[serial] async fn test_multipart_upload_returns_version_id_with_versioning() { init_logging(); info!("🧪 TEST: CompleteMultipartUpload returns version_id with versioning enabled"); @@ -260,7 +256,6 @@ mod tests { /// Test 4: PutObject should NOT return version_id when versioning is NOT enabled /// This ensures we didn't break non-versioned buckets #[tokio::test] - #[serial] async fn test_put_object_without_versioning() { init_logging(); info!("🧪 TEST: PutObject behavior without versioning (no regression)"); @@ -296,7 +291,6 @@ mod tests { /// Test 5: Basic S3 operations still work correctly (no regression) #[tokio::test] - #[serial] async fn test_basic_s3_operations_no_regression() { init_logging(); info!("🧪 TEST: Basic S3 operations work correctly (no regression)"); @@ -363,7 +357,6 @@ mod tests { /// Test 6: Veeam-specific scenario simulation /// Simulates the exact workflow that Veeam uses when backing up data #[tokio::test] - #[serial] async fn test_veeam_backup_workflow_simulation() { init_logging(); info!("🧪 TEST: Veeam VBR backup workflow simulation (Issue #1066)"); @@ -413,7 +406,6 @@ mod tests { } #[tokio::test] - #[serial] async fn test_terraform_put_after_delete() -> Result<(), Box> { init_logging(); @@ -456,7 +448,6 @@ mod tests { /// Test 7: PutObject should omit version_id when versioning is Suspended #[tokio::test] - #[serial] async fn test_put_object_omits_version_id_with_suspended_versioning() { init_logging(); info!("🧪 TEST: PutObject omits version_id with versioning suspended"); @@ -500,7 +491,6 @@ mod tests { /// Test 8: CopyObject should omit version_id when versioning is Suspended #[tokio::test] - #[serial] async fn test_copy_object_omits_version_id_with_suspended_versioning() { init_logging(); info!("🧪 TEST: CopyObject omits version_id with versioning suspended"); @@ -551,7 +541,6 @@ mod tests { /// Test 9: CompleteMultipartUpload should omit version_id when versioning is Suspended #[tokio::test] - #[serial] async fn test_multipart_upload_omits_version_id_with_suspended_versioning() { init_logging(); info!("🧪 TEST: CompleteMultipartUpload omits version_id with versioning suspended"); From 13a9d195056c73e504157db47db37498c732bb72 Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 19 Aug 2026 11:02:07 +0800 Subject: [PATCH 05/27] fix(admin): report refreshed lock lease TTLs (#6230) --- crates/lock/src/client/local.rs | 88 +++++- crates/lock/src/client/mod.rs | 9 +- crates/lock/src/fast_lock/manager.rs | 26 +- crates/lock/src/fast_lock/shard.rs | 45 ++- crates/lock/src/lib.rs | 4 +- crates/lock/src/types.rs | 15 + rustfs/src/admin/handlers/diagnostics.rs | 373 ++++++++++++++++++++--- 7 files changed, 491 insertions(+), 69 deletions(-) diff --git a/crates/lock/src/client/local.rs b/crates/lock/src/client/local.rs index 3b2a185aa..95adaffc3 100644 --- a/crates/lock/src/client/local.rs +++ b/crates/lock/src/client/local.rs @@ -47,6 +47,7 @@ pub struct LocalClient { #[derive(Debug)] struct LocalGuardEntry { guard: FastLockGuard, + acquired_at: SystemTime, expires_at: SystemTime, deadline: Instant, ttl: Duration, @@ -54,11 +55,12 @@ struct LocalGuardEntry { impl LocalGuardEntry { fn new(guard: FastLockGuard, ttl: Duration) -> Self { - let now = SystemTime::now(); + let acquired_at = SystemTime::now(); let monotonic_now = Instant::now(); Self { guard, - expires_at: now.checked_add(ttl).unwrap_or(now), + acquired_at, + expires_at: acquired_at.checked_add(ttl).unwrap_or(acquired_at), deadline: monotonic_now.checked_add(ttl).unwrap_or(monotonic_now), ttl, } @@ -231,13 +233,14 @@ impl LockClient for LocalClient { match lock_manager.acquire_lock(build_lock_request(remaining)).await { Ok(guard) => { let lock_id = request.lock_id.clone(); - let acquired_at = SystemTime::now(); - let expires_at = acquired_at.checked_add(request.ttl).unwrap_or(acquired_at); + let entry = LocalGuardEntry::new(guard, request.ttl); + let acquired_at = entry.acquired_at; + let expires_at = entry.expires_at; { let shard = self.get_shard(&lock_id); let mut guards = shard.write().await; - guards.insert(lock_id.clone(), LocalGuardEntry::new(guard, request.ttl)); + guards.insert(lock_id.clone(), entry); } let lock_info = LockInfo { @@ -342,7 +345,7 @@ impl LockClient for LocalClient { lock_type, status, owner: entry.guard.owner().to_string(), - acquired_at: SystemTime::now(), + acquired_at: entry.acquired_at, expires_at: entry.expires_at, last_refreshed: SystemTime::now(), metadata: LockMetadata::default(), @@ -354,6 +357,25 @@ impl LockClient for LocalClient { } } + async fn list_lock_leases(&self) -> Vec { + let mut leases = Vec::new(); + for shard in self.guard_storage.iter() { + let guards = shard.read().await; + leases.reserve(guards.len()); + leases.extend(guards.iter().map(|(lock_id, entry)| crate::LockLeaseInfo { + resource: lock_id.resource.clone(), + lock_type: match entry.guard.mode() { + crate::LockMode::Shared => LockType::Shared, + crate::LockMode::Exclusive => LockType::Exclusive, + }, + owner: entry.guard.owner().to_string(), + acquired_at: entry.acquired_at, + remaining_ttl: entry.deadline.saturating_duration_since(Instant::now()), + })); + } + leases + } + async fn get_stats(&self) -> Result { Ok(LockStats::default()) } @@ -403,6 +425,10 @@ mod tests { assert!(client.check_status(&lock_id).await.unwrap().is_some()); tokio::time::sleep(Duration::from_millis(15)).await; wait_until_reaped(&client, &lock_id).await; + assert!( + client.list_lock_leases().await.is_empty(), + "reaped guards must disappear from lease diagnostics" + ); let direct = manager .acquire_lock(crate::ObjectLockRequest::new_write(request.resource.clone(), "owner-b")) @@ -442,6 +468,56 @@ mod tests { wait_until_reaped(&client, &lock_id).await; } + #[tokio::test(start_paused = true)] + async fn lease_snapshot_tracks_refresh_without_resetting_acquisition_time() { + let manager = Arc::new(GlobalLockManager::new()); + let client = LocalClient::with_manager_and_reaper_interval(manager, Duration::from_secs(60)); + client.reaper_started.store(true, Ordering::Release); + let lock_request = request(crate::ObjectKey::new("bucket", "lease-snapshot"), "owner-a", Duration::from_secs(30)); + let lock_id = lock_request.lock_id.clone(); + + assert!( + client + .acquire_lock(&lock_request) + .await + .expect("lease-backed lock should acquire") + .success + ); + let initial = client.list_lock_leases().await.pop().expect("acquired lock should be listed"); + + tokio::time::advance(Duration::from_secs(20)).await; + let aging = client + .list_lock_leases() + .await + .pop() + .expect("held lock should remain listed before refresh"); + assert_eq!(aging.remaining_ttl, Duration::from_secs(10)); + assert!(client.refresh(&lock_id).await.expect("refresh should return a result")); + + let refreshed = client + .list_lock_leases() + .await + .pop() + .expect("refreshed lock should be listed"); + let status = client + .check_status(&lock_id) + .await + .expect("lock status should be readable") + .expect("refreshed lock should remain held"); + + assert_eq!(refreshed.acquired_at, initial.acquired_at); + assert_eq!(status.acquired_at, initial.acquired_at); + assert_eq!(refreshed.remaining_ttl, Duration::from_secs(30)); + + tokio::time::advance(Duration::from_secs(30)).await; + let expired = client + .list_lock_leases() + .await + .pop() + .expect("unreaped lease should remain listed"); + assert_eq!(expired.remaining_ttl, Duration::ZERO); + } + #[tokio::test(start_paused = true)] async fn refresh_after_expiry_releases_guard_without_reviving_it() { let manager = Arc::new(GlobalLockManager::new()); diff --git a/crates/lock/src/client/mod.rs b/crates/lock/src/client/mod.rs index d16db4bb6..bf378c3bf 100644 --- a/crates/lock/src/client/mod.rs +++ b/crates/lock/src/client/mod.rs @@ -15,7 +15,7 @@ pub mod local; // pub mod remote; -use crate::{LockId, LockInfo, LockRequest, LockResponse, LockStats, Result}; +use crate::{LockId, LockInfo, LockLeaseInfo, LockRequest, LockResponse, LockStats, Result}; use async_trait::async_trait; use futures::future::join_all; use std::sync::Arc; @@ -54,6 +54,13 @@ pub trait LockClient: Send + Sync + std::fmt::Debug { /// Check lock status async fn check_status(&self, lock_id: &LockId) -> Result>; + /// Return authoritative lease information when this client owns lease state. + /// + /// Clients that do not manage renewable leases return an empty snapshot. + async fn list_lock_leases(&self) -> Vec { + Vec::new() + } + /// Get statistics async fn get_stats(&self) -> Result; diff --git a/crates/lock/src/fast_lock/manager.rs b/crates/lock/src/fast_lock/manager.rs index b24f96d6f..30a1c4d18 100644 --- a/crates/lock/src/fast_lock/manager.rs +++ b/crates/lock/src/fast_lock/manager.rs @@ -295,9 +295,17 @@ impl FastObjectLockManager { /// Powers the admin "top locks" view. Order is shard-then-insertion and is /// not otherwise stable across calls. pub fn list_locks(&self) -> Vec { + self.list_locks_with_holder_counts() + .into_iter() + .map(|(info, _)| info) + .collect() + } + + /// Enumerate held locks with the number of guards represented by each owner. + pub fn list_locks_with_holder_counts(&self) -> Vec<(crate::fast_lock::types::ObjectLockInfo, u32)> { let mut infos = Vec::new(); for shard in &self.shards { - infos.extend(shard.list_locks()); + infos.extend(shard.list_locks_with_holder_counts()); } infos } @@ -556,6 +564,10 @@ mod tests { .acquire_read_lock(read_key.clone(), "reader") .await .expect("read lock should acquire"); + let _second_read_guard = manager + .acquire_read_lock(read_key.clone(), "reader") + .await + .expect("second read lock should acquire"); let mut locks = manager.list_locks(); locks.sort_by(|a, b| a.key.object.cmp(&b.key.object)); @@ -569,6 +581,18 @@ mod tests { assert_eq!(write.mode, LockMode::Exclusive); assert_eq!(write.owner.as_ref(), "writer"); + let counts = manager.list_locks_with_holder_counts(); + let (_, read_holder_count) = counts + .iter() + .find(|(info, _)| info.key == read_key) + .expect("read holder count listed"); + assert_eq!(*read_holder_count, 2); + let (_, write_holder_count) = counts + .iter() + .find(|(info, _)| info.key == write_key) + .expect("write holder count listed"); + assert_eq!(*write_holder_count, 1); + manager.shutdown().await; } diff --git a/crates/lock/src/fast_lock/shard.rs b/crates/lock/src/fast_lock/shard.rs index a0ca7eaa7..8b55b968b 100644 --- a/crates/lock/src/fast_lock/shard.rs +++ b/crates/lock/src/fast_lock/shard.rs @@ -544,6 +544,13 @@ impl LockShard { /// holder. Entries for objects that are tracked but not currently locked /// (e.g. pooled-but-idle state) are skipped. pub fn list_locks(&self) -> Vec { + self.list_locks_with_holder_counts() + .into_iter() + .map(|(info, _)| info) + .collect() + } + + pub(crate) fn list_locks_with_holder_counts(&self) -> Vec<(crate::fast_lock::types::ObjectLockInfo, u32)> { let objects = self.objects.read(); let mut infos = Vec::new(); for (key, state) in objects.iter() { @@ -558,14 +565,17 @@ impl LockShard { .acquired_at .checked_add(info.lock_timeout) .unwrap_or_else(|| info.acquired_at + crate::fast_lock::DEFAULT_LOCK_TIMEOUT); - infos.push(crate::fast_lock::types::ObjectLockInfo { - key: key.clone(), - mode, - owner: info.owner, - acquired_at: info.acquired_at, - expires_at, - priority, - }); + infos.push(( + crate::fast_lock::types::ObjectLockInfo { + key: key.clone(), + mode, + owner: info.owner, + acquired_at: info.acquired_at, + expires_at, + priority, + }, + 1, + )); } } LockMode::Shared => { @@ -574,14 +584,17 @@ impl LockShard { .acquired_at .checked_add(entry.lock_timeout) .unwrap_or_else(|| entry.acquired_at + crate::fast_lock::DEFAULT_LOCK_TIMEOUT); - infos.push(crate::fast_lock::types::ObjectLockInfo { - key: key.clone(), - mode, - owner: entry.owner.clone(), - acquired_at: entry.acquired_at, - expires_at, - priority, - }); + infos.push(( + crate::fast_lock::types::ObjectLockInfo { + key: key.clone(), + mode, + owner: entry.owner.clone(), + acquired_at: entry.acquired_at, + expires_at, + priority, + }, + entry.count, + )); } } } diff --git a/crates/lock/src/lib.rs b/crates/lock/src/lib.rs index 6562f8675..be1f1c68d 100644 --- a/crates/lock/src/lib.rs +++ b/crates/lock/src/lib.rs @@ -51,8 +51,8 @@ pub use crate::{ namespace::{NamespaceLock, NamespaceLockGuard, NamespaceLockWrapper}, // Core types types::{ - HealthInfo, HealthStatus, LockId, LockInfo, LockMetadata, LockPriority, LockRequest, LockResponse, LockStats, LockStatus, - LockType, + HealthInfo, HealthStatus, LockId, LockInfo, LockLeaseInfo, LockMetadata, LockPriority, LockRequest, LockResponse, + LockStats, LockStatus, LockType, }, }; diff --git a/crates/lock/src/types.rs b/crates/lock/src/types.rs index eebe158ff..5336fb93e 100644 --- a/crates/lock/src/types.rs +++ b/crates/lock/src/types.rs @@ -79,6 +79,21 @@ pub struct LockInfo { pub wait_start_time: Option, } +/// Point-in-time lease information exposed by lock clients for diagnostics. +#[derive(Debug, Clone)] +pub struct LockLeaseInfo { + /// Resource protected by the lock. + pub resource: ObjectKey, + /// Shared or exclusive lock mode. + pub lock_type: LockType, + /// Lock owner recorded by the local lock backend. + pub owner: String, + /// Original acquisition time. Refreshes do not change this value. + pub acquired_at: SystemTime, + /// Remaining lease duration derived from the monotonic lease deadline. + pub remaining_ttl: Duration, +} + impl LockInfo { /// Check if the lock has expired pub fn has_expired(&self) -> bool { diff --git a/rustfs/src/admin/handlers/diagnostics.rs b/rustfs/src/admin/handlers/diagnostics.rs index 523df19db..35410c159 100644 --- a/rustfs/src/admin/handlers/diagnostics.rs +++ b/rustfs/src/admin/handlers/diagnostics.rs @@ -28,17 +28,19 @@ use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::storage_api::access::spawn_traced; use crate::auth::{check_key_valid, get_session_token}; use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use crate::storage::storage_api::get_global_lock_clients; use bytes::Bytes; -use futures::{Stream, StreamExt}; +use futures::{Stream, StreamExt, future::join_all}; use http::{HeaderMap, HeaderValue, Uri, header::CONTENT_LENGTH}; use hyper::{Method, StatusCode}; use matchit::Params; -use rustfs_lock::{LockMode, ObjectKey, get_global_lock_manager}; +use rustfs_lock::{LockLeaseInfo, LockMode, LockType, ObjectKey, get_global_lock_manager}; use rustfs_policy::policy::action::{Action, AdminAction}; use s3s::header::CONTENT_TYPE; use s3s::stream::{ByteStream, DynByteStream}; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, StdError, s3_error}; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::pin::Pin; use std::task::{Context, Poll}; use std::time::{Duration, SystemTime}; @@ -212,7 +214,142 @@ fn system_time_to_rfc3339(t: SystemTime) -> Option { dt.format(&time::format_description::well_known::Rfc3339).ok() } -fn collect_top_locks(limit: usize) -> TopLocksResponse { +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum TopLockMode { + Read, + Write, +} + +impl TopLockMode { + fn label(self) -> &'static str { + match self { + Self::Read => "READ", + Self::Write => "WRITE", + } + } +} + +#[derive(Debug, PartialEq, Eq, Hash)] +struct LockHolderKey { + resource: ObjectKey, + mode: TopLockMode, + owner: String, +} + +#[derive(Debug)] +struct TopLockState { + acquired_at: SystemTime, + ttl_secs: u64, + priority: &'static str, +} + +#[derive(Debug)] +struct LeaseHolderState { + acquired_at: SystemTime, + ttl_secs: u64, + holder_count: u32, +} + +fn build_top_locks_response( + limit: usize, + now: SystemTime, + lease_infos: Vec, + fast_infos: Vec<(rustfs_lock::ObjectLockInfo, u32)>, +) -> TopLocksResponse { + let mut lease_holders = HashMap::with_capacity(lease_infos.len()); + + for info in lease_infos { + let mode = match info.lock_type { + LockType::Shared => TopLockMode::Read, + LockType::Exclusive => TopLockMode::Write, + }; + let key = LockHolderKey { + resource: info.resource, + mode, + owner: info.owner, + }; + let ttl_secs = info.remaining_ttl.as_secs(); + lease_holders + .entry(key) + .and_modify(|state: &mut LeaseHolderState| { + if info.acquired_at < state.acquired_at { + state.acquired_at = info.acquired_at; + } + state.ttl_secs = state.ttl_secs.max(ttl_secs); + state.holder_count = state.holder_count.saturating_add(1); + }) + .or_insert(LeaseHolderState { + acquired_at: info.acquired_at, + ttl_secs, + holder_count: 1, + }); + } + + let mut infos: Vec<_> = fast_infos + .into_iter() + .map(|(info, holder_count)| { + let mode = match info.mode { + LockMode::Shared => TopLockMode::Read, + LockMode::Exclusive => TopLockMode::Write, + }; + let key = LockHolderKey { + resource: info.key, + mode, + owner: info.owner.to_string(), + }; + let priority = lock_priority_label(info.priority); + // Shared-owner timestamps do not roll back when a newer sibling releases, so only their count is stable. + let state = match lease_holders.remove(&key) { + Some(lease) + if lease.holder_count == holder_count + && (mode == TopLockMode::Read || info.acquired_at <= lease.acquired_at) => + { + TopLockState { + acquired_at: lease.acquired_at, + ttl_secs: lease.ttl_secs, + priority, + } + } + _ => TopLockState { + acquired_at: info.acquired_at, + ttl_secs: info.expires_at.duration_since(now).unwrap_or(Duration::ZERO).as_secs(), + priority, + }, + }; + (key, state) + }) + .collect(); + // Longest-held first, matching MinIO's `top locks` ordering intent. + infos.sort_by_key(|(_, state)| state.acquired_at); + let total = infos.len(); + let truncated = total > limit; + + let locks = infos + .into_iter() + .take(limit) + .map(|(holder, state)| LockEntry { + resource: format!("{}/{}", holder.resource.bucket, holder.resource.object), + bucket: holder.resource.bucket.to_string(), + object: holder.resource.object.to_string(), + version: holder.resource.version.as_ref().map(|version| version.to_string()), + lock_type: holder.mode.label(), + owner: holder.owner, + priority: state.priority, + since: system_time_to_rfc3339(state.acquired_at), + elapsed_secs: now.duration_since(state.acquired_at).unwrap_or(Duration::ZERO).as_secs(), + ttl_secs: state.ttl_secs, + }) + .collect(); + + TopLocksResponse { + total, + truncated, + locks, + capability_note: None, + } +} + +async fn collect_top_locks(limit: usize) -> TopLocksResponse { let manager = get_global_lock_manager(); let Some(fast) = manager.as_fast_lock_manager() else { return TopLocksResponse { @@ -225,43 +362,19 @@ fn collect_top_locks(limit: usize) -> TopLocksResponse { }; }; - let now = SystemTime::now(); - let mut infos = fast.list_locks(); - // Longest-held first, matching MinIO's `top locks` ordering intent. - infos.sort_by_key(|i| i.acquired_at); - let total = infos.len(); - let truncated = total > limit; + let lease_infos = if let Some(clients) = get_global_lock_clients() { + join_all(clients.values().map(|client| client.list_lock_leases())) + .await + .into_iter() + .flatten() + .collect() + } else { + Vec::new() + }; + // Capture holders last so released or replaced lease guards fail the merge checks. + let fast_infos = fast.list_locks_with_holder_counts(); - let locks = infos - .into_iter() - .take(limit) - .map(|info| { - let elapsed_secs = now.duration_since(info.acquired_at).unwrap_or(Duration::ZERO).as_secs(); - let ttl_secs = info.expires_at.duration_since(now).unwrap_or(Duration::ZERO).as_secs(); - LockEntry { - resource: format!("{}/{}", info.key.bucket, info.key.object), - bucket: info.key.bucket.to_string(), - object: info.key.object.to_string(), - version: info.key.version.as_ref().map(|v| v.to_string()), - lock_type: match info.mode { - LockMode::Exclusive => "WRITE", - LockMode::Shared => "READ", - }, - owner: info.owner.to_string(), - priority: lock_priority_label(info.priority), - since: system_time_to_rfc3339(info.acquired_at), - elapsed_secs, - ttl_secs, - } - }) - .collect(); - - TopLocksResponse { - total, - truncated, - locks, - capability_note: None, - } + build_top_locks_response(limit, SystemTime::now(), lease_infos, fast_infos) } fn parse_top_locks_limit(uri: &Uri) -> usize { @@ -279,7 +392,7 @@ impl Operation for TopLocksHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { authorize(&req, AdminAction::TopLocksAdminAction).await?; let limit = parse_top_locks_limit(&req.uri); - let response = collect_top_locks(limit); + let response = collect_top_locks(limit).await; json_response(StatusCode::OK, &response) } } @@ -1107,6 +1220,180 @@ mod tests { assert_eq!(parse_top_locks_limit(&Uri::from_static("/x?count=999999")), TOP_LOCKS_MAX_LIMIT); } + #[test] + fn top_locks_prefers_renewable_lease_deadlines() { + let now = SystemTime::UNIX_EPOCH + Duration::from_secs(100); + let leased_resource = ObjectKey::new("bucket", "shared-object"); + let exclusive_resource = ObjectKey::new("bucket", "write-object"); + let direct_resource = ObjectKey::new("bucket", "direct-object"); + let mixed_resource = ObjectKey::new("bucket", "mixed-object"); + let replaced_resource = ObjectKey::new("bucket", "replaced-object"); + let remaining_shared_resource = ObjectKey::new("bucket", "remaining-shared-object"); + + let response = build_top_locks_response( + TOP_LOCKS_DEFAULT_LIMIT, + now, + vec![ + LockLeaseInfo { + resource: leased_resource.clone(), + lock_type: LockType::Shared, + owner: "owner-a".to_string(), + acquired_at: now - Duration::from_secs(50), + remaining_ttl: Duration::from_secs(5), + }, + LockLeaseInfo { + resource: leased_resource.clone(), + lock_type: LockType::Shared, + owner: "owner-a".to_string(), + acquired_at: now - Duration::from_secs(40), + remaining_ttl: Duration::from_secs(20), + }, + LockLeaseInfo { + resource: mixed_resource.clone(), + lock_type: LockType::Shared, + owner: "owner-c".to_string(), + acquired_at: now - Duration::from_secs(30), + remaining_ttl: Duration::from_secs(25), + }, + LockLeaseInfo { + resource: exclusive_resource.clone(), + lock_type: LockType::Exclusive, + owner: "owner-d".to_string(), + acquired_at: now - Duration::from_secs(15), + remaining_ttl: Duration::from_secs(18), + }, + LockLeaseInfo { + resource: replaced_resource.clone(), + lock_type: LockType::Exclusive, + owner: "owner-e".to_string(), + acquired_at: now - Duration::from_secs(30), + remaining_ttl: Duration::from_secs(25), + }, + LockLeaseInfo { + resource: remaining_shared_resource.clone(), + lock_type: LockType::Shared, + owner: "owner-f".to_string(), + acquired_at: now - Duration::from_secs(30), + remaining_ttl: Duration::from_secs(22), + }, + ], + vec![ + ( + rustfs_lock::ObjectLockInfo { + key: replaced_resource, + mode: LockMode::Exclusive, + owner: "owner-e".into(), + acquired_at: now - Duration::from_secs(5), + expires_at: now + Duration::from_secs(4), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 1, + ), + ( + rustfs_lock::ObjectLockInfo { + key: remaining_shared_resource, + mode: LockMode::Shared, + owner: "owner-f".into(), + acquired_at: now - Duration::from_secs(5), + expires_at: now + Duration::from_secs(3), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 1, + ), + ( + rustfs_lock::ObjectLockInfo { + key: exclusive_resource, + mode: LockMode::Exclusive, + owner: "owner-d".into(), + acquired_at: now - Duration::from_secs(15), + expires_at: now + Duration::from_secs(2), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 1, + ), + ( + rustfs_lock::ObjectLockInfo { + key: leased_resource, + mode: LockMode::Shared, + owner: "owner-a".into(), + acquired_at: now - Duration::from_secs(50), + expires_at: now + Duration::from_secs(1), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 2, + ), + ( + rustfs_lock::ObjectLockInfo { + key: direct_resource, + mode: LockMode::Exclusive, + owner: "owner-b".into(), + acquired_at: now - Duration::from_secs(10), + expires_at: now + Duration::from_secs(7), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 1, + ), + ( + rustfs_lock::ObjectLockInfo { + key: mixed_resource, + mode: LockMode::Shared, + owner: "owner-c".into(), + acquired_at: now - Duration::from_secs(30), + expires_at: now + Duration::from_secs(9), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 2, + ), + ], + ); + + assert_eq!(response.total, 6); + let leased = response + .locks + .iter() + .find(|entry| entry.object == "shared-object") + .expect("lease-backed shared owner should be listed once"); + assert_eq!(leased.lock_type, "READ"); + assert_eq!(leased.elapsed_secs, 50); + assert_eq!(leased.ttl_secs, 20); + + let exclusive = response + .locks + .iter() + .find(|entry| entry.object == "write-object") + .expect("lease-backed exclusive holder should be listed"); + assert_eq!(exclusive.lock_type, "WRITE"); + assert_eq!(exclusive.ttl_secs, 18); + + let direct = response + .locks + .iter() + .find(|entry| entry.object == "direct-object") + .expect("direct fast lock should remain visible"); + assert_eq!(direct.ttl_secs, 7); + + let mixed = response + .locks + .iter() + .find(|entry| entry.object == "mixed-object") + .expect("mixed direct and leased shared holders should remain visible"); + assert_eq!(mixed.ttl_secs, 9); + + let replaced = response + .locks + .iter() + .find(|entry| entry.object == "replaced-object") + .expect("a replaced lease holder should remain visible"); + assert_eq!(replaced.ttl_secs, 4); + + let remaining_shared = response + .locks + .iter() + .find(|entry| entry.object == "remaining-shared-object") + .expect("an older surviving shared lease should remain lease-backed"); + assert_eq!(remaining_shared.ttl_secs, 22); + } + #[tokio::test] async fn collect_top_locks_reports_live_lock() { // Acquire a real lock through the global manager and confirm it surfaces. @@ -1115,20 +1402,20 @@ mod tests { // The fast-lock manager exposes the acquire API; if the lock subsystem is // disabled in this environment, the response must carry a capability note. let Some(fast) = manager.as_fast_lock_manager() else { - let response = collect_top_locks(TOP_LOCKS_DEFAULT_LIMIT); + let response = collect_top_locks(TOP_LOCKS_DEFAULT_LIMIT).await; assert!(response.capability_note.is_some() || response.locks.is_empty()); return; }; let guard = match fast.acquire_write_lock(key.clone(), "diag-owner").await { Ok(g) => g, Err(_) => { - let response = collect_top_locks(TOP_LOCKS_DEFAULT_LIMIT); + let response = collect_top_locks(TOP_LOCKS_DEFAULT_LIMIT).await; assert!(response.capability_note.is_some() || response.locks.is_empty()); return; } }; - let response = collect_top_locks(TOP_LOCKS_DEFAULT_LIMIT); + let response = collect_top_locks(TOP_LOCKS_DEFAULT_LIMIT).await; let found = response .locks .iter() From 19cbba7ec816d1bcc933019f5733059f13f0c802 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 19 Aug 2026 11:02:18 +0800 Subject: [PATCH 06/27] refactor(app): resolve the bucket default SSE in one place (#6236) --- rustfs/src/app/object_usecase.rs | 192 +++++++++++++++++++------------ 1 file changed, 121 insertions(+), 71 deletions(-) diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index b63110d39..55d863588 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -169,8 +169,8 @@ use s3s::dto::{ ObjectLockLegalHoldStatus, ObjectLockMode, ObjectLockRetention, ObjectLockRetentionMode, ObjectPart, PutObjectInput, PutObjectOutput, Range, RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreStatus, SSECustomerAlgorithm, SSECustomerKeyMD5, SSEKMSKeyId, SelectObjectContentInput, SelectObjectContentOutput, ServerSideEncryption, - ServerSideEncryptionByDefault, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader, Timestamp, TimestampFormat, - WebsiteRedirectLocation, + ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, + TaggingHeader, Timestamp, TimestampFormat, WebsiteRedirectLocation, }; use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH}; use s3s::stream::{ByteStream, DynByteStream, RemainingLength}; @@ -2695,6 +2695,39 @@ fn bucket_default_write_sse(sse: &ServerSideEncryptionByDefault) -> ServerSideEn } } +/// Resolve the effective server-side encryption for a write against the bucket's +/// default encryption configuration. +/// +/// A request-level value always wins; the bucket default only fills a gap, and +/// the unknown-algorithm fallback lives once in [`bucket_default_write_sse`]. +/// +/// `has_explicit_ssec` suppresses the default entirely. Only COPY passes `true` +/// today: its destination may carry SSE-C, which must not also be given managed +/// encryption. PUT and extract pass `false`, matching their current behaviour — +/// see backlog#1826 for the divergence that leaves. +/// +/// Callers layering further overrides (PUT's `ciphertext_passthrough`) apply +/// them to the returned pair. +fn resolve_bucket_default_sse( + bucket_sse_config: Option<&ServerSideEncryptionConfiguration>, + requested_sse: Option, + requested_kms_key_id: Option, + has_explicit_ssec: bool, +) -> (Option, Option) { + let bucket_default = || { + if has_explicit_ssec { + return None; + } + bucket_sse_config + .and_then(|config| config.rules.first()) + .and_then(|rule| rule.apply_server_side_encryption_by_default.as_ref()) + }; + + let effective_sse = requested_sse.or_else(|| bucket_default().map(bucket_default_write_sse)); + let effective_kms_key_id = requested_kms_key_id.or_else(|| bucket_default().and_then(|sse| sse.kms_master_key_id.clone())); + (effective_sse, effective_kms_key_id) +} + fn should_use_small_eager_put_path( size: i64, headers: &HeaderMap, @@ -5823,19 +5856,12 @@ impl DefaultObjectUsecase { ); let original_sse = server_side_encryption.clone(); - let mut effective_sse = server_side_encryption.or_else(|| { - bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default.as_ref().map(|sse| { - match sse.sse_algorithm.as_str() { - "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), - _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), // fallback to AES256 - } - }) - }) - }) - }); + let (mut effective_sse, mut effective_kms_key_id) = resolve_bucket_default_sse( + bucket_sse_config.as_ref().map(|(config, _timestamp)| config), + server_side_encryption, + ssekms_key_id, + false, + ); debug!( target: "rustfs::app::object_usecase", component = "app", @@ -5847,16 +5873,6 @@ impl DefaultObjectUsecase { "Resolved effective SSE configuration" ); - let mut effective_kms_key_id = ssekms_key_id.or_else(|| { - bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default - .as_ref() - .and_then(|sse| sse.kms_master_key_id.clone()) - }) - }) - }); - if ciphertext_passthrough { // The replica keeps the source's SSE-C metadata; the bucket // default must not claim managed encryption on it. @@ -7658,30 +7674,12 @@ impl DefaultObjectUsecase { } }; - let mut effective_sse = requested_sse.or_else(|| { - if has_explicit_ssec { - return None; - } - bucket_sse_config.as_ref().and_then(|(config, _)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default - .as_ref() - .map(bucket_default_write_sse) - }) - }) - }); - let mut effective_kms_key_id = requested_kms_key_id.or_else(|| { - if has_explicit_ssec { - return None; - } - bucket_sse_config.as_ref().and_then(|(config, _)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default - .as_ref() - .and_then(|sse| sse.kms_master_key_id.clone()) - }) - }) - }); + let (mut effective_sse, mut effective_kms_key_id) = resolve_bucket_default_sse( + bucket_sse_config.as_ref().map(|(config, _)| config), + requested_sse, + requested_kms_key_id, + has_explicit_ssec, + ); let h = build_ssec_read_headers( copy_source_sse_customer_algorithm.as_ref(), @@ -9587,28 +9585,12 @@ impl DefaultObjectUsecase { let original_sse = server_side_encryption.or(extract_server_side_encryption_from_headers(&req.headers)?); let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok(); - let mut effective_sse = original_sse.or_else(|| { - bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default - .as_ref() - .map(|sse| match sse.sse_algorithm.as_str() { - "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), - _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - }) - }) - }) - }); - let mut effective_kms_key_id = ssekms_key_id.or_else(|| { - bucket_sse_config.as_ref().and_then(|(config, _timestamp)| { - config.rules.first().and_then(|rule| { - rule.apply_server_side_encryption_by_default - .as_ref() - .and_then(|sse| sse.kms_master_key_id.clone()) - }) - }) - }); + let (mut effective_sse, mut effective_kms_key_id) = resolve_bucket_default_sse( + bucket_sse_config.as_ref().map(|(config, _timestamp)| config), + original_sse, + ssekms_key_id, + false, + ); if effective_sse .as_ref() .is_some_and(|sse| sse.as_str().eq_ignore_ascii_case(ServerSideEncryption::AWS_KMS)) @@ -10397,6 +10379,74 @@ mod tests { } } + fn bucket_sse_config_with(algorithm: &str, kms_key_id: Option<&str>) -> ServerSideEncryptionConfiguration { + ServerSideEncryptionConfiguration { + rules: vec![ServerSideEncryptionRule { + apply_server_side_encryption_by_default: Some(ServerSideEncryptionByDefault { + sse_algorithm: ServerSideEncryption::from(String::from(algorithm)), + kms_master_key_id: kms_key_id.map(|id| SSEKMSKeyId::from(id.to_string())), + }), + bucket_key_enabled: None, + }], + } + } + + #[test] + fn resolve_bucket_default_sse_prefers_the_request_over_the_bucket_default() { + let config = bucket_sse_config_with(ServerSideEncryption::AWS_KMS, Some("bucket-key")); + + let (sse, kms_key_id) = resolve_bucket_default_sse( + Some(&config), + Some(ServerSideEncryption::from_static(ServerSideEncryption::AES256)), + Some(SSEKMSKeyId::from("request-key".to_string())), + false, + ); + + assert_eq!(sse.as_ref().map(|sse| sse.as_str()), Some(ServerSideEncryption::AES256)); + assert_eq!(kms_key_id.as_deref(), Some("request-key")); + } + + #[test] + fn resolve_bucket_default_sse_fills_gaps_from_the_bucket_default() { + let config = bucket_sse_config_with(ServerSideEncryption::AWS_KMS, Some("bucket-key")); + + let (sse, kms_key_id) = resolve_bucket_default_sse(Some(&config), None, None, false); + + assert_eq!(sse.as_ref().map(|sse| sse.as_str()), Some(ServerSideEncryption::AWS_KMS)); + assert_eq!(kms_key_id.as_deref(), Some("bucket-key")); + } + + #[test] + fn resolve_bucket_default_sse_falls_back_to_aes256_for_an_unknown_algorithm() { + // Reachable only through corrupt or hand-edited bucket metadata; + // PutBucketEncryption rejects unknown algorithms. All three call sites + // now share this single decision (backlog#1826). + let config = bucket_sse_config_with("garbage", None); + + let (sse, kms_key_id) = resolve_bucket_default_sse(Some(&config), None, None, false); + + assert_eq!(sse.as_ref().map(|sse| sse.as_str()), Some(ServerSideEncryption::AES256)); + assert!(kms_key_id.is_none()); + } + + #[test] + fn resolve_bucket_default_sse_suppresses_the_default_for_explicit_ssec() { + let config = bucket_sse_config_with(ServerSideEncryption::AES256, Some("bucket-key")); + + let (sse, kms_key_id) = resolve_bucket_default_sse(Some(&config), None, None, true); + + assert!(sse.is_none(), "an SSE-C destination must not also get managed encryption"); + assert!(kms_key_id.is_none()); + } + + #[test] + fn resolve_bucket_default_sse_returns_nothing_without_a_bucket_default() { + let (sse, kms_key_id) = resolve_bucket_default_sse(None, None, None, false); + + assert!(sse.is_none()); + assert!(kms_key_id.is_none()); + } + #[test] fn put_request_user_metadata_cannot_suppress_bucket_default_retention() { let mut metadata = From b8686b471a8ea6ff2ed3b80325c7d36febeac0ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E5=B0=8F=E9=B8=AD?= Date: Wed, 19 Aug 2026 11:41:05 +0800 Subject: [PATCH 07/27] fix(site-replication): clear pending_remove on join, surface failures (#6244) --- crates/madmin/src/site_replication.rs | 10 + rustfs/src/admin/handlers/site_replication.rs | 714 +++++++++++++++--- scripts/test/site_replication_smoke.py | 114 ++- 3 files changed, 732 insertions(+), 106 deletions(-) diff --git a/crates/madmin/src/site_replication.rs b/crates/madmin/src/site_replication.rs index 740924405..1dcee9350 100644 --- a/crates/madmin/src/site_replication.rs +++ b/crates/madmin/src/site_replication.rs @@ -83,6 +83,16 @@ pub struct SiteReplicationInfo { pub service_account_access_key: String, #[serde(rename = "apiVersion", skip_serializing_if = "Option::is_none")] pub api_version: Option, + /// Outstanding peer deliveries. Absent when the retry queue is empty, so a + /// healthy site serializes exactly as it did before this field existed. + /// Present means peer operations are failing even if `enabled` is true. + #[serde(rename = "retryStats", default, skip_serializing_if = "Option::is_none")] + pub retry_stats: Option, + /// A multi-step lifecycle operation this site has not finished — most + /// importantly a removal that could not reach its peers, which makes the + /// site reject peer operations while `enabled` may still read true. + #[serde(rename = "pendingOperation", default, skip_serializing_if = "Option::is_none")] + pub pending_operation: Option, } #[derive(Debug, Clone, Serialize, Deserialize, Default)] diff --git a/rustfs/src/admin/handlers/site_replication.rs b/rustfs/src/admin/handlers/site_replication.rs index 219ebbffe..1bb0379df 100644 --- a/rustfs/src/admin/handlers/site_replication.rs +++ b/rustfs/src/admin/handlers/site_replication.rs @@ -126,6 +126,10 @@ const SITE_REPLICATION_JOIN_ADMISSION_LOCK_PATH: &str = "config/site-replication const SITE_REPL_ADD_SUCCESS: &str = "Requested sites were configured for replication successfully."; const SITE_REPL_EDIT_SUCCESS: &str = "Requested site was updated successfully."; const SITE_REPL_REMOVE_SUCCESS: &str = "Requested site(s) were removed from cluster replication successfully."; +/// Local removal committed, but at least one peer could not be told. The +/// cluster is diverged until the removal finishes — the reconcile tick keeps +/// retrying it, and `replicate info` reports the pending operation meanwhile. +const SITE_REPL_REMOVE_PARTIAL: &str = "Partial"; const SITE_REPL_RESYNC_START: &str = "start"; const SITE_REPL_RESYNC_CANCEL: &str = "cancel"; const SITE_REPL_RESYNC_STATUS: &str = "status"; @@ -713,6 +717,16 @@ struct SRPeerJoinResponse { peer: PeerInfo, #[serde(rename = "initialSyncErrorMessage", default, skip_serializing_if = "String::is_empty")] initial_sync_error_message: String, + /// Whether the receiving site actually applied this join. + /// + /// Three-valued on purpose. `None` means the peer did not report — MinIO + /// answers a successful `SRPeerJoin` with an empty body, and RustFS peers + /// older than this field say nothing either — so the initiator must NOT + /// read it as a failure. `Some(false)` is an explicit no-op: the peer had + /// already moved past the snapshot it was sent and wrote nothing, which + /// used to be indistinguishable from success (rustfs/rustfs#5963). + #[serde(default, skip_serializing_if = "Option::is_none")] + applied: Option, } #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -2568,6 +2582,21 @@ fn apply_peer_join( state.peers = normalize_join_peers_for_local(local_peer, join_req.peers); initialize_join_peer_sync_state(&mut state.peers, defer_sync_state_enable); state.sync_state_initialized = true; + // An accepted join supersedes a half-finished removal this site started: + // the sender's snapshot IS the new topology, while the pending record only + // exists to keep notifying peers about the OLD one. Leaving it set is what + // kept a recovered site rejecting every peer bucket-op forever — + // `SRPeerBucketOpsHandler` short-circuits on `pending_remove` BEFORE it + // consults `enabled()`, so a successful re-add restored the topology on + // both sides while replication stayed dead (rustfs/rustfs#5963). + // + // Safe against a concurrent removal: `SiteReplicationRemoveHandler` and + // the join admission both hold the lifecycle guard, so a join is only ever + // admitted before that handler starts or after it has returned. + // + // Deliberately NOT cleared here: the peer-edit high-water marks (see this + // function's doc comment) — those fence edit ordering, not lifecycle. + state.pending_remove = None; state.name = state .peers .get(&local_peer.deployment_id) @@ -3002,8 +3031,17 @@ fn reconcile_site_replication_wiring() -> std::pin::Pin { - if state.pending_endpoint_refresh.is_some() || state.pending_remove.is_some() || state.pending_rotation.is_some() - { + if state.pending_endpoint_refresh.is_some() || state.pending_rotation.is_some() { + return; + } + // A removal whose peers were unreachable is the one pending + // marker that nothing else re-drives, and it wedges the site + // while it sits there. Push it forward here rather than giving + // up the round (rustfs/rustfs#5963). The reconcilers below + // still skip this round either way: the topology is only + // settled once the removal clears, and the next tick sees it. + if let Some(pending_remove) = state.pending_remove.clone() { + resume_pending_remove(&state, &pending_remove).await; return; } } @@ -7041,15 +7079,31 @@ async fn dequeue_site_replication_retry_event_for_generation(peer: &PeerInfo, pa } } +/// The removal's client-facing verdict. +/// +/// A fully-notified removal keeps answering with the historical success string, +/// byte for byte, so healthy runs stay wire-identical for every existing +/// client. Only the path that used to LIE — peers that could not be notified, +/// reported as unqualified success while the cluster silently diverged +/// (rustfs/rustfs#5963) — now says `Partial`, matching the vocabulary +/// `SRRotateServiceAccountHandler` already uses for the same situation. fn site_replication_remove_status(peer_errors: &[String]) -> ReplicateRemoveStatus { + if peer_errors.is_empty() { + return ReplicateRemoveStatus { + status: SITE_REPL_REMOVE_SUCCESS.to_string(), + err_detail: String::new(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }; + } + + let summaries: Vec = peer_errors.iter().map(|error| summarize_peer_error_detail(error)).collect(); ReplicateRemoveStatus { - status: SITE_REPL_REMOVE_SUCCESS.to_string(), - err_detail: if peer_errors.is_empty() { - String::new() - } else { - let summaries: Vec = peer_errors.iter().map(|error| summarize_peer_error_detail(error)).collect(); - summarize_peer_error_detail(&format!("failed to notify {} peer(s): {}", summaries.len(), summaries.join("; "))) - }, + status: SITE_REPL_REMOVE_PARTIAL.to_string(), + err_detail: summarize_peer_error_detail(&format!( + "failed to notify {} peer(s): {}", + summaries.len(), + summaries.join("; ") + )), api_version: Some(SITE_REPL_API_VERSION.to_string()), } } @@ -7226,6 +7280,137 @@ async fn clear_pending_remove(remove_id: &str) -> S3Result<()> { .await } +/// Push a half-finished removal one step forward: notify every peer that has +/// not acked yet, then finalize locally if that completed the set. Returns the +/// per-peer failures and whether the removal is now finished. +/// +/// Shared by the operator-driven `SiteReplicationRemoveHandler` and the +/// reconcile tick. The tick is what makes this self-healing: a removal whose +/// peers were unreachable used to sit in `pending_remove` forever, and that one +/// field gates every peer bucket-op (`SRPeerBucketOpsHandler` checks it first) +/// plus every reconciler — so the site stayed wedged until an operator happened +/// to re-run `replicate remove` (rustfs/rustfs#5963). +/// +/// Callers must hold the lifecycle guard: this both notifies peers and, on the +/// final step, takes the bucket-op write lock to clean up local rules. +async fn drive_pending_remove(pending_remove: &PendingRemove, local_peer: &PeerInfo) -> S3Result<(Vec, bool)> { + let mut peer_errors = Vec::new(); + let mut secret_candidates = pending_remove.secret_candidates.clone(); + if pending_remove.service_account_access_key.is_empty() { + peer_errors.push("site replication service account unavailable".to_string()); + } else if let Ok(service_account_secret_key) = + site_replicator_service_account_secret(&pending_remove.service_account_access_key).await + { + record_pending_remove_secret_candidate(&pending_remove.id, service_account_secret_key.clone()).await?; + push_unique_secret_candidate(&mut secret_candidates, service_account_secret_key); + } + + if secret_candidates.is_empty() { + peer_errors.push("site replication service account secret unavailable".to_string()); + } else { + for peer in pending_remove.original_peers.values() { + if same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) + || pending_remove.acked_deployment_ids.contains(&peer.deployment_id) + { + continue; + } + if let Err(err) = send_peer_admin_request_with_secret_candidates( + &runtime_peer_connection(peer)?, + SITE_REPLICATION_PEER_REMOVE_PATH, + &pending_remove.service_account_access_key, + &secret_candidates, + &pending_remove.req, + ) + .await + { + let err_detail = summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint)); + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + peer = %peer.endpoint, + result = "peer_remove_notification_failed", + error = %err_detail, + "admin site replication state" + ); + peer_errors.push(err_detail); + } else { + mark_pending_remove_peer_acked(&pending_remove.id, &peer.deployment_id).await?; + } + } + } + + let finalize_candidate = pending_remove_ready_to_finalize(&pending_remove.id, local_peer).await?; + let complete = if let Some(finalized_remove) = finalize_candidate { + let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.write().await; + let removed_deployment_ids = removed_deployment_ids_for_pending_remove(&finalized_remove, local_peer); + match cleanup_removed_site_replication_buckets(&removed_deployment_ids).await { + Ok(removed) => { + if removed > 0 { + info!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + removed, + result = "remove_cleanup_completed", + "admin site replication state" + ); + } + clear_pending_remove(&pending_remove.id).await?; + true + } + Err(err) => { + peer_errors.push(summarize_peer_error_detail(&format!("local remove cleanup failed: {err}"))); + false + } + } + } else { + false + }; + + Ok((peer_errors, complete)) +} + +/// The reconcile tick's half of [`drive_pending_remove`]: resume the removal +/// this site could not finish, and report the outcome. Runs under the tick's +/// lifecycle guard, which is what keeps it from racing an operator re-running +/// `replicate remove` (that handler takes the same guard). +async fn resume_pending_remove(state: &SiteReplicationState, pending_remove: &PendingRemove) { + let local_peer = current_local_runtime_peer(state); + match drive_pending_remove(pending_remove, &local_peer).await { + Ok((peer_errors, complete)) => { + if complete && peer_errors.is_empty() { + info!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "pending_remove_resumed", + "admin site replication state" + ); + } else { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "pending_remove_still_pending", + error_count = peer_errors.len(), + "admin site replication state" + ); + } + } + Err(err) => { + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "pending_remove_resume_failed", + error = ?err, + "admin site replication state" + ); + } + } +} + fn removed_deployment_ids_for_pending_remove(pending: &PendingRemove, local_peer: &PeerInfo) -> HashSet { if pending.req.remove_all || pending.req.site_names.iter().any(|name| name == &local_peer.name) { return pending @@ -9656,9 +9841,12 @@ pub struct SiteReplicationAddHandler {} /// peer identity from the add preflight metainfo in that case. fn parse_peer_join_response(body: &[u8], fallback_peer: PeerInfo) -> Result { if body.iter().all(u8::is_ascii_whitespace) { + // MinIO's empty-body success. `applied` stays `None`: the peer told us + // nothing, which must not be reported as a no-op join. return Ok(SRPeerJoinResponse { peer: fallback_peer, initial_sync_error_message: String::new(), + applied: None, }); } serde_json::from_slice(body) @@ -9761,6 +9949,19 @@ impl Operation for SiteReplicationAddHandler { if !join_response.initial_sync_error_message.is_empty() { initial_sync_errors.push(format!("{}: {}", site.endpoint, join_response.initial_sync_error_message)); } + // An explicit no-op join. The peer answered 200 but wrote nothing — + // its persisted state is already newer than the snapshot it was + // sent — so the add is only PARTIALLY configured and saying + // "configured successfully" would be a lie (rustfs/rustfs#5963). + // `None` (a MinIO peer, or one older than the field) is not a + // no-op signal and is deliberately not reported. + if join_response.applied == Some(false) { + initial_sync_errors.push(format!( + "{}: peer did not apply the join (its site replication state is newer than the snapshot it was sent); \ + the site is not configured against this peer", + site.endpoint + )); + } state = reconcile_peer_with_actual_identity(state, join_response.peer); let reconciled_peer = existing_peer_for_endpoint(&state, &site.endpoint).ok_or_else(|| { S3Error::with_message( @@ -9933,79 +10134,7 @@ impl Operation for SiteReplicationRemoveHandler { .await? }; - let mut peer_errors = Vec::new(); - let mut secret_candidates = pending_remove.secret_candidates.clone(); - if pending_remove.service_account_access_key.is_empty() { - peer_errors.push("site replication service account unavailable".to_string()); - } else if let Ok(service_account_secret_key) = - site_replicator_service_account_secret(&pending_remove.service_account_access_key).await - { - record_pending_remove_secret_candidate(&pending_remove.id, service_account_secret_key.clone()).await?; - push_unique_secret_candidate(&mut secret_candidates, service_account_secret_key); - } - - if secret_candidates.is_empty() { - peer_errors.push("site replication service account secret unavailable".to_string()); - } else { - for peer in pending_remove.original_peers.values() { - if same_identity_endpoint(&peer.endpoint, &local_peer.endpoint) - || pending_remove.acked_deployment_ids.contains(&peer.deployment_id) - { - continue; - } - if let Err(err) = send_peer_admin_request_with_secret_candidates( - &runtime_peer_connection(peer)?, - SITE_REPLICATION_PEER_REMOVE_PATH, - &pending_remove.service_account_access_key, - &secret_candidates, - &pending_remove.req, - ) - .await - { - let err_detail = summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint)); - warn!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - peer = %peer.endpoint, - result = "peer_remove_notification_failed", - error = %err_detail, - "admin site replication state" - ); - peer_errors.push(err_detail); - } else { - mark_pending_remove_peer_acked(&pending_remove.id, &peer.deployment_id).await?; - } - } - } - - let finalize_candidate = pending_remove_ready_to_finalize(&pending_remove.id, &local_peer).await?; - let complete = if let Some(finalized_remove) = finalize_candidate { - let _bucket_op_guard = SITE_REPLICATION_BUCKET_OP_LOCK.write().await; - let removed_deployment_ids = removed_deployment_ids_for_pending_remove(&finalized_remove, &local_peer); - match cleanup_removed_site_replication_buckets(&removed_deployment_ids).await { - Ok(removed) => { - if removed > 0 { - info!( - event = EVENT_ADMIN_SITE_REPLICATION_STATE, - component = LOG_COMPONENT_ADMIN, - subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, - removed, - result = "remove_cleanup_completed", - "admin site replication state" - ); - } - clear_pending_remove(&pending_remove.id).await?; - true - } - Err(err) => { - peer_errors.push(summarize_peer_error_detail(&format!("local remove cleanup failed: {err}"))); - false - } - } - } else { - false - }; + let (mut peer_errors, complete) = drive_pending_remove(&pending_remove, &local_peer).await?; if !complete && peer_errors.is_empty() { peer_errors.push("site replication remove is still pending".to_string()); } @@ -10019,6 +10148,25 @@ impl Operation for SiteReplicationRemoveHandler { } } +/// The `replicate info` projection. +/// +/// Carries the peer-facing health this endpoint used to omit entirely: a peer +/// rejecting every operation, or a removal stuck mid-flight, left `info` +/// reporting a perfectly healthy cluster while replication was dead — both were +/// only visible through `replicate status --json` (rustfs/rustfs#5963). Split +/// out so that omission is a test failure rather than an invisible regression. +fn site_replication_info_for(state: &SiteReplicationState, local_peer: &PeerInfo) -> SiteReplicationInfo { + SiteReplicationInfo { + enabled: state.enabled(), + name: local_peer.name.clone(), + sites: state.peers.values().cloned().collect(), + service_account_access_key: state.service_account_access_key.clone(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + retry_stats: retry_stats_for_state(state), + pending_operation: pending_operation_for_state(state, local_peer), + } +} + pub struct SiteReplicationInfoHandler {} #[async_trait::async_trait] @@ -10027,14 +10175,7 @@ impl Operation for SiteReplicationInfoHandler { validate_site_replication_admin_request(&req, AdminAction::SiteReplicationInfoAction).await?; let state = load_site_replication_state().await?; let local_peer = current_local_peer(&req, &state); - let info = SiteReplicationInfo { - enabled: state.enabled(), - name: local_peer.name, - sites: state.peers.values().cloned().collect(), - service_account_access_key: state.service_account_access_key, - api_version: Some(SITE_REPL_API_VERSION.to_string()), - }; - json_response(&info) + json_response(&site_replication_info_for(&state, &local_peer)) } } @@ -10253,6 +10394,28 @@ async fn apply_peer_join_service_account(join_req: SRPeerJoinReq) -> S3Result<() Ok(()) } +/// The answer to a join this site refused to apply because it had already +/// moved past the sender's snapshot. Split out so the verdict itself is +/// testable: answering `applied: Some(true)` here (or omitting the field) is +/// exactly the silent no-op that made `replicate add` report success against a +/// peer that wrote nothing (rustfs/rustfs#5963). +fn superseded_join_response(peer: PeerInfo) -> SRPeerJoinResponse { + SRPeerJoinResponse { + peer, + initial_sync_error_message: String::new(), + applied: Some(false), + } +} + +/// The answer to a join this site committed. +fn applied_join_response(peer: PeerInfo, initial_sync_error_message: String) -> SRPeerJoinResponse { + SRPeerJoinResponse { + peer, + initial_sync_error_message, + applied: Some(true), + } +} + #[async_trait::async_trait] impl Operation for SRPeerJoinHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { @@ -10275,10 +10438,14 @@ impl Operation for SRPeerJoinHandler { let (state, local_peer) = match committed { PeerJoinOutcome::Applied(state, local_peer) => (*state, local_peer), PeerJoinOutcome::Superseded(peer) => { - return json_response(&SRPeerJoinResponse { - peer, - ..Default::default() - }); + warn!( + event = EVENT_ADMIN_SITE_REPLICATION_STATE, + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_SITE_REPLICATION, + result = "join_superseded", + "admin site replication state" + ); + return json_response(&superseded_join_response(peer)); } }; // Fix 1 (receiving side): ensure the joining peer also sets up replication for any @@ -10297,10 +10464,10 @@ impl Operation for SRPeerJoinHandler { "admin site replication state" ); } - json_response(&SRPeerJoinResponse { - peer: state.peers.get(&local_peer.deployment_id).cloned().unwrap_or(local_peer), - initial_sync_error_message: backfill_errors.render(), - }) + json_response(&applied_join_response( + state.peers.get(&local_peer.deployment_id).cloned().unwrap_or(local_peer), + backfill_errors.render(), + )) } } @@ -11344,7 +11511,11 @@ impl Operation for SRRotateServiceAccountHandler { { continue; } - if let Err(err) = send_peer_admin_request_with_secret_candidates( + // A superseded join returns BEFORE `apply_iam`, so a no-op answer + // means the peer never installed the new secret. Acking it would + // finalize a rotation half the mesh cannot authenticate against + // (rustfs/rustfs#5963). + let rotation_error = match send_peer_admin_request_with_secret_candidates( &runtime_peer_connection(peer)?, SITE_REPLICATION_PEER_JOIN_PATH, &pending_rotation.access_key, @@ -11353,7 +11524,20 @@ impl Operation for SRRotateServiceAccountHandler { ) .await { - let detail = summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint)); + Err(err) => Some(summarize_peer_error_detail(&format!("{}: {err}", peer.endpoint))), + Ok(body) => match parse_peer_join_response(&body, peer.clone()) { + Ok(response) if response.applied == Some(false) => Some(summarize_peer_error_detail(&format!( + "{}: peer did not apply the rotation join (its site replication state is newer than the snapshot it \ + was sent); the new service account secret was not installed", + peer.endpoint + ))), + // Unparseable bodies keep the pre-existing behaviour: the + // transport succeeded, and MinIO peers answer with an empty + // body this helper already tolerates. + Ok(_) | Err(_) => None, + }, + }; + if let Some(detail) = rotation_error { warn!( event = EVENT_ADMIN_SITE_REPLICATION_STATE, component = LOG_COMPONENT_ADMIN, @@ -15639,9 +15823,17 @@ mod tests { site_replication_remove_status(&["peer request to https://remote.example.com failed with 403 Forbidden".to_string()]); assert!(state.peers.is_empty()); - assert_eq!(status.status, SITE_REPL_REMOVE_SUCCESS); + assert_eq!( + status.status, SITE_REPL_REMOVE_PARTIAL, + "a removal whose peer could not be notified must not report unqualified success" + ); assert!(status.err_detail.contains("failed to notify 1 peer")); assert!(status.err_detail.contains("403 Forbidden")); + + // The fully-notified path stays byte-identical for existing clients. + let clean = site_replication_remove_status(&[]); + assert_eq!(clean.status, SITE_REPL_REMOVE_SUCCESS); + assert!(clean.err_detail.is_empty()); } #[test] @@ -16982,16 +17174,22 @@ mod tests { assert_eq!(response.peer.deployment_id, "remote-deployment"); assert_eq!(response.peer.endpoint, "https://remote.example.com"); assert!(response.initial_sync_error_message.is_empty()); + assert_eq!( + response.applied, None, + "a MinIO empty-body success reports nothing; it must not read as a no-op join" + ); } let json = serde_json::to_vec(&SRPeerJoinResponse { peer: peer("actual", "https://actual.example.com"), initial_sync_error_message: "sync failed".to_string(), + applied: Some(true), }) .expect("serialize join response"); let response = parse_peer_join_response(&json, fallback.clone()).expect("parse join response body"); assert_eq!(response.peer.endpoint, "https://actual.example.com"); assert_eq!(response.initial_sync_error_message, "sync failed"); + assert_eq!(response.applied, Some(true)); assert!(parse_peer_join_response(b"not-json", fallback).is_err()); } @@ -17712,13 +17910,319 @@ mod tests { .expect("parse legacy peer join response"); assert!(response.initial_sync_error_message.is_empty()); + assert_eq!( + response.applied, None, + "a peer older than the field says nothing about whether it applied the join" + ); let value = serde_json::to_value(SRPeerJoinResponse { peer: peer("remote", "https://remote.example.com"), initial_sync_error_message: "bucket setup failed".to_string(), + applied: Some(true), }) .expect("serialize peer join response"); assert_eq!(value.get("initialSyncErrorMessage").and_then(Value::as_str), Some("bucket setup failed")); + assert_eq!(value.get("applied").and_then(Value::as_bool), Some(true)); + + // An unset verdict must not appear on the wire, so a peer that never + // learned the field keeps deserializing byte-identical payloads. + let value = serde_json::to_value(SRPeerJoinResponse { + peer: peer("remote", "https://remote.example.com"), + initial_sync_error_message: String::new(), + applied: None, + }) + .expect("serialize peer join response"); + assert!(value.get("applied").is_none(), "an unset verdict must be omitted: {value}"); + } + + /// rustfs/rustfs#5963: a removal that could not notify its peers leaves + /// `pending_remove` set, and that field alone makes `SRPeerBucketOpsHandler` + /// reject every peer operation — before it ever consults `enabled()`. A + /// later join restored the topology but left the marker, so a "successful" + /// re-add produced a cluster that reported Enabled/2-sites on both sides + /// while replication stayed dead. The join must clear it. + #[test] + fn peer_join_clears_a_stuck_pending_remove() { + let local = PeerInfo { + deployment_id: "site-b".to_string(), + ..peer("site-b", "https://site-b.example.com") + }; + let remote = PeerInfo { + deployment_id: "site-a".to_string(), + ..peer("site-a", "https://site-a.example.com") + }; + let mut state = SiteReplicationState { + peers: BTreeMap::from([(local.deployment_id.clone(), local.clone())]), + pending_remove: Some(PendingRemove { + id: "stuck-remove".to_string(), + req: SRRemoveReq { + remove_all: true, + ..Default::default() + }, + service_account_access_key: SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(), + secret_candidates: Vec::new(), + original_peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]), + acked_deployment_ids: BTreeSet::new(), + updated_at: Some(OffsetDateTime::now_utc()), + }), + ..Default::default() + }; + + apply_peer_join( + &mut state, + &local, + SRPeerJoinReq { + svc_acct_access_key: SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(), + svc_acct_secret_key: "svc-secret".to_string(), + svc_acct_parent: "root".to_string(), + peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote), + ]), + updated_at: Some(OffsetDateTime::now_utc()), + }, + false, + ); + + assert!( + state.pending_remove.is_none(), + "an accepted join supersedes the half-finished removal it lands on" + ); + assert!(state.enabled(), "the join restores the two-site topology"); + // The guard `SRPeerBucketOpsHandler` evaluates, asserted directly: with + // the marker cleared and the topology back, peer bucket-ops are + // admitted again. + assert!( + state.pending_remove.is_none() && state.enabled(), + "the bucket-ops admission predicate must now pass" + ); + } + + /// The fence marks are lifecycle-independent and must survive the clearing + /// above — wiping them would reopen the rollback window the fence closes. + #[test] + fn peer_join_clearing_pending_remove_keeps_edit_generation_marks() { + let local = PeerInfo { + deployment_id: "site-b".to_string(), + ..peer("site-b", "https://site-b.example.com") + }; + let remote = PeerInfo { + deployment_id: "site-a".to_string(), + ..peer("site-a", "https://site-a.example.com") + }; + let mut state = SiteReplicationState { + peers: BTreeMap::from([(local.deployment_id.clone(), local.clone())]), + applied_edit_generations: BTreeMap::from([(remote.deployment_id.clone(), 7)]), + pending_remove: Some(PendingRemove { + id: "stuck-remove".to_string(), + req: SRRemoveReq { + remove_all: true, + ..Default::default() + }, + service_account_access_key: SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(), + secret_candidates: Vec::new(), + original_peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]), + acked_deployment_ids: BTreeSet::new(), + updated_at: Some(OffsetDateTime::now_utc()), + }), + ..Default::default() + }; + + apply_peer_join( + &mut state, + &local, + SRPeerJoinReq { + svc_acct_access_key: SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(), + svc_acct_secret_key: "svc-secret".to_string(), + svc_acct_parent: "root".to_string(), + peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]), + updated_at: Some(OffsetDateTime::now_utc()), + }, + false, + ); + + assert!(state.pending_remove.is_none()); + assert_eq!( + state.applied_edit_generations.get(&remote.deployment_id), + Some(&7), + "clearing the lifecycle marker must not touch the ordering fence" + ); + } + + /// rustfs/rustfs#5963: the two join verdicts must be distinguishable on the + /// wire. `Some(true)`/`Some(false)` is what lets the initiator tell a real + /// configuration from a 200 that wrote nothing; flipping either one back to + /// an unset verdict re-hides the no-op. + #[test] + fn join_verdicts_are_distinguishable_on_the_wire() { + let remote = peer("remote", "https://remote.example.com"); + + let superseded = superseded_join_response(remote.clone()); + assert_eq!( + superseded.applied, + Some(false), + "a join this site refused to apply must say so explicitly" + ); + assert!(superseded.initial_sync_error_message.is_empty()); + + let applied = applied_join_response(remote, "bucket setup failed".to_string()); + assert_eq!(applied.applied, Some(true)); + assert_eq!(applied.initial_sync_error_message, "bucket setup failed"); + + // Round-tripping through the wire keeps the two apart — the initiator + // only ever sees the serialized form. + let decoded: SRPeerJoinResponse = + serde_json::from_slice(&serde_json::to_vec(&superseded_join_response(peer("r", "https://r.example.com"))).unwrap()) + .expect("round-trip superseded verdict"); + assert_eq!(decoded.applied, Some(false)); + } + + /// rustfs/rustfs#5963: a stuck removal must be visible on the endpoint + /// operators actually run. `replicate info` used to report only + /// `enabled: false`, which reads as "never configured" rather than "a + /// removal is wedged here and this site rejects every peer operation". + #[test] + fn site_replication_info_reports_a_wedged_removal() { + let local = PeerInfo { + deployment_id: "site-b".to_string(), + ..peer("site-b", "https://site-b.example.com") + }; + let remote = PeerInfo { + deployment_id: "site-a".to_string(), + ..peer("site-a", "https://site-a.example.com") + }; + let state = SiteReplicationState { + name: "site-b".to_string(), + peers: BTreeMap::from([(local.deployment_id.clone(), local.clone())]), + pending_remove: Some(PendingRemove { + id: "stuck-remove".to_string(), + req: SRRemoveReq { + remove_all: true, + ..Default::default() + }, + service_account_access_key: SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(), + secret_candidates: Vec::new(), + original_peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]), + acked_deployment_ids: BTreeSet::new(), + updated_at: Some(OffsetDateTime::now_utc()), + }), + ..Default::default() + }; + + let info = site_replication_info_for(&state, &local); + assert!(!info.enabled, "the peer set is already torn down"); + let pending = info + .pending_operation + .as_ref() + .expect("a wedged removal must surface as a pending operation"); + assert_eq!(pending.operation, "remove"); + assert!( + pending.pending_peers.contains(&remote.deployment_id), + "the peer that was never notified must be named: {pending:?}" + ); + } + + /// The source side of the same failure: peer operations are being rejected, + /// the topology still looks like a healthy two-site cluster, and `info` has + /// to say the deliveries are failing. + #[test] + fn site_replication_info_reports_failing_peer_deliveries() { + let local = PeerInfo { + deployment_id: "site-a".to_string(), + ..peer("site-a", "https://site-a.example.com") + }; + let remote = PeerInfo { + deployment_id: "site-b".to_string(), + ..peer("site-b", "https://site-b.example.com") + }; + let state = SiteReplicationState { + name: "site-a".to_string(), + peers: BTreeMap::from([ + (local.deployment_id.clone(), local.clone()), + (remote.deployment_id.clone(), remote.clone()), + ]), + retry_queue: vec![SiteReplicationRetryEvent { + id: "evt".to_string(), + peer_deployment_id: remote.deployment_id.clone(), + peer_endpoint: remote.endpoint, + path: "/rustfs/admin/v3/site-replication/peer/bucket-ops?bucket=demo&operation=make-with-versioning".to_string(), + retry_count: 9, + failed: true, + last_error: "site replication is not enabled".to_string(), + updated_at: Some(OffsetDateTime::now_utc()), + edit_generation: None, + }], + ..Default::default() + }; + + let info = site_replication_info_for(&state, &local); + assert!(info.enabled, "the topology still reports two sites — that was the trap"); + let stats = info + .retry_stats + .as_ref() + .expect("a peer rejecting every delivery must be visible in `info`"); + assert_eq!(stats.failed, 1); + assert_eq!(stats.last_error, "site replication is not enabled"); + + // A healthy site must stay wire-identical to before the field existed. + let healthy = SiteReplicationState { + retry_queue: Vec::new(), + ..state + }; + let info = site_replication_info_for(&healthy, &local); + assert!(info.retry_stats.is_none()); + assert!(info.pending_operation.is_none()); + } + + /// rustfs/rustfs#5963: `replicate info` reported a healthy cluster while + /// every peer operation was failing. The health it used to omit now rides + /// along, and a healthy site still serializes without the new fields. + #[test] + fn site_replication_info_health_fields_are_absent_when_healthy() { + let healthy = SiteReplicationInfo { + enabled: true, + name: "site-a".to_string(), + sites: vec![peer("site-a", "https://site-a.example.com")], + service_account_access_key: SITE_REPLICATOR_SERVICE_ACCOUNT.to_string(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + retry_stats: None, + pending_operation: None, + }; + let value = serde_json::to_value(&healthy).expect("serialize info"); + assert!(value.get("retryStats").is_none(), "a healthy site must not grow fields: {value}"); + assert!(value.get("pendingOperation").is_none(), "a healthy site must not grow fields: {value}"); + + let degraded = SiteReplicationInfo { + retry_stats: Some(SRRetryStats { + pending: 1, + failed: 4, + last_error: "site replication is not enabled".to_string(), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + ..healthy + }; + let value = serde_json::to_value(°raded).expect("serialize info"); + assert_eq!( + value.pointer("/retryStats/failed").and_then(Value::as_u64), + Some(4), + "a source site whose peer rejects everything must say so in `info`" + ); + assert_eq!( + value.pointer("/retryStats/lastError").and_then(Value::as_str), + Some("site replication is not enabled") + ); } // Fix 5: remove --all must purge local state unconditionally even when peer errors occur @@ -17766,12 +18270,14 @@ mod tests { assert!(state.peers.is_empty(), "peers must be cleared on remove --all"); assert!(state.resync_status.is_empty(), "resync_status must be cleared on remove --all"); - // Even if peers returned 403 (desynced account), status still reports success + // The local side is torn down either way, but a peer that returned 403 + // (desynced account) leaves the cluster diverged — the response must + // say so instead of reporting unqualified success (rustfs/rustfs#5963). let status = site_replication_remove_status(&["https://remote.example.com: peer/remove returned 403 Forbidden".to_string()]); assert_eq!( - status.status, SITE_REPL_REMOVE_SUCCESS, - "local remove reports success even when peer notifications fail" + status.status, SITE_REPL_REMOVE_PARTIAL, + "local remove must report a partial result when peer notifications fail" ); assert!( status.err_detail.contains("403 Forbidden"), diff --git a/scripts/test/site_replication_smoke.py b/scripts/test/site_replication_smoke.py index be6b94563..259a98e79 100755 --- a/scripts/test/site_replication_smoke.py +++ b/scripts/test/site_replication_smoke.py @@ -27,6 +27,7 @@ Usage: ./scripts/test/site_replication_smoke.py # up: start both + pair ./scripts/test/site_replication_smoke.py status # process + pair status ./scripts/test/site_replication_smoke.py smoke # bidirectional object check + ./scripts/test/site_replication_smoke.py diverge # rustfs/rustfs#5963 regression ./scripts/test/site_replication_smoke.py logs # tail both server logs ./scripts/test/site_replication_smoke.py down # stop both processes ./scripts/test/site_replication_smoke.py clean # down + wipe site data @@ -337,11 +338,12 @@ def ensure_pair(site_a: Site, site_b: Site) -> None: print(f"[ok] site replication configured: {result.get('status', '')}") -def remove_pair(site: Site) -> None: +def remove_pair(site: Site) -> dict: status, body = admin(site, "PUT", "site-replication/remove", payload={"all": True}) if status != 200: raise SystemExit(f"[fail] site-replication remove: HTTP {status} {body.decode(errors='replace')}") print(f"[ok] site replication removed: {body.decode(errors='replace')}") + return json.loads(body) # --------------------------------------------------------------------------- @@ -397,6 +399,112 @@ def smoke(site_a: Site, site_b: Site, timeout: float) -> None: print(f"[ok] bidirectional replication verified via bucket {bucket}") +# --------------------------------------------------------------------------- +# Divergence regression (rustfs/rustfs#5963) +# --------------------------------------------------------------------------- + + +def wait_for(description: str, probe, timeout: float): + """Poll `probe` until it returns a truthy value; return it. SystemExit on timeout.""" + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + try: + result = probe() + except (urllib.error.URLError, OSError, TimeoutError, SystemExit) as err: + last = err + result = None + if result: + return result + time.sleep(1.0) + raise SystemExit(f"[fail] {description} within {timeout:.0f}s (last: {last})") + + +def diverge(site_a: Site, site_b: Site, binary: Path, console: bool, timeout: float) -> None: + """Reproduce rustfs/rustfs#5963 end to end and assert the cluster recovers. + + Before the fix, step 7 left site-b rejecting every peer bucket-op forever: + `pending_remove` gates `SRPeerBucketOpsHandler` ahead of `enabled()`, and a + join never cleared it — so a *successful* re-add produced a cluster that + reported Enabled/2-sites on both sides while replication stayed dead. + """ + ensure_pair(site_a, site_b) + + # 1. Take site-a down so it cannot be told about the removal. + print("[..] step 1: stopping site-a so it cannot be notified") + stop_site(site_a) + + # 2. Remove from site-b. The local teardown commits either way, but the + # response must NOT claim unqualified success (P2-5). + print("[..] step 2: removing site replication from site-b while site-a is down") + status = remove_pair(site_b) + if not status.get("errorDetail"): + raise SystemExit(f"[fail] remove hid the unreachable peer; expected errorDetail: {json.dumps(status)}") + if status.get("status") == "Requested site(s) were removed from cluster replication successfully.": + raise SystemExit(f"[fail] remove reported unqualified success despite an unnotified peer: {json.dumps(status)}") + print(f"[ok] remove reported a partial result: status={status.get('status')!r}") + + # 3. The wedged removal must be visible on `info`, not just in status --json (P1-4). + info_b = pair_state(site_b) + pending = info_b.get("pendingOperation") + if not pending or pending.get("operation") != "remove": + raise SystemExit(f"[fail] site-b hides the wedged removal in `info`: {json.dumps(info_b, indent=2)}") + print(f"[ok] site-b reports the wedged removal: pendingPeers={pending.get('pendingPeers')}") + + # 4. Bring site-a back. It still believes in a healthy 2-site cluster. + print("[..] step 4: restarting site-a") + start_site(site_a, binary, console) + wait_ready([site_a], timeout) + info_a = pair_state(site_a) + if not info_a.get("enabled"): + raise SystemExit(f"[fail] site-a lost its own state: {json.dumps(info_a, indent=2)}") + print("[ok] site-a still reports an enabled cluster (the divergence)") + + # 5. A bucket created on site-a cannot reach site-b. The failure must become + # visible on the SOURCE, which used to report a perfectly healthy cluster. + bucket = f"sr-diverge-{uuid.uuid4().hex[:8]}" + sig_status, body = signed_request(site_a, "PUT", f"/{bucket}") + if sig_status != 200: + raise SystemExit(f"[fail] create bucket {bucket} on site-a: HTTP {sig_status} {body.decode(errors='replace')}") + print(f"[ok] created {bucket} on site-a (locally succeeds, peer push is rejected)") + + stats = wait_for( + "site-a did not surface the failing peer deliveries in `info`", + lambda: pair_state(site_a).get("retryStats"), + timeout, + ) + print(f"[ok] site-a reports failing deliveries: pending={stats.get('pending')} failed={stats.get('failed')} " + f"lastError={stats.get('lastError')!r}") + + # 6. Re-add. This is the operator's natural recovery move. + print("[..] step 6: re-adding the pair from site-a") + peers = [ + {"name": s.name, "endpoints": s.endpoint, "accessKey": s.access_key, "secretKey": s.secret_key} + for s in (site_a, site_b) + ] + add_status, add_body = admin(site_a, "PUT", "site-replication/add", "replicateILMExpiry=false", peers) + if add_status != 200: + raise SystemExit(f"[fail] re-add: HTTP {add_status} {add_body.decode(errors='replace')}") + print(f"[ok] re-add accepted: {add_body.decode(errors='replace')}") + + # 7. The join must have cleared site-b's pending_remove (P0-1). Without the + # fix this assertion is exactly what fails while everything above passes. + info_b = pair_state(site_b) + if info_b.get("pendingOperation"): + raise SystemExit( + "[fail] the join did not clear site-b's wedged removal; peer bucket-ops stay rejected forever: " + f"{json.dumps(info_b, indent=2)}" + ) + if not info_b.get("enabled"): + raise SystemExit(f"[fail] site-b did not rejoin: {json.dumps(info_b, indent=2)}") + print("[ok] site-b cleared the wedged removal and rejoined") + + # 8. The symptom the issue actually reported: replication works again. + print("[..] step 8: verifying replication actually flows again") + smoke(site_a, site_b, timeout) + print("[ok] rustfs/rustfs#5963 regression passed") + + # --------------------------------------------------------------------------- # Commands # --------------------------------------------------------------------------- @@ -464,7 +572,7 @@ def main() -> None: "command", nargs="?", default="up", - choices=["up", "down", "restart", "status", "logs", "smoke", "info", "remove", "clean"], + choices=["up", "down", "restart", "status", "logs", "smoke", "diverge", "info", "remove", "clean"], ) parser.add_argument("--port-a", type=int, default=9000, help="site A S3 port (default: %(default)s)") parser.add_argument("--port-b", type=int, default=9020, help="site B S3 port (default: %(default)s)") @@ -495,6 +603,8 @@ def main() -> None: cmd_logs(sites, args.lines) elif args.command == "smoke": smoke(site_a, site_b, args.timeout) + elif args.command == "diverge": + diverge(site_a, site_b, args.binary, args.console, args.timeout) elif args.command == "info": print(json.dumps(pair_state(site_a), indent=2, ensure_ascii=False)) elif args.command == "remove": From 1741f79d7dbbffa2ff2cf612826307329ce16d47 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 19 Aug 2026 12:20:54 +0800 Subject: [PATCH 08/27] test(audit,heal): assert three more smoke tests (#6248) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `audit_runtime_facade_stops_empty_replay_workers` called the stop path and checked nothing, the same shape as the notify facade test in the previous commit. It now asserts the worker manager is empty afterwards and that a second call — which shutdown paths make — stays harmless. The two heal timestamp tests bound their fields to `_`. Both timestamps come from `SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default()`, so a pre-epoch clock yields 0; binding to `_` could not tell that apart from a real reading, which is precisely what the tests said they were guarding. They now require the value to be past 2020-01-01, and `last_update` not to predate `start_time`. `test_config_parsing_with_multiple_instances` is left as it was — see the issue comment. Asserting on it turned up something bigger than a missing assertion. Refs backlog#1836 --- crates/audit/tests/pipeline_layer_test.rs | 11 ++++++-- crates/heal/tests/heal_bug_fixes_test.rs | 33 +++++++++++++++-------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/crates/audit/tests/pipeline_layer_test.rs b/crates/audit/tests/pipeline_layer_test.rs index 3b28b5863..5d5608db2 100644 --- a/crates/audit/tests/pipeline_layer_test.rs +++ b/crates/audit/tests/pipeline_layer_test.rs @@ -236,12 +236,19 @@ async fn audit_pipeline_reports_empty_runtime_snapshots() { } #[tokio::test] -async fn audit_runtime_facade_stops_empty_replay_workers() { +async fn stopping_audit_replay_workers_is_a_no_op_when_there_are_none() { let registry = Arc::new(Mutex::new(AuditRegistry::new())); let replay_workers = Arc::new(RwLock::new(rustfs_targets::ReplayWorkerManager::new())); - let facade = AuditRuntimeFacade::new(registry, replay_workers); + let facade = AuditRuntimeFacade::new(registry, Arc::clone(&replay_workers)); facade.stop_replay_workers().await; + + // The stop path takes the manager's workers and hands them to the adapter, + // so an empty facade must leave it empty rather than wedge it, and a second + // call — which shutdown paths make — must stay harmless (rustfs/backlog#1836). + assert!(replay_workers.read().await.is_empty()); + facade.stop_replay_workers().await; + assert!(replay_workers.read().await.is_empty()); } #[tokio::test] diff --git a/crates/heal/tests/heal_bug_fixes_test.rs b/crates/heal/tests/heal_bug_fixes_test.rs index ba59142fd..dc5addef2 100644 --- a/crates/heal/tests/heal_bug_fixes_test.rs +++ b/crates/heal/tests/heal_bug_fixes_test.rs @@ -117,13 +117,16 @@ fn test_format_set_disk_id_from_i32_valid() { assert_eq!(result.unwrap(), "pool_0_set_1"); } +/// A wall-clock lower bound for "the timestamp was actually read from the +/// clock": 2020-01-01. `unwrap_or_default()` on a pre-epoch clock yields 0, and +/// the old versions of these tests bound the fields to `_` and so could not tell +/// that apart from a real reading (rustfs/backlog#1836). +const SANE_EPOCH_SECS: u64 = 1_577_836_800; + #[test] fn test_resume_state_timestamp_handling() { use rustfs_heal::heal::resume::ResumeState; - // Test that ResumeState creation doesn't panic even if system time is before epoch - // This is a theoretical test - in practice, system time should never be before epoch - // But we want to ensure unwrap_or_default handles edge cases let state = ResumeState::new( "test-task".to_string(), "test-type".to_string(), @@ -131,22 +134,30 @@ fn test_resume_state_timestamp_handling() { vec!["bucket1".to_string()], ); - // Verify fields are initialized (u64 is always >= 0) - // The important thing is that unwrap_or_default prevents panic - let _ = state.start_time; - let _ = state.last_update; + assert!( + state.start_time > SANE_EPOCH_SECS, + "start_time fell back to the default instead of reading the clock: {}", + state.start_time + ); + assert!( + state.last_update >= state.start_time, + "last_update {} must not predate start_time {}", + state.last_update, + state.start_time + ); } #[test] fn test_resume_checkpoint_timestamp_handling() { use rustfs_heal::heal::resume::ResumeCheckpoint; - // Test that ResumeCheckpoint creation doesn't panic let checkpoint = ResumeCheckpoint::new("test-task".to_string()); - // Verify field is initialized (u64 is always >= 0) - // The important thing is that unwrap_or_default prevents panic - let _ = checkpoint.checkpoint_time; + assert!( + checkpoint.checkpoint_time > SANE_EPOCH_SECS, + "checkpoint_time fell back to the default instead of reading the clock: {}", + checkpoint.checkpoint_time + ); } #[test] From d030719dbc8f47e55c2f9a17e8634435798b98f4 Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 19 Aug 2026 13:01:39 +0800 Subject: [PATCH 09/27] docs(scanner): record heal/scanner MinIO parity decisions (backlog#1878 HS-14/16/18) (#6250) * refactor(scanner): drop the always-None single-disk default cycle hook single_disk_default_cycle_secs returned None for every maintenance feature combination, so the single-disk startup path already resolved its default cycle from the speed preset (60s at 'default'). Remove the never-wired hook and its pin tests, keep the explicit reset, and record the decision: no special single-disk cycle override without measured cold-start ILM latency evidence; clean-idle backoff already stretches idle cadence (backlog#1878 HS-16). Co-Authored-By: heihutu * docs(operations): add heal/scanner MinIO parity decision notes Document the HS-14/16/18 decision batch from backlog#1878: the scanner idle throttling semantics matrix (RUSTFS_SCANNER_IDLE_MODE x speed preset x foreground read backoff) side by side with MinIO's current static idle_speed switch as verified against upstream master, the migration warnings for env names and value vocabularies, the bitrot cycle default divergence (30d vs off), the stale-multipart / tmp / trash three-stage cleanup comparison with the crash-residue window grading, and the single-disk default cycle decision. Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- crates/scanner/src/scanner.rs | 50 +------- .../heal-scanner-parity-notes-zh.md | 109 ++++++++++++++++++ 2 files changed, 114 insertions(+), 45 deletions(-) create mode 100644 docs/operations/heal-scanner-parity-notes-zh.md diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index fe743a1c6..eb2dc9d4b 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -1425,10 +1425,6 @@ fn maintenance_inspection_decision(generation: u64, current_generation: u64, att } } -fn single_disk_default_cycle_secs(_features: ScannerMaintenanceFeatures) -> Option { - None -} - fn single_disk_default_speed() -> ScannerSpeed { ScannerSpeed::Default } @@ -1592,9 +1588,12 @@ async fn configure_scanner_defaults( scanner_maintenance_generation(), ) }); - let default_cycle_secs = single_disk_default_cycle_secs(features); + // Single-disk keeps the speed-preset-derived default cycle (60s at the + // `default` preset) instead of a special shorter cycle: no measured + // cold-start ILM latency basis for an override, and clean-idle backoff + // already stretches idle cadence. Decision record: backlog#1878 (HS-16). set_scanner_default_speed(single_disk_default_speed()); - set_scanner_default_cycle_secs(default_cycle_secs); + set_scanner_default_cycle_secs(None); info!( target: "rustfs::scanner", event = EVENT_SCANNER_RUNTIME_CONFIG, @@ -1603,7 +1602,6 @@ async fn configure_scanner_defaults( env_speed = ENV_SCANNER_SPEED, env_cycle = ENV_SCANNER_CYCLE, env_start_delay = ENV_SCANNER_START_DELAY_SECS, - ?default_cycle_secs, lifecycle_active = features.lifecycle, replication_active = features.replication, feature_inspection_failed = features.inspection_failed, @@ -6950,11 +6948,6 @@ mod tests { }); } - #[test] - fn test_single_disk_default_cycle_uses_speed_based_interval_without_maintenance_features() { - assert_eq!(single_disk_default_cycle_secs(ScannerMaintenanceFeatures::default()), None); - } - #[test] fn test_single_disk_default_speed_uses_regular_scanner_default() { assert_eq!(single_disk_default_speed(), ScannerSpeed::Default); @@ -7415,39 +7408,6 @@ mod tests { }); } - #[test] - fn test_single_disk_default_cycle_preserves_regular_cycle_for_lifecycle() { - assert_eq!( - single_disk_default_cycle_secs(ScannerMaintenanceFeatures { - lifecycle: true, - ..Default::default() - }), - None - ); - } - - #[test] - fn test_single_disk_default_cycle_preserves_regular_cycle_for_replication() { - assert_eq!( - single_disk_default_cycle_secs(ScannerMaintenanceFeatures { - replication: true, - ..Default::default() - }), - None - ); - } - - #[test] - fn test_single_disk_default_cycle_preserves_regular_cycle_on_inspection_failure() { - assert_eq!( - single_disk_default_cycle_secs(ScannerMaintenanceFeatures { - inspection_failed: true, - ..Default::default() - }), - None - ); - } - #[test] #[serial] fn test_cycle_interval_keeps_default_cycle_with_explicit_speed() { diff --git a/docs/operations/heal-scanner-parity-notes-zh.md b/docs/operations/heal-scanner-parity-notes-zh.md new file mode 100644 index 000000000..027926747 --- /dev/null +++ b/docs/operations/heal-scanner-parity-notes-zh.md @@ -0,0 +1,109 @@ +# Heal/Scanner 配置与语义对照（MinIO parity 决策记录） + +对应 backlog rustfs/backlog#1878（父 #1862，批 HS-14/HS-16/HS-18）。本页沉淀三项"决策 + 文档化"结论：scanner idle 节流语义对照与迁移警告（HS-14）、单机默认扫描周期决策（HS-16）、stale multipart 与 tmp/.trash 清理三段核对（HS-18），并顺带收录 bitrot_cycle 与 alert_excess_folders 两项已确认的默认值差异。所有 MinIO 侧结论均于 2026-08 按 minio/minio master 逐源码核对（引用文件为上游路径），不转述二手资料。 + +运行时旋钮的完整清单、状态端点与调参流程见 [Scanner Runtime Controls](scanner-runtime-controls.md)；excess 告警阈值差异见 [Scanner Excess Alerts](scanner-excess-alerts_zh.md)；heal 并发模型对照见 [Heal 并发安全说明](heal-concurrency-safety-notes-zh.md)。 + +## 1. HS-14：scanner idle 节流语义对照 + +### RustFS 当前语义（三因子） + +RustFS 的 scanner 步进节流由三个因子共同决定（crates/scanner/src/sleeper.rs）： + +1. **总闸 `scanner.idle_mode` / `RUSTFS_SCANNER_IDLE_MODE`（默认 `true`）**：`false` 时所有节流 sleep 全部跳过，scanner 全速推进；`true` 时按下面两因子计算 sleep。 +2. **速度档**（`scanner.speed` / `RUSTFS_SCANNER_SPEED`，默认 `default`）：档位表与 MinIO 完全一致（见下表）。目录级 sleep = `1ms × factor`（上限 `max_wait`）；对象级 sleep = `本对象处理耗时 × factor`，下限 1ms、上限 `max_wait`。 +3. **前台读退避下限**：`current_foreground_read_activity()` 取并发 GetObject 请求数（rustfs/src/storage/concurrency/request_guard.rs 的 `GetObjectGuard`）与流式读计数（`ForegroundReadGuard`）的较大值，换算为 `10ms × 活跃读数`、封顶 250ms 的下限；该下限对目录级与对象级 sleep 都生效（`.max(foreground_sleep)`），且**可以超过速度档的 `max_wait`**（自身封顶 250ms）。速度档为 `fastest`（factor=0）时预设 sleep 为 0，但只要 `idle_mode=true`，前台读下限仍然生效。 + +| 速度档 | sleep factor | 单次 sleep 上限 | 周期间隔 | +|---|---:|---:|---:| +| `fastest` | 0 | 0 | 1s | +| `fast` | 1× | 100ms | 1m | +| `default` | 2× | 1s | 1m | +| `slow` | 10× | 15s | 1m | +| `slowest` | 100× | 15s | 30m | + +实际行为矩阵（RustFS）： + +| `idle_mode` | 速度档 | 前台并发读 = 0 | 前台并发读 > 0 | +|---|---|---|---| +| `false` | 任意 | 完全不休眠，全速 | 完全不休眠，全速（前台退避也被总闸关闭） | +| `true` | `fastest` | 预设 sleep = 0，等效全速 | 每步 sleep = 前台读下限（10ms×读数，封顶 250ms） | +| `true` | 其余档 | 每步 sleep = 预设值（1ms~15s 封顶） | 每步 sleep = max(预设值, 前台读下限) | + +周期间隔的解析优先级为 env `RUSTFS_SCANNER_CYCLE` > 持久化 `scanner.cycle` > `scanner.start_delay` > 启动期默认覆盖（当前恒无）> 速度档派生（crates/scanner/src/runtime_config.rs）。另有 `scanner.yield_every_n_objects`（默认 128）的协作式让出，与节流 sleep 相互独立。 + +### MinIO 当前语义（master 逐源码核对） + +MinIO 的对应开关是 `scanner:idle_speed` / `MINIO_SCANNER_IDLE_SPEED`（internal/config/scanner/scanner.go）：取值为空串或 `on`（默认）时 `IdleMode=0`，取值 `off` 时 `IdleMode=1`。启动/配置加载时一次性写入 `scannerIdleMode`（cmd/config-current.go），扫描侧闭包 `weSleep = scannerIdleMode.Load() == 0`（cmd/xl-storage-disk-id-check.go）：**`on`（默认）= 目录级与对象级节流 sleep 始终插入（按速度档 factor，minSleep 100µs）；`off` = 两条节流路径完全不 sleep，全速扫描**。当前上游没有任何按 S3 请求/磁盘活动动态调整节流的逻辑——这是静态开关。 + +命名具有误导性，是历史残留：2024-01 之前 `weSleep` 由磁盘活动驱动（"Entire queue is full, so we sleep"，即有并发 S3/heal 活动才 sleep），minio/minio#18734（commit 7705605b）把该活动门替换为上述静态配置（初版取值 `throttled`/`full`，后改为 `on`/`off`），上游残留注释 "default is throttled when idle"、"Sleep always or based on incoming S3 requests" 均是替换前的语义描述，与现行代码不符。 + +### 对照与迁移警告 + +| 维度 | RustFS | MinIO（master） | +|---|---|---| +| 开关名 | `scanner.idle_mode` / `RUSTFS_SCANNER_IDLE_MODE` | `scanner:idle_speed` / `MINIO_SCANNER_IDLE_SPEED` | +| 取值 | 布尔 `true`/`false` | `on`/`off` | +| 默认 | `true`（节流开启） | `on`（节流开启） | +| 开 = | 节流总闸开：速度档 sleep + 前台读下限 | 节流总闸开：速度档 sleep | +| 关 = | 完全不休眠（含前台读下限一并失效） | 完全不休眠 | +| 活动耦合 | 有：前台并发读抬高 sleep 下限（10ms×读数，封顶 250ms） | 无（2024-01 起为静态开关） | +| 速度档表 | 两边完全一致（上表） | 同左 | + +迁移警告： + +- **环境变量名不可照搬**：RustFS 只读取 `RUSTFS_*` 前缀，不解析 `MINIO_SCANNER_*` 任何别名（crates/scanner、crates/utils 的 env 读取无别名链，测试还专门断言 `MINIO_SCANNER_SPEED`/`MINIO_SCANNER_CYCLE` 不泄漏生效）。照搬 `MINIO_SCANNER_IDLE_SPEED=off` 到 RustFS 会静默无效，必须改写成 `RUSTFS_SCANNER_IDLE_MODE=false`。 +- **取值词表不同**：`on/off` vs `true/false`，不能原样复制。 +- **方向澄清（修正父 issue 的预设）**：按当前上游源码，MinIO `idle_speed` 与 RustFS `idle_mode` 在"开=节流、关=全速"方向上是一致的，并非反向；父 issue 中"MinIO on=集群空闲才节流、off=始终按 delay 节流"的矩阵描述的是 2024-01 之前的活动耦合行为与反向解读，与 master 不符。真正需要写进迁移手册的差异是：MinIO 的 `idle_speed` 名称暗示"空闲时才慢"但实际是静态总闸；RustFS 的 `idle_mode=true` 在总闸之上还叠加了 MinIO 没有的前台读保护下限。 +- **`false` 是大锤**：RustFS `idle_mode=false` 会连前台读退避一起关闭，scanner 与前台读完全抢盘；仅在 benchmark 或可独占 IO 的窗口使用。 + +**决策（HS-14）：保持现状。** RustFS 语义更直观（`idle_mode` = 节流总闸，`true` 即自适应限速），且比 MinIO 多一层前台读保护；不新增 `RUSTFS_SCANNER_IDLE_SPEED` 兼容别名（无社区强诉求不做，避免双入口漂移）。本节即对照表与迁移警告的正式落点。 + +## 2. bitrot_cycle 默认差异 + +| 项 | RustFS | MinIO | +|---|---|---| +| 键 | `heal.bitrot_cycle` / `RUSTFS_SCANNER_BITROT_CYCLE_SECS`（scanner.bitrot_cycle 为兼容旧键） | `heal:bitrotscan` / `MINIO_HEAL_BITROTSCAN` | +| 默认 | 30 天（crates/config/src/constants/heal.rs 的 `DEFAULT_HEAL_BITROT_CYCLE_SECS`）：按墙钟周期把扫描切深扫（deep bitrot） | `off`（internal/config/heal/heal.go 默认 `EnableOff`）：不做周期性深扫，仅普通扫描 + 管理端手动深扫 | +| 对齐方式 | 迁移 MinIO 行为：`heal.bitrot_cycle=off` 或 `RUSTFS_SCANNER_BITROT_CYCLE_SECS=disabled` | 反向：`heal:bitrotscan=<秒>` | + +RustFS 的 30 天默认是刻意的耐用性默认（周期性全量 bitrot 校验），代价是每 30 天一轮深扫 IO；单机场景另有清洁空闲退避封顶约 42 分钟的墙钟保护（见 scanner-runtime-controls.md）。这是行为差异而非缺陷，文档化即可。 + +## 3. alert_excess_folders 默认差异 + +RustFS 默认 65538（容纳 Proxmox Backup Server 每目录 65536 chunk 的布局），MinIO 默认 50000。差异原因、另两个 excess 阈值（versions=100 相同、version_size TiB vs TB）、事件名映射与冷却语义已完整记录在 [Scanner Excess Alerts](scanner-excess-alerts_zh.md)，此处不重复。 + +## 4. HS-18：stale multipart 与 tmp/.trash 清理三段核对 + +MinIO 把"清理已删除数据"拆成三段：stale upload 先 rename 进 `.minio.sys/tmp/.trash/` 隔离（rename 快、原子）；trash 由独立例程排空；tmp 下非 trash 的旧目录单独回收。逐段核对 RustFS： + +| 段 | MinIO | RustFS | 判定 | +|---|---|---|---| +| stale multipart → 隔离 | `cleanupStaleUploadsOnDisk`（cmd/erasure-multipart.go）逐盘列出 multipart 目录，按 uploadID 目录名里的 UnixNano 判龄，超过 `stale_uploads_expiry`（默认 24h）即 `renameAll` 进 `.minio.sys/tmp/.trash/`，空 sha 目录、tmp 旧目录同法 | `cleanup_stale_multipart_uploads_in_set`（crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs）发现候选后取 ns 写锁 + 重查（`lock_stale_multipart_cleanup`），`delete_all_with_quorum` 扇出逐盘递归删除，而 LocalDisk 的递归删除内部就是 `move_to_trash`（crates/ecstore/src/disk/local.rs）把目录 rename 进 `.rustfs.sys/tmp/.trash/` | 行为等价（都是先隔离后清理）；RustFS 额外有写锁 + quorum 重查 + 锁丢失 fence（crates/ecstore/src/set_disk/ops/multipart.rs 的 `StaleMultipartCleanupGuard`），防并发 CompleteMultipartUpload 竞争，安全性强于 MinIO 的无锁 rename | +| trash 排空 | 每 `delete_cleanup_interval`（默认 5m，internal/config/api/api.go）逐盘删 `.trash` 内条目，逐条以 `deleteCleanupSleeper`（factor 5 / 25ms，cmd/globals.go）节流 | 每盘独立 `cleanup_deleted_objects_loop`，`DELETED_OBJECTS_CLEANUP_INTERVAL` = 5m（crates/ecstore/src/disk/local.rs），先排空 `.trash` 再回收 tmp 旧目录；排空为顺序 `remove_dir_all`/`remove_file`，**无逐条 sleep 节流** | 基本等价；唯一差异是 RustFS 排空不节流，trash 积压大时单轮 IO 更突发（5m 周期天然限频），文档化，如实测出现清理风暴再补节流 | +| tmp 非 trash 旧目录 | 并在 `cleanupStaleUploadsOnDisk` 内：非 `.trash` 的 tmp 目录超过 `stale_uploads_expiry`（24h）rename 进 trash（随 6h 任务） | `cleanup_stale_tmp_objects`（crates/ecstore/src/disk/local.rs）随 5m 循环执行：非 `.trash` 目录超过 `STALE_TMP_OBJECT_EXPIRY` = 24h 即 rename 进 trash；另有启动时 tmp → tmp-old 整体换名 + 后台删除的崩溃安全路径 | 行为等价（阈值同为 24h）；RustFS 检查频率 5m vs MinIO 6h，回收更及时 | + +周期与环境变量默认值对照（两边一致）： + +| 项 | RustFS | MinIO | +|---|---|---| +| stale upload 过期阈值 | `RUSTFS_API_STALE_UPLOADS_EXPIRY`，默认 24h | `MINIO_API_STALE_UPLOADS_EXPIRY`，默认 24h | +| stale multipart 清理周期 | `RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL`，默认 6h | `MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL`，默认 6h | +| trash 排空周期 | 5m（常量，暂无开关） | `MINIO_API_DELETE_CLEANUP_INTERVAL`，默认 5m | + +关于 rustfs/src/delete_tail_activity.rs：它**不覆盖三段中的任何一段**。该模块是 delete 尾部活动的进程内指标计数（inflight gauge + 耗时 histogram），供 allocator 回收压力判断（rustfs/src/allocator_reclaim.rs）使用；生产代码目前只在对象复用路径使用 `Replication`/`Notify` 两个 stage 计数，`Tail`/`Cleanup` 枚举值暂无调用点。 + +崩溃残留窗口结论： + +- trash 内部残留（排空中途崩溃）：`.trash/` 是自包含目录，下一轮 5m tick 重扫 `.trash` 自然收敛，与 MinIO 相同。 +- 跨盘扇出中途崩溃（部分盘已 rename 进 trash、其余未动）：若剩余盘数仍满足写 quorum，下一轮 6h 任务重新发现候选并重删，自然收敛；若已清理盘数超过 parity（剩余低于写 quorum），`check_multipart_upload_path_exists` 因 `FileNotFound` 不在 `OBJECT_OP_IGNORED_ERRS`（crates/ecstore/src/disk/error_reduce.rs）而判 quorum 失败，候选被跳过，残留 uploadID 目录不会被该任务收敛（不可见于 S3 API，仅占盘空间）。该窗口极窄（逐盘 rename 为毫秒级，需恰在扇出中途且已过 parity 盘时进程死亡）。MinIO 同场景会收敛（逐盘独立处理、无 quorum 闸门）。**分级：有崩溃残留窗口（极窄）→ 登记后续修复**；修复需为清理守卫提供把"已不存在"计为达成终态的专用 quorum 变体（不能改共享的 `check_multipart_upload_path_exists` 语义，它同时服务 CompleteMultipartUpload），超出本批"几行小修"边界，不在本 PR 扩 scope。 + +## 5. HS-16：单机（ErasureSD）默认扫描周期决策 + +启动期曾有预留钩子 `single_disk_default_cycle_secs`，可按维护特征（lifecycle/replication/巡检失败）为单机覆盖默认周期，但从未接线、恒返回 `None`，已删除（本批 PR）。决策：**单机默认周期保持速度档派生（`default` 档 = 60s），不做特殊覆盖**。理由：其一，无任何实测依据表明单机冷启动 ILM 延迟需要更短周期，凭空缩短只会放大空闲扫描频次；其二，单机已有清洁空闲退避（连续干净周期间隔翻倍，默认 bitrot 窗口下封顶约 42 分钟，见 scanner-runtime-controls.md），空闲时的周期压力已被消化；其三，若确有诉求，用户可用 `RUSTFS_SCANNER_CYCLE` / `scanner.cycle` 显式配置，无需内置特殊路径。需要更激进短周期的场景应先拿实测数据再议。 + +## 6. 决策摘要 + +- HS-14：保持 `RUSTFS_SCANNER_IDLE_MODE` 现语义（true=节流总闸+前台读下限，false=全速），文档化对照表与迁移警告，不做兼容别名。 +- HS-16：删除恒 `None` 的单机默认周期钩子，单机周期保持速度档派生 + 清洁空闲退避。 +- HS-18：三段清理行为等价（trash 排空无逐条节流、tmp 回收频率 5m vs 6h 两处小差异文档化）；跨盘扇出的极窄崩溃残留窗口登记后续；周期默认值 24h/6h/5m 与 MinIO 对齐。 From e4eae22a701cfcc0d81e60124df44d72176a3cc1 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 19 Aug 2026 13:10:53 +0800 Subject: [PATCH 10/27] test(crypto): replace the one-file key scan with a repo-wide guard (#6255) From ceb6f779fbc1257bfa613f82b36290784be031d0 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 19 Aug 2026 13:11:32 +0800 Subject: [PATCH 11/27] chore(rustfs): adjudicate 37 bare dead_code allows in the four densest files (#6254) --- rustfs/src/admin/console.rs | 29 ++++++++----------- rustfs/src/admin/handlers/tier.rs | 8 ----- rustfs/src/auth.rs | 28 ------------------ rustfs/src/storage/concurrency/io_schedule.rs | 27 +---------------- 4 files changed, 13 insertions(+), 79 deletions(-) diff --git a/rustfs/src/admin/console.rs b/rustfs/src/admin/console.rs index acf820e6b..5ee314044 100644 --- a/rustfs/src/admin/console.rs +++ b/rustfs/src/admin/console.rs @@ -119,6 +119,10 @@ async fn static_handler(uri: Uri) -> impl IntoResponse { #[derive(Debug, Serialize, Clone)] pub(crate) struct Config { #[serde(skip)] + #[allow( + dead_code, + reason = "reachable only from this file's tests: no route registers config_handler (backlog#1823)" + )] port: u16, api: Api, s3: S3, @@ -176,11 +180,14 @@ impl Config { } } + #[allow( + dead_code, + reason = "reachable only from this file's tests: no route registers config_handler (backlog#1823)" + )] fn to_json(&self) -> String { serde_json::to_string(self).unwrap_or_default() } - #[allow(dead_code)] pub(crate) fn version_info(&self) -> String { format!( "RELEASE.{}@{} (rust {} {})", @@ -190,21 +197,6 @@ impl Config { build::BUILD_TARGET ) } - - #[allow(dead_code)] - pub(crate) fn version(&self) -> String { - self.release.version.clone() - } - - #[allow(dead_code)] - pub(crate) fn license(&self) -> String { - format!("{} {}", self.license.name.clone(), self.license.url.clone()) - } - - #[allow(dead_code)] - pub(crate) fn doc(&self) -> String { - self.doc.clone() - } } fn build_console_api_base_url(base_url: &str) -> String { @@ -353,7 +345,10 @@ async fn version_handler() -> impl IntoResponse { /// - 200 OK with JSON body containing the console configuration if initialized. /// - 500 Internal Server Error if configuration is not initialized. #[instrument(fields(uri))] -#[allow(dead_code)] +#[allow( + dead_code, + reason = "reachable only from this file's tests: no route registers it (backlog#1823)" +)] async fn config_handler(uri: Uri, headers: HeaderMap) -> impl IntoResponse { // Get the scheme from the headers or use the URI scheme let scheme = headers diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs index 0290ebbcb..03cd43714 100644 --- a/rustfs/src/admin/handlers/tier.rs +++ b/rustfs/src/admin/handlers/tier.rs @@ -53,25 +53,18 @@ const EVENT_ADMIN_TIER_STATE: &str = "admin_tier_state"; #[derive(Debug, Clone, serde::Deserialize, Default)] pub struct AddTierQuery { #[serde(rename = "accessKey")] - #[allow(dead_code)] pub access_key: Option, - #[allow(dead_code)] pub status: Option, #[serde(rename = "secretKey")] - #[allow(dead_code)] pub secret_key: Option, #[serde(rename = "serviceName")] - #[allow(dead_code)] pub service_name: Option, #[serde(rename = "sessionToken")] - #[allow(dead_code)] pub session_token: Option, pub tier: Option, #[serde(rename = "tierName")] - #[allow(dead_code)] pub tier_name: Option, #[serde(rename = "tierType")] - #[allow(dead_code)] pub tier_type: Option, pub force: Option, } @@ -532,7 +525,6 @@ impl Operation for EditTier { #[derive(Debug, Clone, serde::Deserialize, Default)] pub struct BucketQuery { #[serde(rename = "bucket")] - #[allow(dead_code)] pub bucket: String, } pub struct ListTiers {} diff --git a/rustfs/src/auth.rs b/rustfs/src/auth.rs index dc254afa8..fa12a564e 100644 --- a/rustfs/src/auth.rs +++ b/rustfs/src/auth.rs @@ -812,12 +812,10 @@ fn is_reserved_condition_key(key: &str, server_derived: &HashMap AuthType { get_request_auth_type_with_query(header, None) } -#[allow(dead_code)] pub(crate) fn get_request_auth_type_with_query(header: &HeaderMap, query: Option<&str>) -> AuthType { if is_request_signature_v2(header) { AuthType::SignedV2 @@ -846,20 +844,6 @@ pub(crate) fn get_request_auth_type_with_query(header: &HeaderMap, query: Option } } -/// Helper function to determine auth type and signature version -/// -/// # Arguments -/// * `header` - HTTP headers of the request -/// -/// # Returns -/// * `(String, String)` - Tuple of auth type and signature version -/// -#[allow(dead_code)] -fn determine_auth_type_and_version(header: &HeaderMap) -> (String, String) { - determine_auth_type_and_version_with_query(header, None) -} - -#[allow(dead_code)] fn determine_auth_type_and_version_with_query(header: &HeaderMap, query: Option<&str>) -> (String, String) { match get_request_auth_type_with_query(header, query) { AuthType::JWT => ("JWT".to_string(), String::new()), @@ -925,18 +909,6 @@ fn is_request_signature_v2(header: &HeaderMap) -> bool { false } -/// Verify if request has AWS PreSign Version '4' -/// -/// # Arguments -/// * `header` - HTTP headers of the request -/// -/// # Returns -/// * `bool` - True if request has AWS PreSign Version '4', false otherwise -#[allow(dead_code)] -pub(crate) fn is_request_presigned_signature_v4(header: &HeaderMap) -> bool { - is_request_presigned_signature_v4_with_query(header, None) -} - pub(crate) fn is_request_presigned_signature_v4_with_query(header: &HeaderMap, query: Option<&str>) -> bool { if let Some(credential) = header.get(AMZ_CREDENTIAL) { return !credential.to_str().unwrap_or("").is_empty(); diff --git a/rustfs/src/storage/concurrency/io_schedule.rs b/rustfs/src/storage/concurrency/io_schedule.rs index 9ec717623..80ebf7c62 100644 --- a/rustfs/src/storage/concurrency/io_schedule.rs +++ b/rustfs/src/storage/concurrency/io_schedule.rs @@ -72,7 +72,6 @@ impl IoLoadLevel { } /// Get the load level as a string for metrics labels. - #[allow(dead_code)] pub fn as_str(&self) -> &'static str { match self { IoLoadLevel::Low => "low", @@ -83,7 +82,6 @@ impl IoLoadLevel { } /// Get the load level as a numeric index (0=Low, 1=Medium, 2=High, 3=Critical). - #[allow(dead_code)] pub fn level_index(&self) -> u8 { match self { IoLoadLevel::Low => 0, @@ -118,7 +116,6 @@ pub enum IoPriority { impl IoPriority { /// Determine priority from request size using scheduler config thresholds. - #[allow(dead_code)] pub fn from_size(size: i64) -> Self { Self::from_size_with_thresholds( size, @@ -152,19 +149,16 @@ impl IoPriority { } /// Check if this is high priority. - #[allow(dead_code)] pub fn is_high(&self) -> bool { matches!(self, IoPriority::High) } /// Check if this is normal priority. - #[allow(dead_code)] pub fn is_normal(&self) -> bool { matches!(self, IoPriority::Normal) } /// Check if this is low priority. - #[allow(dead_code)] pub fn is_low(&self) -> bool { matches!(self, IoPriority::Low) } @@ -403,7 +397,6 @@ impl IoSchedulerConfig { /// I/O queue status for monitoring. #[derive(Debug, Clone, Default)] -#[allow(dead_code)] pub struct IoQueueStatus { /// Total permits available. pub total_permits: usize, @@ -520,7 +513,6 @@ pub struct IoStrategyCore { impl IoStrategyCore { /// Create a minimal IoStrategyCore with essential fields only. - #[allow(dead_code)] pub fn new(storage_media: StorageMedia, access_pattern: AccessPattern, buffer_size: usize) -> Self { Self { storage_media, @@ -1194,7 +1186,6 @@ impl IoStrategy { } /// Get a human-readable description of the current I/O strategy. - #[allow(dead_code)] pub fn description(&self) -> String { format!( "IoStrategy[{:?}]: buffer={}KB, multiplier={:.2}, readahead={}, wait={:?}", @@ -1282,14 +1273,6 @@ impl IoLoadMetrics { IoLoadLevel::from_wait_duration(self.average_wait()) } - /// Get the overall average wait since startup - #[allow(dead_code)] - pub(crate) fn lifetime_average_wait(&self) -> Duration { - let total = self.total_wait_ns.load(Ordering::Relaxed); - let count = self.observation_count.load(Ordering::Relaxed); - total.checked_div(count).map(Duration::from_nanos).unwrap_or(Duration::ZERO) - } - /// Get the total observation count pub(crate) fn observation_count(&self) -> u64 { self.observation_count.load(Ordering::Relaxed) @@ -1450,13 +1433,13 @@ use tracing::warn; /// Queued I/O request with metadata. #[derive(Debug)] -#[allow(dead_code)] struct QueuedRequest { /// The actual request payload. request: T, /// Time when the request was enqueued. enqueue_time: Instant, /// Original priority assigned to the request. + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] original_priority: IoPriority, /// Current priority (may be boosted for starvation prevention). current_priority: IoPriority, @@ -1466,7 +1449,6 @@ struct QueuedRequest { /// Queue statistics for monitoring. #[derive(Debug, Clone, Default)] -#[allow(dead_code)] struct QueueStats { /// Number of high priority requests processed. high_processed: u64, @@ -1552,7 +1534,6 @@ impl Default for IoPriorityQueueConfig { impl IoPriorityQueueConfig { /// Load configuration from environment. - #[allow(dead_code)] pub fn from_env() -> Self { Self { queue_high_capacity: rustfs_utils::get_env_usize( @@ -1603,7 +1584,6 @@ impl IoPriorityQueueConfig { impl IoPriorityQueue { /// Create a new priority queue with the given configuration. - #[allow(dead_code)] pub fn new(config: IoPriorityQueueConfig) -> Self { let config_clone = config.clone(); Self { @@ -1617,7 +1597,6 @@ impl IoPriorityQueue { } /// Enqueue a request with the given priority. - #[allow(dead_code)] pub async fn enqueue(&self, priority: IoPriority, request: T) { let queued = QueuedRequest { request, @@ -1638,7 +1617,6 @@ impl IoPriorityQueue { /// /// This method performs starvation prevention checks before dequeuing. /// Returns `None` if all queues are empty. - #[allow(dead_code)] pub async fn dequeue(&self) -> Option<(T, IoPriority)> { // 1. Check for starvation prevention self.check_starvation().await; @@ -1716,7 +1694,6 @@ impl IoPriorityQueue { } /// Get current queue status for monitoring. - #[allow(dead_code)] pub async fn status(&self) -> IoQueueStatus { let high_queue = self.high_queue.lock().await; let normal_queue = self.normal_queue.lock().await; @@ -1737,7 +1714,6 @@ impl IoPriorityQueue { } /// Get the total number of queued requests. - #[allow(dead_code)] pub async fn len(&self) -> usize { let high_queue = self.high_queue.lock().await; let normal_queue = self.normal_queue.lock().await; @@ -1747,7 +1723,6 @@ impl IoPriorityQueue { } /// Check if all queues are empty. - #[allow(dead_code)] pub async fn is_empty(&self) -> bool { self.len().await == 0 } From d404e1bb8ae082be2fbdb0635fecd380375c6547 Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 19 Aug 2026 13:11:46 +0800 Subject: [PATCH 12/27] refactor(heal,scanner): clean up dead heal/scanner code, flags, and metrics (HS-09/10/19/20) (#6256) --- crates/config/src/constants/scanner.rs | 9 - crates/heal/src/heal/channel.rs | 36 +- crates/heal/src/heal/erasure_healer.rs | 32 +- crates/heal/src/heal/manager.rs | 40 -- crates/heal/src/heal/storage.rs | 615 +++-------------------- crates/heal/src/heal/task.rs | 47 +- crates/heal/src/lib.rs | 44 +- crates/heal/tests/heal_bug_fixes_test.rs | 72 +-- crates/scanner/src/scanner_folder.rs | 59 --- rustfs/src/admin/handlers/heal.rs | 181 +++---- rustfs/src/admin/storage_api.rs | 4 - rustfs/src/storage/rpc/node_service.rs | 46 +- rustfs/src/storage/storage_api.rs | 2 - 13 files changed, 138 insertions(+), 1049 deletions(-) diff --git a/crates/config/src/constants/scanner.rs b/crates/config/src/constants/scanner.rs index bcb730c2d..8086c3229 100644 --- a/crates/config/src/constants/scanner.rs +++ b/crates/config/src/constants/scanner.rs @@ -228,15 +228,6 @@ pub const DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS: usize = 4; /// Default object interval for cooperative scanner yields. pub const DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS: u64 = 128; -/// Compatibility flag kept for Patch 3 rollback windows. -/// -/// Inline scanner heal execution has been removed in favor of heal-candidate enqueue. -/// When this flag is enabled, RustFS logs a warning and continues to use enqueue-based heal. -pub const ENV_SCANNER_INLINE_HEAL_ENABLE: &str = "RUSTFS_SCANNER_INLINE_HEAL_ENABLE"; - -/// Default inline scanner heal compatibility mode. -pub const DEFAULT_SCANNER_INLINE_HEAL_ENABLE: bool = false; - /// Scanner speed preset controlling throttling behavior. /// /// Each preset defines three parameters: diff --git a/crates/heal/src/heal/channel.rs b/crates/heal/src/heal/channel.rs index 8f4ab9e8a..edfcf2613 100644 --- a/crates/heal/src/heal/channel.rs +++ b/crates/heal/src/heal/channel.rs @@ -759,7 +759,7 @@ impl HealChannelProcessor { #[cfg(test)] mod tests { - use super::super::{DiskStore, Endpoint}; + use super::super::DiskStore; use super::*; use crate::heal::manager::HealConfig; use crate::heal::storage::{HealObjectInfo, HealStorageAPI}; @@ -776,45 +776,18 @@ mod tests { async fn get_object_meta(&self, _bucket: &str, _object: &str) -> crate::Result> { Ok(None) } - async fn get_object_data(&self, _bucket: &str, _object: &str) -> crate::Result>> { - Ok(None) - } - async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> crate::Result<()> { - Ok(()) - } - async fn delete_object(&self, _bucket: &str, _object: &str) -> crate::Result<()> { - Ok(()) - } - async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> crate::Result { - Ok(true) - } async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> crate::Result> { Ok(vec![]) } - async fn get_disk_status(&self, _endpoint: &Endpoint) -> crate::Result { - Ok(crate::heal::storage::DiskStatus::Ok) - } - async fn format_disk(&self, _endpoint: &Endpoint) -> crate::Result<()> { - Ok(()) - } async fn get_bucket_info(&self, _bucket: &str) -> crate::Result> { Ok(None) } - async fn heal_bucket_metadata(&self, _bucket: &str) -> crate::Result<()> { - Ok(()) - } async fn list_buckets(&self) -> crate::Result> { Ok(vec![]) } async fn object_exists(&self, _bucket: &str, _object: &str) -> crate::Result { Ok(false) } - async fn get_object_size(&self, _bucket: &str, _object: &str) -> crate::Result> { - Ok(None) - } - async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> crate::Result> { - Ok(None) - } async fn heal_object( &self, _bucket: &str, @@ -837,13 +810,6 @@ mod tests { ) -> crate::Result<(rustfs_madmin::heal_commands::HealResultItem, Option)> { Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None)) } - async fn list_objects_for_heal( - &self, - _bucket: &str, - _prefix: &str, - ) -> crate::Result> { - Ok(vec![]) - } async fn list_objects_for_heal_page( &self, _bucket: &str, diff --git a/crates/heal/src/heal/erasure_healer.rs b/crates/heal/src/heal/erasure_healer.rs index 04075837c..920fffcf5 100644 --- a/crates/heal/src/heal/erasure_healer.rs +++ b/crates/heal/src/heal/erasure_healer.rs @@ -1267,7 +1267,7 @@ mod resume_loop_tests { CheckpointManager, RESUME_CHECKPOINT_FILE, ReplacementTargetIdentity, ResumeDeleteFailure, ResumeManager, ResumeUtils, compose_key, }; - use crate::heal::storage::{DiskStatus, HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI}; + use crate::heal::storage::{HealLifecycleExpiryContext, HealListItem, HealObjectInfo, HealStorageAPI}; use crate::heal::storage_api::status::BucketInfo; use crate::heal::{ BUCKET_META_PREFIX, DiskOption, DiskStore, EcstoreError, Endpoint, HealDiskExt as _, RUSTFS_META_BUCKET, new_disk, @@ -1448,36 +1448,15 @@ mod resume_loop_tests { async fn get_object_meta(&self, _b: &str, _o: &str) -> Result> { Ok(None) } - async fn get_object_data(&self, _b: &str, _o: &str) -> Result>> { - Ok(None) - } - async fn put_object_data(&self, _b: &str, _o: &str, _d: &[u8]) -> Result<()> { - Ok(()) - } - async fn delete_object(&self, _b: &str, _o: &str) -> Result<()> { - Ok(()) - } - async fn verify_object_integrity(&self, _b: &str, _o: &str) -> Result { - Ok(true) - } async fn ec_decode_rebuild(&self, _b: &str, _o: &str) -> Result> { Ok(Vec::new()) } - async fn get_disk_status(&self, _e: &Endpoint) -> Result { - Ok(DiskStatus::Ok) - } - async fn format_disk(&self, _e: &Endpoint) -> Result<()> { - Ok(()) - } async fn get_bucket_info(&self, bucket: &str) -> Result> { Ok(Some(BucketInfo { name: bucket.to_string(), ..Default::default() })) } - async fn heal_bucket_metadata(&self, _b: &str) -> Result<()> { - Ok(()) - } async fn list_buckets(&self) -> Result> { Ok(Vec::new()) } @@ -1485,12 +1464,6 @@ mod resume_loop_tests { // Must never be consulted: the resume loop always goes through heal_object. panic!("object_exists must not be called by the resume heal loop"); } - async fn get_object_size(&self, _b: &str, _o: &str) -> Result> { - Ok(None) - } - async fn get_object_checksum(&self, _b: &str, _o: &str) -> Result> { - Ok(None) - } async fn load_heal_lifecycle_expiry_context(&self, _bucket: &str) -> Result> { Ok((!self.lifecycle_expired.lock().unwrap().is_empty()).then(HealLifecycleExpiryContext::test)) } @@ -1556,9 +1529,6 @@ mod resume_loop_tests { ReplacementCommitEvidence::Error(message) => Err(Error::other(message)), } } - async fn list_objects_for_heal(&self, _b: &str, _p: &str) -> Result> { - Ok(Vec::new()) - } async fn list_objects_for_heal_page( &self, _bucket: &str, diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 3a243c88e..4c3c622c7 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -3882,42 +3882,14 @@ mod tests { Ok(None) } - async fn get_object_data(&self, _bucket: &str, _object: &str) -> Result>> { - Ok(None) - } - - async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> Result<()> { - Ok(()) - } - - async fn delete_object(&self, _bucket: &str, _object: &str) -> Result<()> { - Ok(()) - } - - async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> Result { - Ok(true) - } - async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> Result> { Ok(Vec::new()) } - async fn get_disk_status(&self, _endpoint: &Endpoint) -> Result { - Ok(crate::heal::storage::DiskStatus::Ok) - } - - async fn format_disk(&self, _endpoint: &Endpoint) -> Result<()> { - Ok(()) - } - async fn get_bucket_info(&self, _bucket: &str) -> Result> { Ok(None) } - async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<()> { - Ok(()) - } - async fn list_buckets(&self) -> Result> { if let Some(hook) = manager_recovery_test_hook() { *hook.listed.lock().expect("manager recovery listed lock should not poison") = true; @@ -3929,14 +3901,6 @@ mod tests { Ok(bucket == "retry-transition") } - async fn get_object_size(&self, _bucket: &str, _object: &str) -> Result> { - Ok(None) - } - - async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> Result> { - Ok(None) - } - async fn heal_object( &self, bucket: &str, @@ -3998,10 +3962,6 @@ mod tests { Ok((HealResultItem::default(), None)) } - async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> Result> { - Ok(Vec::new()) - } - async fn list_objects_for_heal_page( &self, _bucket: &str, diff --git a/crates/heal/src/heal/storage.rs b/crates/heal/src/heal/storage.rs index fbe51f050..63ca002e4 100644 --- a/crates/heal/src/heal/storage.rs +++ b/crates/heal/src/heal/storage.rs @@ -27,7 +27,7 @@ use super::storage_api::storage::{ BucketInfo, BucketOperations, DiskSetSelector, HealOperations as _, ListOperations as _, ObjectIO as _, ObjectOperations as _, StorageAdminApi, }; -use super::{DiskStore, ECStore, Endpoint, HealDiskExt as _, StorageError, resume::ReplacementTargetIdentity}; +use super::{DiskStore, ECStore, HealDiskExt as _, StorageError, resume::ReplacementTargetIdentity}; pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader}; #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] @@ -65,7 +65,6 @@ const LOG_COMPONENT_HEAL: &str = "heal"; const LOG_SUBSYSTEM_STORAGE: &str = "storage"; const EVENT_HEAL_STORAGE_OBJECT_IO: &str = "heal_storage_object_io"; const EVENT_HEAL_STORAGE_OBJECT_READ_LIMIT: &str = "heal_storage_object_read_limit"; -const EVENT_HEAL_STORAGE_OBJECT_VERIFY: &str = "heal_storage_object_verify"; const EVENT_HEAL_STORAGE_ADMIN_OP: &str = "heal_storage_admin_op"; const EVENT_HEAL_STORAGE_REPAIR_OP: &str = "heal_storage_repair_op"; @@ -312,56 +311,23 @@ pub struct HealListItem { pub is_delete_marker: bool, } -/// Disk status for heal operations -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum DiskStatus { - /// Ok - Ok, - /// Offline - Offline, - /// Corrupt - Corrupt, - /// Missing - Missing, - /// Permission denied - PermissionDenied, - /// Faulty - Faulty, - /// Root mount - RootMount, - /// Unknown - Unknown, - /// Unformatted - Unformatted, -} - /// Heal storage layer interface #[async_trait] pub trait HealStorageAPI: Send + Sync { /// Get object meta + /// + /// Reserved for HS-01 MRF wiring (rustfs/backlog#1865): MRF intents + /// currently execute through `heal_object`; keep this entry point for the + /// metadata-corruption variant that must inspect metadata first. async fn get_object_meta(&self, bucket: &str, object: &str) -> Result>; - /// Get object data - async fn get_object_data(&self, bucket: &str, object: &str) -> Result>>; - - /// Put object data - async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()>; - - /// Delete object - async fn delete_object(&self, bucket: &str, object: &str) -> Result<()>; - - /// Check object integrity - async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result; - /// EC decode rebuild + /// + /// Reserved for HS-01 MRF wiring (rustfs/backlog#1865): urgent ECDecode + /// requests currently execute through `heal_object`; keep the explicit + /// rebuild-and-read path for the decode-failure fast variant. async fn ec_decode_rebuild(&self, bucket: &str, object: &str) -> Result>; - /// Get disk status - async fn get_disk_status(&self, endpoint: &Endpoint) -> Result; - - /// Format disk - async fn format_disk(&self, endpoint: &Endpoint) -> Result<()>; - /// Get bucket info async fn get_bucket_info(&self, bucket: &str) -> Result>; @@ -387,21 +353,12 @@ pub trait HealStorageAPI: Send + Sync { Ok(false) } - /// Fix bucket metadata - async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()>; - /// Get all buckets async fn list_buckets(&self) -> Result>; /// Check object exists async fn object_exists(&self, bucket: &str, object: &str) -> Result; - /// Get object size - async fn get_object_size(&self, bucket: &str, object: &str) -> Result>; - - /// Get object checksum - async fn get_object_checksum(&self, bucket: &str, object: &str) -> Result>; - /// Heal object using ecstore async fn heal_object( &self, @@ -453,12 +410,6 @@ pub trait HealStorageAPI: Send + Sync { Ok(false) } - /// List object versions for healing (returns all versions, may use significant memory for large buckets) - /// - /// WARNING: This method loads all object versions into memory at once. For buckets with many - /// objects/versions, consider using `list_objects_for_heal_page` instead to process versions in pages. - async fn list_objects_for_heal(&self, bucket: &str, prefix: &str) -> Result>; - /// List object versions for healing with pagination (returns one page and continuation token) /// Returns (versions, next_continuation_token, is_truncated). The continuation token is an /// opaque composite `(marker, version_marker)` value — see `encode_heal_token`/`decode_heal_token`. @@ -527,89 +478,11 @@ impl ECStoreHealStorage { pub fn new(ecstore: Arc) -> Self { Self { ecstore } } -} - -fn is_transient_object_exists_message(message: &str) -> bool { - let message = message.to_ascii_lowercase(); - - [ - "failed to acquire read lock", - "lock acquisition failed", - "lock acquisition timeout", - "quorum not reached", - "deadline has elapsed", - "timed out", - "network error", - "transport error", - "connection refused", - ] - .iter() - .any(|pattern| message.contains(pattern)) -} - -fn is_transient_object_exists_error(err: &StorageError) -> bool { - if err.is_quorum_error() { - return true; - } - - match err { - StorageError::Lock(lock_err) => lock_err.is_retryable() || is_transient_object_exists_message(&lock_err.to_string()), - StorageError::Io(io_err) => is_transient_object_exists_message(&io_err.to_string()), - StorageError::SlowDown | StorageError::OperationCanceled => true, - _ => false, - } -} - -#[async_trait] -impl HealStorageAPI for ECStoreHealStorage { - async fn get_object_meta(&self, bucket: &str, object: &str) -> Result> { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "get_object_meta", - bucket, - object, - "Heal storage request started" - ); - - match self.ecstore.get_object_info(bucket, object, &Default::default()).await { - Ok(info) => Ok(Some(info)), - Err(e) => { - // Map ObjectNotFound to None to align with Option return type - if matches!(e, StorageError::ObjectNotFound(_, _)) { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "get_object_meta", - bucket, - object, - result = "not_found", - "Heal storage object metadata missing" - ); - Ok(None) - } else { - error!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "get_object_meta", - bucket, - object, - result = "failed", - error = %e, - "Heal storage request failed" - ); - Err(Error::other(e)) - } - } - } - } + /// Read back an object's bytes, capped to bound memory. + /// + /// Private support for the reserved `ec_decode_rebuild` (HS-01); not part + /// of the storage trait surface. async fn get_object_data(&self, bucket: &str, object: &str) -> Result>> { debug!( target: "rustfs::heal::storage", @@ -695,196 +568,85 @@ impl HealStorageAPI for ECStoreHealStorage { } Ok(Some(buf)) } +} - async fn put_object_data(&self, bucket: &str, object: &str, data: &[u8]) -> Result<()> { +fn is_transient_object_exists_message(message: &str) -> bool { + let message = message.to_ascii_lowercase(); + + [ + "failed to acquire read lock", + "lock acquisition failed", + "lock acquisition timeout", + "quorum not reached", + "deadline has elapsed", + "timed out", + "network error", + "transport error", + "connection refused", + ] + .iter() + .any(|pattern| message.contains(pattern)) +} + +fn is_transient_object_exists_error(err: &StorageError) -> bool { + if err.is_quorum_error() { + return true; + } + + match err { + StorageError::Lock(lock_err) => lock_err.is_retryable() || is_transient_object_exists_message(&lock_err.to_string()), + StorageError::Io(io_err) => is_transient_object_exists_message(&io_err.to_string()), + StorageError::SlowDown | StorageError::OperationCanceled => true, + _ => false, + } +} + +#[async_trait] +impl HealStorageAPI for ECStoreHealStorage { + async fn get_object_meta(&self, bucket: &str, object: &str) -> Result> { debug!( target: "rustfs::heal::storage", event = EVENT_HEAL_STORAGE_OBJECT_IO, component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "put_object_data", - bucket, - object, - bytes = data.len(), - "Heal storage request started" - ); - - let mut reader = HealPutObjReader::from_vec(data.to_vec()); - match (*self.ecstore) - .put_object(bucket, object, &mut reader, &Default::default()) - .await - { - Ok(_) => { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "put_object_data", - bucket, - object, - result = "ok", - "Heal storage object write completed" - ); - Ok(()) - } - Err(e) => { - error!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "put_object_data", - bucket, - object, - result = "failed", - error = %e, - "Heal storage request failed" - ); - Err(Error::other(e)) - } - } - } - - async fn delete_object(&self, bucket: &str, object: &str) -> Result<()> { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "delete_object", + operation = "get_object_meta", bucket, object, "Heal storage request started" ); - match self.ecstore.delete_object(bucket, object, Default::default()).await { - Ok(_) => { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "delete_object", - bucket, - object, - result = "ok", - "Heal storage object delete completed" - ); - Ok(()) - } + match self.ecstore.get_object_info(bucket, object, &Default::default()).await { + Ok(info) => Ok(Some(info)), Err(e) => { - error!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "delete_object", - bucket, - object, - result = "failed", - error = %e, - "Heal storage request failed" - ); - Err(Error::other(e)) - } - } - } - - async fn verify_object_integrity(&self, bucket: &str, object: &str) -> Result { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_VERIFY, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - bucket, - object, - state = "started", - "Heal storage object verification started" - ); - - // Check object metadata first - match self.get_object_meta(bucket, object).await? { - Some(obj_info) => { - if obj_info.size < 0 { - warn!( + // Map ObjectNotFound to None to align with Option return type + if matches!(e, StorageError::ObjectNotFound(_, _)) { + debug!( target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_VERIFY, + event = EVENT_HEAL_STORAGE_OBJECT_IO, component = LOG_COMPONENT_HEAL, subsystem = LOG_SUBSYSTEM_STORAGE, + operation = "get_object_meta", bucket, object, - state = "invalid_size", - "Heal storage object verification failed" + result = "not_found", + "Heal storage object metadata missing" ); - return Ok(false); + Ok(None) + } else { + error!( + target: "rustfs::heal::storage", + event = EVENT_HEAL_STORAGE_OBJECT_IO, + component = LOG_COMPONENT_HEAL, + subsystem = LOG_SUBSYSTEM_STORAGE, + operation = "get_object_meta", + bucket, + object, + result = "failed", + error = %e, + "Heal storage request failed" + ); + Err(Error::other(e)) } - - // Stream-read the object to a sink to avoid loading into memory - match (*self.ecstore) - .get_object_reader(bucket, object, None, Default::default(), &Default::default()) - .await - { - Ok(reader) => { - let mut stream = reader.stream; - match tokio::io::copy(&mut stream, &mut tokio::io::sink()).await { - Ok(_) => { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_VERIFY, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - bucket, - object, - state = "ok", - "Heal storage object verified" - ); - Ok(true) - } - Err(e) => { - warn!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_VERIFY, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - bucket, - object, - state = "stream_read_failed", - error = %e, - "Heal storage object verification failed" - ); - Ok(false) - } - } - } - Err(e) => { - warn!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_VERIFY, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - bucket, - object, - state = "reader_open_failed", - error = %e, - "Heal storage object verification failed" - ); - Ok(false) - } - } - } - None => { - warn!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_VERIFY, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - bucket, - object, - state = "metadata_missing", - "Heal storage object verification failed" - ); - Ok(false) } } } @@ -976,81 +738,6 @@ impl HealStorageAPI for ECStoreHealStorage { } } - async fn get_disk_status(&self, endpoint: &Endpoint) -> Result { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "get_disk_status", - endpoint = ?endpoint, - state = "started", - "Heal storage admin operation started" - ); - - // TODO: implement disk status check using ecstore - // For now, return Ok status - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "get_disk_status", - endpoint = ?endpoint, - result = "ok", - disk_status = "ok", - "Heal storage disk status resolved" - ); - Ok(DiskStatus::Ok) - } - - async fn format_disk(&self, endpoint: &Endpoint) -> Result<()> { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "format_disk", - endpoint = ?endpoint, - state = "started", - "Heal storage admin operation started" - ); - - // Use ecstore's heal_format - match self.heal_format(false).await { - Ok((_, error)) => { - if error.is_some() { - return Err(Error::other(format!("Format failed: {error:?}"))); - } - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "format_disk", - endpoint = ?endpoint, - result = "ok", - "Heal storage disk format completed" - ); - Ok(()) - } - Err(e) => { - error!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "format_disk", - endpoint = ?endpoint, - result = "failed", - error = %e, - "Heal storage admin operation failed" - ); - Err(e) - } - } - } - async fn get_bucket_info(&self, bucket: &str) -> Result> { debug!( target: "rustfs::heal::storage", @@ -1161,61 +848,6 @@ impl HealStorageAPI for ECStoreHealStorage { } } - async fn heal_bucket_metadata(&self, bucket: &str) -> Result<()> { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_REPAIR_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "heal_bucket_metadata", - bucket, - state = "started", - "Heal storage repair started" - ); - - let heal_opts = HealOpts { - recursive: true, - dry_run: false, - remove: false, - recreate: false, - scan_mode: HealScanMode::Normal, - update_parity: false, - no_lock: false, - pool: None, - set: None, - }; - - match self.heal_bucket(bucket, &heal_opts).await { - Ok(_) => { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_REPAIR_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "heal_bucket_metadata", - bucket, - result = "ok", - "Heal storage bucket metadata repaired" - ); - Ok(()) - } - Err(e) => { - error!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_REPAIR_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "heal_bucket_metadata", - bucket, - result = "failed", - error = %e, - "Heal storage repair failed" - ); - Err(e) - } - } - } - async fn list_buckets(&self) -> Result> { debug!( target: "rustfs::heal::storage", @@ -1315,48 +947,6 @@ impl HealStorageAPI for ECStoreHealStorage { } } - async fn get_object_size(&self, bucket: &str, object: &str) -> Result> { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "get_object_size", - bucket, - object, - "Heal storage request started" - ); - - match self.get_object_meta(bucket, object).await { - Ok(Some(obj_info)) => Ok(Some(obj_info.size as u64)), - Ok(None) => Ok(None), - Err(e) => Err(e), - } - } - - async fn get_object_checksum(&self, bucket: &str, object: &str) -> Result> { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_OBJECT_IO, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "get_object_checksum", - bucket, - object, - "Heal storage request started" - ); - - match self.get_object_meta(bucket, object).await { - Ok(Some(obj_info)) => { - // Convert checksum bytes to hex string - let checksum = obj_info.checksum.iter().map(|b| format!("{b:02x}")).collect::(); - Ok(Some(checksum)) - } - Ok(None) => Ok(None), - Err(e) => Err(e), - } - } - async fn heal_object( &self, bucket: &str, @@ -1547,65 +1137,6 @@ impl HealStorageAPI for ECStoreHealStorage { .map_err(Error::Storage) } - async fn list_objects_for_heal(&self, bucket: &str, prefix: &str) -> Result> { - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "list_objects_for_heal", - bucket, - prefix, - state = "started", - "Heal storage admin operation started" - ); - warn!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "list_objects_for_heal", - bucket, - prefix, - state = "memory_heavy", - "Heal storage version listing loads all versions into memory (footprint is per-version, not per-object)" - ); - - let mut all_objects: Vec = Vec::new(); - let mut continuation_token: Option = None; - - loop { - let (page_objects, next_token, is_truncated) = self - .list_objects_for_heal_page(bucket, prefix, continuation_token.as_deref(), false) - .await?; - - all_objects.extend(page_objects); - - if !is_truncated { - break; - } - - continuation_token = next_heal_listing_token(bucket, prefix, next_token, is_truncated)?; - if continuation_token.is_none() { - break; - } - } - - debug!( - target: "rustfs::heal::storage", - event = EVENT_HEAL_STORAGE_ADMIN_OP, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_STORAGE, - operation = "list_objects_for_heal", - bucket, - prefix, - object_count = all_objects.len(), - result = "ok", - "Heal storage object listing completed" - ); - Ok(all_objects) - } - async fn list_objects_for_heal_page( &self, bucket: &str, diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index 2c0626d8f..7b1d33348 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -2822,7 +2822,7 @@ impl std::fmt::Debug for HealTask { mod tests { use super::super::{DiskOption, DiskStore, Endpoint, HealDiskExt as _, new_disk}; use super::*; - use crate::heal::storage::{DiskStatus, HealListItem, HealObjectInfo}; + use crate::heal::storage::{HealListItem, HealObjectInfo}; use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, TraceSubscription, TraceVal, subscribe_trace_events}; use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem, Infos}; use std::collections::{HashMap, VecDeque}; @@ -3354,7 +3354,6 @@ mod tests { object_exists_by_name: Mutex>, heal_object_outcome: Mutex>, heal_object_outcomes: Mutex>>, - deleted_objects: Mutex>, format_no_heal_required: Mutex, global_format_calls: Mutex, replacement_format_calls: Mutex)>>, @@ -3547,35 +3546,10 @@ mod tests { Ok(None) } - async fn get_object_data(&self, _bucket: &str, _object: &str) -> Result>> { - Ok(None) - } - - async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> Result<()> { - Ok(()) - } - - async fn delete_object(&self, _bucket: &str, object: &str) -> Result<()> { - self.deleted_objects.lock().unwrap().push(object.to_string()); - Ok(()) - } - - async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> Result { - Ok(true) - } - async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> Result> { Ok(Vec::new()) } - async fn get_disk_status(&self, _endpoint: &Endpoint) -> Result { - Ok(DiskStatus::Ok) - } - - async fn format_disk(&self, _endpoint: &Endpoint) -> Result<()> { - Ok(()) - } - async fn get_bucket_info(&self, bucket: &str) -> Result> { Ok(Some(BucketInfo { name: bucket.to_string(), @@ -3590,10 +3564,6 @@ mod tests { Ok(*self.usage_baseline.lock().unwrap()) } - async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<()> { - Ok(()) - } - async fn list_buckets(&self) -> Result> { let buckets = self .listed_buckets @@ -3621,14 +3591,6 @@ mod tests { Ok(self.object_exists.lock().unwrap().unwrap_or(true)) } - async fn get_object_size(&self, _bucket: &str, _object: &str) -> Result> { - Ok(None) - } - - async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> Result> { - Ok(None) - } - async fn heal_object( &self, bucket: &str, @@ -3764,10 +3726,6 @@ mod tests { Ok(*self.replacement_targets_ready.lock().unwrap()) } - async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> Result> { - Ok(vec![heal_item("object-a"), heal_item("object-b")]) - } - async fn list_objects_for_heal_page( &self, bucket: &str, @@ -4858,7 +4816,7 @@ mod tests { } #[tokio::test] - async fn test_heal_failure_with_remove_corrupted_does_not_delete_object() { + async fn test_heal_failure_with_remove_corrupted_propagates_remove_flag() { let storage = Arc::new(MockStorage { object_exists: Mutex::new(Some(true)), heal_object_outcome: Mutex::new(Some(MockHealObjectOutcome::OkWithOtherError( @@ -4884,7 +4842,6 @@ mod tests { let err = task.execute().await.expect_err("heal failure should still be reported"); assert!(matches!(err, Error::TaskExecutionFailed { .. })); - assert!(storage.deleted_objects.lock().unwrap().is_empty()); assert!(storage.object_heal_opts.lock().unwrap()[0].remove); } diff --git a/crates/heal/src/lib.rs b/crates/heal/src/lib.rs index f1ec4cebe..29444d0f5 100644 --- a/crates/heal/src/lib.rs +++ b/crates/heal/src/lib.rs @@ -352,8 +352,8 @@ pub(crate) fn set_heal_queue_length(count: usize) { mod tests { use super::{ Error, HEAL_RUNTIME_INIT_TEST_HOOK, HealRuntimeInitTestHook, get_heal_channel_processor, get_heal_manager, - heal::DiskStore, heal::Endpoint, heal::manager::HealConfig, heal::storage::DiskStatus, heal::storage::HealListItem, - heal::storage::HealObjectInfo, heal::storage::HealStorageAPI, init_heal_manager, run_owned_initialization, + heal::DiskStore, heal::manager::HealConfig, heal::storage::HealListItem, heal::storage::HealObjectInfo, + heal::storage::HealStorageAPI, init_heal_manager, run_owned_initialization, }; use crate::heal::storage_api::status::BucketInfo; use rustfs_common::heal_channel::HealOpts; @@ -370,42 +370,14 @@ mod tests { Ok(None) } - async fn get_object_data(&self, _bucket: &str, _object: &str) -> Result>, Error> { - Ok(None) - } - - async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> Result<(), Error> { - Ok(()) - } - - async fn delete_object(&self, _bucket: &str, _object: &str) -> Result<(), Error> { - Ok(()) - } - - async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> Result { - Ok(true) - } - async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> Result, Error> { Ok(Vec::new()) } - async fn get_disk_status(&self, _endpoint: &Endpoint) -> Result { - Ok(DiskStatus::Ok) - } - - async fn format_disk(&self, _endpoint: &Endpoint) -> Result<(), Error> { - Ok(()) - } - async fn get_bucket_info(&self, _bucket: &str) -> Result, Error> { Ok(None) } - async fn heal_bucket_metadata(&self, _bucket: &str) -> Result<(), Error> { - Ok(()) - } - async fn list_buckets(&self) -> Result, Error> { Ok(Vec::new()) } @@ -414,14 +386,6 @@ mod tests { Ok(false) } - async fn get_object_size(&self, _bucket: &str, _object: &str) -> Result, Error> { - Ok(None) - } - - async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> Result, Error> { - Ok(None) - } - async fn heal_object( &self, _bucket: &str, @@ -440,10 +404,6 @@ mod tests { Ok((HealResultItem::default(), None)) } - async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> Result, Error> { - Ok(Vec::new()) - } - async fn list_objects_for_heal_page( &self, _bucket: &str, diff --git a/crates/heal/tests/heal_bug_fixes_test.rs b/crates/heal/tests/heal_bug_fixes_test.rs index dc5addef2..1bdb00348 100644 --- a/crates/heal/tests/heal_bug_fixes_test.rs +++ b/crates/heal/tests/heal_bug_fixes_test.rs @@ -184,45 +184,18 @@ fn test_heal_task_status_atomic_update() { async fn get_object_meta(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { Ok(None) } - async fn get_object_data(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result>> { - Ok(None) - } - async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> rustfs_heal::Result<()> { - Ok(()) - } - async fn delete_object(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<()> { - Ok(()) - } - async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result { - Ok(true) - } async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { Ok(vec![]) } - async fn get_disk_status(&self, _endpoint: &Endpoint) -> rustfs_heal::Result { - Ok(rustfs_heal::heal::storage::DiskStatus::Ok) - } - async fn format_disk(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<()> { - Ok(()) - } async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result> { Ok(None) } - async fn heal_bucket_metadata(&self, _bucket: &str) -> rustfs_heal::Result<()> { - Ok(()) - } async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(vec![]) } async fn object_exists(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result { Ok(false) } - async fn get_object_size(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { - Ok(None) - } - async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { - Ok(None) - } async fn heal_object( &self, _bucket: &str, @@ -245,9 +218,6 @@ fn test_heal_task_status_atomic_update() { ) -> rustfs_heal::Result<(rustfs_madmin::heal_commands::HealResultItem, Option)> { Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None)) } - async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> rustfs_heal::Result> { - Ok(vec![]) - } async fn list_objects_for_heal_page( &self, _bucket: &str, @@ -289,7 +259,7 @@ fn test_heal_task_status_atomic_update() { #[tokio::test] async fn test_heal_task_transient_object_exists_skip_avoids_recreate() { - use rustfs_heal::heal::storage::{DiskStatus, HealListItem, HealObjectInfo, HealStorageAPI}; + use rustfs_heal::heal::storage::{HealListItem, HealObjectInfo, HealStorageAPI}; use rustfs_heal::heal::task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType}; use std::sync::{ Arc, @@ -307,42 +277,14 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() { Ok(None) } - async fn get_object_data(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result>> { - Ok(None) - } - - async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> rustfs_heal::Result<()> { - Ok(()) - } - - async fn delete_object(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<()> { - Ok(()) - } - - async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result { - Ok(true) - } - async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { Ok(Vec::new()) } - async fn get_disk_status(&self, _endpoint: &Endpoint) -> rustfs_heal::Result { - Ok(DiskStatus::Ok) - } - - async fn format_disk(&self, _endpoint: &Endpoint) -> rustfs_heal::Result<()> { - Ok(()) - } - async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result> { Ok(None) } - async fn heal_bucket_metadata(&self, _bucket: &str) -> rustfs_heal::Result<()> { - Ok(()) - } - async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(Vec::new()) } @@ -354,14 +296,6 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() { )) } - async fn get_object_size(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { - Ok(None) - } - - async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { - Ok(None) - } - async fn heal_object( &self, _bucket: &str, @@ -388,10 +322,6 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() { Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None)) } - async fn list_objects_for_heal(&self, _bucket: &str, _prefix: &str) -> rustfs_heal::Result> { - Ok(Vec::new()) - } - async fn list_objects_for_heal_page( &self, _bucket: &str, diff --git a/crates/scanner/src/scanner_folder.rs b/crates/scanner/src/scanner_folder.rs index c573065f7..c15ed6c41 100644 --- a/crates/scanner/src/scanner_folder.rs +++ b/crates/scanner/src/scanner_folder.rs @@ -71,7 +71,6 @@ const EVENT_SCANNER_METADATA_CORRUPT: &str = "scanner_metadata_corrupt"; const EVENT_SCANNER_LIFECYCLE_ACTION: &str = "scanner_lifecycle_action"; const EVENT_SCANNER_HEAL_ADMISSION: &str = "scanner_heal_admission"; const EVENT_SCANNER_ALERT_STATE: &str = "scanner_alert_state"; -const EVENT_SCANNER_COMPAT_STATE: &str = "scanner_compat_state"; const DATA_USAGE_UPDATE_DIR_CYCLES: u32 = 16; const DATA_SCANNER_COMPACT_LEAST_OBJECT: usize = 500; @@ -92,7 +91,6 @@ const ENV_FAILED_OBJECTS_MAX: &str = "RUSTFS_DATA_USAGE_FAILED_OBJECTS_MAX"; const DEFAULT_FAILED_OBJECT_TTL_SECS: u32 = 86_400; const DEFAULT_FAILED_OBJECTS_MAX: u32 = 10_000; const DEFAULT_SCANNER_DEEP_VERIFY_COOLDOWN_SECS: u64 = 60; -const METRIC_SCANNER_INLINE_HEAL_TOTAL: &str = "rustfs_scanner_inline_heal_total"; const METRIC_SCANNER_EXCESS_OBJECT_VERSIONS_TOTAL: &str = "rustfs_scanner_excess_object_versions_total"; const METRIC_SCANNER_EXCESS_OBJECT_VERSION_SIZE_TOTAL: &str = "rustfs_scanner_excess_object_version_size_total"; const METRIC_SCANNER_EXCESS_FOLDERS_TOTAL: &str = "rustfs_scanner_excess_folders_total"; @@ -196,8 +194,6 @@ fn emit_scanner_alert_event(event_name: &str, bucket: &str, object: &str, size: } const MAX_PENDING_SCANNER_HEALS_PER_BUCKET: usize = 10_000; -static SCANNER_INLINE_HEAL_WARN_ONCE: Once = Once::new(); -static SCANNER_INLINE_HEAL_METRICS_ONCE: Once = Once::new(); static SCANNER_ALERT_METRICS_ONCE: Once = Once::new(); #[cfg(test)] @@ -251,27 +247,6 @@ fn effective_object_heal_scan_mode(heal_bitrot: bool, mod_time: Option bool { - scanner_inline_heal_enabled_from_value(std::env::var(rustfs_config::ENV_SCANNER_INLINE_HEAL_ENABLE).ok().as_deref()) -} - -fn scanner_inline_heal_enabled_from_value(value: Option<&str>) -> bool { - match value { - Some(value) => matches!(value.trim().to_ascii_lowercase().as_str(), "1" | "true" | "on" | "yes"), - None => rustfs_config::DEFAULT_SCANNER_INLINE_HEAL_ENABLE, - } -} - -fn ensure_scanner_inline_heal_metric_registered() { - SCANNER_INLINE_HEAL_METRICS_ONCE.call_once(|| { - describe_counter!( - METRIC_SCANNER_INLINE_HEAL_TOTAL, - "Total number of inline heal operations executed directly by scanner." - ); - counter!(METRIC_SCANNER_INLINE_HEAL_TOTAL).increment(0); - }); -} - fn ensure_scanner_alert_metrics_registered() { SCANNER_ALERT_METRICS_ONCE.call_once(|| { describe_counter!( @@ -505,24 +480,6 @@ fn should_alert_excessive_versions(remaining_versions: usize, cumulative_size: i (too_many_versions, too_large_versions) } -fn warn_inline_heal_compat_requested() { - if !scanner_inline_heal_enabled() { - return; - } - - SCANNER_INLINE_HEAL_WARN_ONCE.call_once(|| { - warn!( - target: "rustfs::scanner::folder", - event = EVENT_SCANNER_COMPAT_STATE, - component = LOG_COMPONENT_SCANNER, - subsystem = LOG_SUBSYSTEM_HEAL, - env = rustfs_config::ENV_SCANNER_INLINE_HEAL_ENABLE, - state = "inline_heal_rollback_unsupported", - "Scanner inline-heal rollback is unsupported; using async heal admission" - ); - }); -} - fn non_negative_i64_to_u64(value: i64) -> u64 { value.max(0) as u64 } @@ -1282,7 +1239,6 @@ impl ScannerItem { async fn heal_actions(&mut self, oi: &ObjectInfo, actual_size: i64, size_summary: &mut SizeSummary) -> i64 { if self.heal_enabled { - warn_inline_heal_compat_requested(); self.enqueue_heal(oi).await; } @@ -3231,8 +3187,6 @@ pub async fn scan_data_folder( ) -> Result { use crate::data_usage_define::DATA_USAGE_ROOT; - ensure_scanner_inline_heal_metric_registered(); - // Check that we're not trying to scan the root if cache.info.name.is_empty() || cache.info.name == DATA_USAGE_ROOT { return Err(ScannerError::Other("internal error: root scan attempted".to_string())); @@ -4325,19 +4279,6 @@ mod tests { assert!(!scanner.new_cache.info.failed_objects.contains_key("expired")); } - #[test] - fn test_scanner_inline_heal_enabled_defaults_to_false() { - assert!(!scanner_inline_heal_enabled_from_value(None)); - } - - #[test] - fn test_scanner_inline_heal_enabled_reads_env_override() { - assert!(scanner_inline_heal_enabled_from_value(Some("true"))); - assert!(scanner_inline_heal_enabled_from_value(Some("YES"))); - assert!(scanner_inline_heal_enabled_from_value(Some("1"))); - assert!(!scanner_inline_heal_enabled_from_value(Some("false"))); - } - #[test] fn test_build_object_heal_request_omits_nil_version_id() { let request = build_object_heal_request( diff --git a/rustfs/src/admin/handlers/heal.rs b/rustfs/src/admin/handlers/heal.rs index 3d98e966a..9ae59353b 100644 --- a/rustfs/src/admin/handlers/heal.rs +++ b/rustfs/src/admin/handlers/heal.rs @@ -14,10 +14,9 @@ use crate::admin::auth::{authenticate_request, validate_admin_request}; use crate::admin::router::{AdminOperation, Operation, S3Router}; -use crate::admin::runtime_sources::{app_context_from_req, object_store_from_extensions}; +use crate::admin::runtime_sources::app_context_from_req; use crate::admin::storage_api::bucket::is_reserved_or_invalid_bucket; use crate::admin::storage_api::bucket::utils::is_valid_object_prefix; -use crate::admin::storage_api::contract::heal::HealOperations as _; use crate::server::ADMIN_PREFIX; use crate::server::RemoteAddr; use crate::storage::rpc::node_service::heal::{ @@ -1219,41 +1218,6 @@ fn validate_heal_request_mode(hip: &HealInitParams) -> S3Result<()> { Ok(()) } -fn should_handle_root_heal_directly(_hip: &HealInitParams) -> bool { - false -} - -fn map_root_heal_status(heal_err: Option) -> S3Result<()> { - match heal_err { - None => Ok(()), - Some(crate::admin::storage_api::error::StorageError::NoHealRequired) => { - info!( - event = EVENT_ADMIN_RESPONSE_EMITTED, - component = LOG_COMPONENT_ADMIN_API, - subsystem = LOG_SUBSYSTEM_HEAL_ADMIN, - operation = "root_heal", - result = "success", - state = "no_heal_required", - "admin response emitted" - ); - Ok(()) - } - Some(err) => { - warn!( - event = EVENT_ADMIN_REQUEST_FAILED, - component = LOG_COMPONENT_ADMIN_API, - subsystem = LOG_SUBSYSTEM_HEAL_ADMIN, - operation = "root_heal", - result = "failed", - reason = "root_heal_failed", - error = %err, - "admin request failed" - ); - Err(s3_error!(InternalError, "root heal failed: {err}")) - } - } -} - fn json_response(status: StatusCode, body: Vec) -> S3Response<(StatusCode, Body)> { let mut headers = HeaderMap::new(); headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); @@ -1358,50 +1322,6 @@ impl Operation for HealHandler { } }; let hip = extract_heal_init_params(&bytes, &req.uri, params)?; - // The heal channel currently models bucket/object work. Root heal reuses the - // existing format-heal path directly so `/v3/heal/` is accepted intentionally. - if should_handle_root_heal_directly(&hip) { - let Some(store) = object_store_from_extensions(&req.extensions) else { - warn!( - event = EVENT_ADMIN_REQUEST_FAILED, - component = LOG_COMPONENT_ADMIN_API, - subsystem = LOG_SUBSYSTEM_HEAL_ADMIN, - operation = "root_heal", - result = "failed", - reason = "server_not_initialized", - "admin request failed" - ); - return Err(s3_error!(InternalError, "server not initialized")); - }; - - let (_, heal_err) = store.heal_format(hip.hs.dry_run).await.map_err(|e| { - warn!( - event = EVENT_ADMIN_REQUEST_FAILED, - component = LOG_COMPONENT_ADMIN_API, - subsystem = LOG_SUBSYSTEM_HEAL_ADMIN, - operation = "root_heal", - result = "failed", - reason = "heal_format_failed", - error = %e, - "admin request failed" - ); - s3_error!(InternalError, "root heal failed: {e}") - })?; - - map_root_heal_status(heal_err)?; - let body = encode_heal_start_success("root-heal".to_string(), client_address)?; - info!( - event = EVENT_ADMIN_RESPONSE_EMITTED, - component = LOG_COMPONENT_ADMIN_API, - subsystem = LOG_SUBSYSTEM_HEAL_ADMIN, - operation = "root_heal", - result = "success", - state = "started", - "admin response emitted" - ); - - return Ok(json_response(StatusCode::OK, body)); - } validate_heal_request_mode(&hip)?; let response_operation = if hip.force_stop { "cancel_heal" @@ -1614,11 +1534,9 @@ mod tests { build_replacement_recovery_status_response, encode_background_heal_status, encode_heal_control_path, encode_heal_start_success, encode_heal_task_status, execute_after_heal_control_capability, heal_channel_response_items, heal_channel_response_progress, heal_channel_response_summary, heal_control_response_id, json_response, - map_heal_response, map_root_heal_status, merge_peer_heal_statuses, peer_topology_complete, query_peer_heal_status, - query_peer_replacement_recovery_status, reject_heal_admission, should_handle_root_heal_directly, - validate_heal_request_mode, validate_heal_target, + map_heal_response, merge_peer_heal_statuses, peer_topology_complete, query_peer_heal_status, + query_peer_replacement_recovery_status, reject_heal_admission, validate_heal_request_mode, validate_heal_target, }; - use crate::admin::storage_api::error::StorageError; use crate::storage::rpc::node_service::heal::{ NodeHealProgress, NodeHealStatusSnapshot, NodeReplacementRecoveryStatusSnapshot, encode_node_replacement_recovery_status, }; @@ -2086,48 +2004,63 @@ mod tests { } #[test] - fn test_should_handle_root_heal_directly_is_disabled_for_root_start_modes() { - assert!(!should_handle_root_heal_directly(&HealInitParams::default())); - assert!(!should_handle_root_heal_directly(&HealInitParams { - force_start: true, - ..Default::default() - })); - } - - #[test] - fn test_should_handle_root_heal_directly_skips_query_cancel_and_bucket_targets() { - assert!(!should_handle_root_heal_directly(&HealInitParams { - client_token: "heal-token".to_string(), - ..Default::default() - })); - assert!(!should_handle_root_heal_directly(&HealInitParams { - force_stop: true, - ..Default::default() - })); - assert!(!should_handle_root_heal_directly(&HealInitParams { - bucket: "bucket".to_string(), - ..Default::default() - })); - assert!(!should_handle_root_heal_directly(&HealInitParams { - hs: HealOpts { - pool: Some(1), - set: Some(2), + fn test_root_heal_shapes_route_through_cluster_coordination() { + // Root heal has no direct local store path: every start shape is either + // rejected by validate_heal_request_mode or submitted to the cluster + // heal channel as an Admin-sourced request (see HealHandler::call). + let accepted_root_starts = [ + HealInitParams { + hs: HealOpts { + recursive: true, + ..Default::default() + }, ..Default::default() }, - ..Default::default() - })); - } + HealInitParams { + force_start: true, + hs: HealOpts { + recursive: true, + ..Default::default() + }, + ..Default::default() + }, + HealInitParams { + hs: HealOpts { + pool: Some(1), + set: Some(2), + ..Default::default() + }, + ..Default::default() + }, + ]; + for hip in accepted_root_starts { + validate_heal_request_mode(&hip).expect("accepted root heal start must reach cluster coordination"); + let request = build_heal_channel_request(&hip); + assert_eq!(request.bucket, "", "root heal must stay cluster-scoped"); + assert_eq!(request.source, HealRequestSource::Admin); + assert!(!request.id.is_empty(), "cluster heal requests carry a dedup id"); + } - #[test] - fn test_map_root_heal_status_allows_no_heal_required() { - map_root_heal_status(Some(StorageError::NoHealRequired)).expect("NoHealRequired should stay non-fatal"); - } - - #[test] - fn test_map_root_heal_status_rejects_fatal_errors() { - let err = map_root_heal_status(Some(StorageError::Unexpected)).expect_err("fatal status must fail"); - assert_eq!(err.code(), &S3ErrorCode::InternalError); - assert!(err.to_string().contains("root heal failed: Unexpected error")); + // Shapes that cannot start a tracked heal (plain start, bare force_start + // without recursive, bare pool) are rejected instead of falling back to + // a direct local path. + for hip in [ + HealInitParams::default(), + HealInitParams { + force_start: true, + ..Default::default() + }, + HealInitParams { + hs: HealOpts { + pool: Some(1), + ..Default::default() + }, + ..Default::default() + }, + ] { + let err = validate_heal_request_mode(&hip).expect_err("unscoped root heal start must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + } } #[test] diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index eb38e9640..318718b49 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -909,10 +909,6 @@ pub(crate) mod contract { }; } - pub(crate) mod heal { - pub(crate) use super::super::storage_contracts::HealOperations; - } - pub(crate) mod list { pub(crate) use super::super::storage_contracts::ListOperations; } diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index 352bfbd58..2503e25c7 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -2206,7 +2206,7 @@ mod tests { previous_scanner_activity_response, remove_heal_control_replay, scanner_activity_response, stop_rebalance_response, }; use crate::storage::rpc::node_service::heal::heal_topology_fingerprint; - use crate::storage::storage_api::rpc_consumer::node_service::{DiskError, HealBucketInfo, HealEndpoint}; + use crate::storage::storage_api::rpc_consumer::node_service::{DiskError, HealBucketInfo}; use crate::storage::storage_api::set_tonic_canonical_body_digest; use crate::storage::storage_api::{ Endpoint, @@ -2334,42 +2334,14 @@ mod tests { Ok(None) } - async fn get_object_data(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result>> { - Ok(None) - } - - async fn put_object_data(&self, _bucket: &str, _object: &str, _data: &[u8]) -> rustfs_heal::Result<()> { - Ok(()) - } - - async fn delete_object(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result<()> { - Ok(()) - } - - async fn verify_object_integrity(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result { - Ok(true) - } - async fn ec_decode_rebuild(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { Ok(Vec::new()) } - async fn get_disk_status(&self, _endpoint: &HealEndpoint) -> rustfs_heal::Result { - Ok(rustfs_heal::heal::storage::DiskStatus::Ok) - } - - async fn format_disk(&self, _endpoint: &HealEndpoint) -> rustfs_heal::Result<()> { - Ok(()) - } - async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result> { Ok(None) } - async fn heal_bucket_metadata(&self, _bucket: &str) -> rustfs_heal::Result<()> { - Ok(()) - } - async fn list_buckets(&self) -> rustfs_heal::Result> { Ok(Vec::new()) } @@ -2378,14 +2350,6 @@ mod tests { Ok(false) } - async fn get_object_size(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { - Ok(None) - } - - async fn get_object_checksum(&self, _bucket: &str, _object: &str) -> rustfs_heal::Result> { - Ok(None) - } - async fn heal_object( &self, _bucket: &str, @@ -2411,14 +2375,6 @@ mod tests { Ok((rustfs_madmin::heal_commands::HealResultItem::default(), None)) } - async fn list_objects_for_heal( - &self, - _bucket: &str, - _prefix: &str, - ) -> rustfs_heal::Result> { - Ok(Vec::new()) - } - async fn list_objects_for_heal_page( &self, _bucket: &str, diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 534269546..ef9a92414 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -252,8 +252,6 @@ pub(crate) mod rpc_consumer { }; pub(crate) type StorageResult = super::super::Result; - #[cfg(test)] - pub(crate) type HealEndpoint = super::super::ecstore_disk::endpoint::Endpoint; #[cfg(test)] pub(crate) type HealBucketInfo = super::super::contract::bucket::BucketInfo; From f7073d0191ab1877d8b171631013488e7e2b69a7 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 19 Aug 2026 13:43:39 +0800 Subject: [PATCH 13/27] refactor(admin): route plugin handler auth through authorize_admin_request (#6247) --- rustfs/src/admin/handlers/cluster_snapshot.rs | 49 ++++++++---- rustfs/src/admin/handlers/extensions.rs | 75 ++++++++++-------- .../src/admin/handlers/object_data_cache.rs | 44 ++++++++--- rustfs/src/admin/handlers/plugins_catalog.rs | 49 ++++++++---- .../src/admin/handlers/plugins_instances.rs | 77 +++++++++++-------- 5 files changed, 185 insertions(+), 109 deletions(-) diff --git a/rustfs/src/admin/handlers/cluster_snapshot.rs b/rustfs/src/admin/handlers/cluster_snapshot.rs index e321f76c3..3694d01a4 100644 --- a/rustfs/src/admin/handlers/cluster_snapshot.rs +++ b/rustfs/src/admin/handlers/cluster_snapshot.rs @@ -14,7 +14,7 @@ use crate::admin::storage_api::cluster::{CapabilityState, CapabilityStatus, ObservabilitySnapshot, TopologySnapshot}; use crate::admin::{ - auth::validate_admin_request, + auth::authorize_admin_request, router::{AdminOperation, Operation, S3Router}, runtime_sources::default_admin_usecase, storage_api::cluster::{ @@ -24,11 +24,10 @@ use crate::admin::{ }, system, }; -use crate::auth::{check_key_valid, get_session_token}; use crate::cluster_snapshot::{ ClusterReadOnlySnapshot, ClusterRuntimeReadinessState, ClusterRuntimeStatusSnapshot, cluster_has_actionable_pressure, }; -use crate::server::{ADMIN_PREFIX, ReadinessDegradedReason, RemoteAddr}; +use crate::server::{ADMIN_PREFIX, ReadinessDegradedReason}; use http::{HeaderMap, HeaderValue, StatusCode}; use hyper::Method; use matchit::Params; @@ -66,23 +65,15 @@ pub(crate) struct ClusterSnapshotDiscoveryResponse { pub components: Option, } +/// The pre-check keeps this endpoint's historical missing-credentials message; +/// the shared gate reports "get cred failed". async fn authorize_cluster_snapshot_request(req: &S3Request) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { + if req.credentials.is_none() { return Err(s3_error!(InvalidRequest, "authentication required")); - }; + } - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)], - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await + authorize_admin_request(req, vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)]).await?; + Ok(()) } fn build_json_response( @@ -953,6 +944,30 @@ mod tests { ); } + /// This endpoint authorizes through the shared admin gate, which reports + /// "get cred failed" for a credential-less request. The pre-check keeps the + /// message it has always returned (rustfs/backlog#1829). + #[tokio::test] + async fn cluster_snapshot_gate_keeps_its_missing_credentials_message() { + let req = s3s::S3Request { + input: s3s::Body::from(String::new()), + method: http::Method::GET, + uri: http::Uri::from_static("/rustfs/admin/v4/cluster/snapshot"), + headers: http::HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + let err = super::authorize_cluster_snapshot_request(&req) + .await + .expect_err("a request without credentials must be rejected"); + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("authentication required")); + } + #[test] fn cluster_snapshot_response_serializes_none_snapshot() { let value = serde_json::to_value(ClusterSnapshotResponse { snapshot: None }).expect("serialize response"); diff --git a/rustfs/src/admin/handlers/extensions.rs b/rustfs/src/admin/handlers/extensions.rs index bc7ec9dc5..6b3e07b26 100644 --- a/rustfs/src/admin/handlers/extensions.rs +++ b/rustfs/src/admin/handlers/extensions.rs @@ -14,7 +14,7 @@ use crate::admin::storage_api::cluster::CapabilityStatus; use crate::admin::{ - auth::validate_admin_request, + auth::authorize_admin_request, handlers::{cluster_snapshot, plugins_instances, system}, plugin_contract::{ PluginContractDomain, PluginInstanceDiagnosticCode, PluginInstanceDiagnosticCount, PluginInstanceEntry, @@ -22,8 +22,7 @@ use crate::admin::{ }, router::{AdminOperation, Operation, S3Router}, }; -use crate::auth::{check_key_valid, get_session_token}; -use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use crate::server::ADMIN_PREFIX; use http::{HeaderMap, HeaderValue, StatusCode}; use hyper::Method; use matchit::Params; @@ -183,42 +182,26 @@ fn map_extension_instance(instance: PluginInstanceEntry) -> ExtensionInstanceEnt } } +/// The pre-check keeps this endpoint's historical missing-credentials message; +/// the shared gate reports "get cred failed". async fn authorize_extension_catalog_request(req: &S3Request) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { + if req.credentials.is_none() { return Err(s3_error!(InvalidRequest, "authentication required")); - }; + } - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)], - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await + authorize_admin_request(req, vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)]).await?; + Ok(()) } +/// The pre-check keeps this endpoint's historical missing-credentials message; +/// the shared gate reports "get cred failed". async fn authorize_extension_instance_request(req: &S3Request) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { + if req.credentials.is_none() { return Err(s3_error!(InvalidRequest, "authentication required")); - }; + } - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(AdminAction::GetBucketTargetAction)], - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await + authorize_admin_request(req, vec![Action::AdminAction(AdminAction::GetBucketTargetAction)]).await?; + Ok(()) } fn build_json_response( @@ -320,6 +303,36 @@ mod tests { ); } + /// Both extension gates authorize through the shared admin gate, which reports + /// "get cred failed" for a credential-less request. The pre-check keeps the + /// message these endpoints have always returned (rustfs/backlog#1829). + #[tokio::test] + async fn extension_gates_keep_their_missing_credentials_message() { + let credential_less_request = || s3s::S3Request { + input: s3s::Body::from(String::new()), + method: http::Method::GET, + uri: http::Uri::from_static("/rustfs/admin/v4/extensions/catalog"), + headers: http::HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + for err in [ + super::authorize_extension_catalog_request(&credential_less_request()) + .await + .expect_err("a request without credentials must be rejected"), + super::authorize_extension_instance_request(&credential_less_request()) + .await + .expect_err("a request without credentials must be rejected"), + ] { + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("authentication required")); + } + } + #[test] fn builtin_ops_schemas_register_cleanly_in_runtime_registries() { let mut diagnostics_registry = rustfs_targets::OpsDiagnosticsRegistry::new(); diff --git a/rustfs/src/admin/handlers/object_data_cache.rs b/rustfs/src/admin/handlers/object_data_cache.rs index d470e8b56..6e45e7cb6 100644 --- a/rustfs/src/admin/handlers/object_data_cache.rs +++ b/rustfs/src/admin/handlers/object_data_cache.rs @@ -21,12 +21,11 @@ //! that bucket, and with `bucket`+`object` it flushes that one identity — the //! only remediation for a poisoned entry short of a node restart. -use crate::admin::auth::validate_admin_request; +use crate::admin::auth::authorize_admin_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::current_object_data_cache; use crate::app::object_data_cache::ObjectDataCacheAdapter; -use crate::auth::{check_key_valid, get_session_token}; -use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use crate::server::ADMIN_PREFIX; use http::{HeaderMap, HeaderValue}; use hyper::{Method, StatusCode}; use matchit::Params; @@ -76,17 +75,14 @@ pub fn register_object_data_cache_route(r: &mut S3Router) -> std Ok(()) } +/// The pre-check keeps these endpoints' historical missing-credentials message; +/// the shared gate reports "get cred failed". async fn authorize(req: &S3Request, action: AdminAction) -> S3Result<()> { - let Some(input_cred) = req.credentials.as_ref() else { + if req.credentials.is_none() { return Err(s3_error!(InvalidRequest, "missing credentials")); - }; - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - let remote_addr = req - .extensions - .get::>() - .and_then(|opt| opt.map(|addr| addr.0)); - validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await + } + authorize_admin_request(req, vec![Action::AdminAction(action)]).await?; + Ok(()) } fn json_response(body: &T) -> S3Result> { @@ -208,6 +204,30 @@ mod tests { assert_eq!(invalidation_outcome(&ObjectDataCacheInvalidationResult::NoOp), ("noop", 0)); } + /// These endpoints authorize through the shared admin gate, which reports + /// "get cred failed" for a credential-less request. The pre-check keeps the + /// message they have always returned (rustfs/backlog#1829). + #[tokio::test] + async fn authorize_keeps_its_missing_credentials_message() { + let req = S3Request { + input: Body::from(String::new()), + method: Method::GET, + uri: "/rustfs/admin/v3/object-data-cache/stats".parse().expect("uri should parse"), + headers: HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + let err = authorize(&req, AdminAction::ServerInfoAdminAction) + .await + .expect_err("a request without credentials must be rejected"); + assert_eq!(err.code(), &S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("missing credentials")); + } + #[test] fn stats_handler_requires_server_info_action() { // Guard the auth contract: the stats endpoint is a read, the flush diff --git a/rustfs/src/admin/handlers/plugins_catalog.rs b/rustfs/src/admin/handlers/plugins_catalog.rs index fc25e9508..8045ce0e1 100644 --- a/rustfs/src/admin/handlers/plugins_catalog.rs +++ b/rustfs/src/admin/handlers/plugins_catalog.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::admin::{ - auth::validate_admin_request, + auth::authorize_admin_request, plugin_contract::{ PluginCatalogAdminDiscovery, PluginCatalogDomainEntry, PluginCatalogEntry, PluginCatalogResponse, PluginContractDomain, PluginContractEntrypointKind, PluginContractPackaging, PluginDistributionContract, PluginRuntimeContract, @@ -21,8 +21,7 @@ use crate::admin::{ router::{AdminOperation, Operation, S3Router}, runtime_sources::default_admin_usecase, }; -use crate::auth::{check_key_valid, get_session_token}; -use crate::server::{ADMIN_PREFIX, RemoteAddr}; +use crate::server::ADMIN_PREFIX; use http::{HeaderMap, HeaderValue, StatusCode}; use hyper::Method; use matchit::Params; @@ -114,23 +113,15 @@ fn merge_catalog_descriptor(plugins: &mut HashMap<&'static str, PluginCatalogEnt } } +/// The pre-check keeps this endpoint's historical missing-credentials message; +/// the shared gate reports "get cred failed". async fn authorize_plugin_catalog_request(req: &S3Request) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { + if req.credentials.is_none() { return Err(s3_error!(InvalidRequest, "authentication required")); - }; + } - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)], - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await + authorize_admin_request(req, vec![Action::AdminAction(AdminAction::ServerInfoAdminAction)]).await?; + Ok(()) } fn build_json_response( @@ -175,6 +166,30 @@ mod tests { ); } + /// This endpoint authorizes through the shared admin gate, which reports + /// "get cred failed" for a credential-less request. The pre-check keeps the + /// message it has always returned (rustfs/backlog#1829). + #[tokio::test] + async fn plugin_catalog_gate_keeps_its_missing_credentials_message() { + let req = s3s::S3Request { + input: s3s::Body::from(String::new()), + method: http::Method::GET, + uri: http::Uri::from_static("/rustfs/admin/v4/plugins/catalog"), + headers: http::HeaderMap::new(), + extensions: http::Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + let err = super::authorize_plugin_catalog_request(&req) + .await + .expect_err("a request without credentials must be rejected"); + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("authentication required")); + } + #[test] fn plugin_catalog_contains_representative_builtin_targets() { let response = build_catalog_response(); diff --git a/rustfs/src/admin/handlers/plugins_instances.rs b/rustfs/src/admin/handlers/plugins_instances.rs index f66f5b693..548720651 100644 --- a/rustfs/src/admin/handlers/plugins_instances.rs +++ b/rustfs/src/admin/handlers/plugins_instances.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::admin::{ - auth::validate_admin_request, + auth::authorize_admin_request, handlers::audit_runtime_config::{load_server_config_from_store, remove_audit_target_config, set_audit_target_config}, handlers::notify_runtime_access::{ load_notification_config_snapshot, remove_notification_target_config, set_notification_target_config, @@ -29,10 +29,9 @@ use crate::admin::{ }, router::{AdminOperation, Operation, S3Router}, }; -use crate::auth::{check_key_valid, get_session_token}; use crate::server::{ - ADMIN_PREFIX, RemoteAddr, is_audit_module_enabled, is_notify_module_enabled, refresh_audit_module_enabled, - refresh_notify_module_enabled, refresh_persisted_module_switches_from_store, + ADMIN_PREFIX, is_audit_module_enabled, is_notify_module_enabled, refresh_audit_module_enabled, refresh_notify_module_enabled, + refresh_persisted_module_switches_from_store, }; use hyper::{Method, StatusCode}; use matchit::Params; @@ -563,42 +562,26 @@ fn plugin_instance_matches_query(instance: &PluginInstanceEntry, query: &str) -> .any(|field| field.to_ascii_lowercase().contains(&query)) } +/// The pre-check keeps this endpoint's historical missing-credentials message; +/// the shared gate reports "get cred failed". async fn authorize_plugin_instance_request(req: &S3Request) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { + if req.credentials.is_none() { return Err(s3_error!(InvalidRequest, "authentication required")); - }; + } - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(AdminAction::GetBucketTargetAction)], - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await + authorize_admin_request(req, vec![Action::AdminAction(AdminAction::GetBucketTargetAction)]).await?; + Ok(()) } +/// The pre-check keeps this endpoint's historical missing-credentials message; +/// the shared gate reports "get cred failed". async fn authorize_plugin_instance_write_request(req: &S3Request) -> S3Result<()> { - let Some(input_cred) = &req.credentials else { + if req.credentials.is_none() { return Err(s3_error!(InvalidRequest, "authentication required")); - }; + } - let (cred, owner) = - check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - - validate_admin_request( - &req.headers, - &cred, - owner, - false, - vec![Action::AdminAction(AdminAction::SetBucketTargetAction)], - req.extensions.get::>().and_then(|opt| opt.map(|a| a.0)), - ) - .await + authorize_admin_request(req, vec![Action::AdminAction(AdminAction::SetBucketTargetAction)]).await?; + Ok(()) } fn plugin_instance_mutation_block_reason( @@ -942,6 +925,36 @@ mod tests { ); } + /// Both instance gates authorize through the shared admin gate, which reports + /// "get cred failed" for a credential-less request. The pre-check keeps the + /// message these endpoints have always returned (rustfs/backlog#1829). + #[tokio::test] + async fn plugin_instance_gates_keep_their_missing_credentials_message() { + let credential_less_request = || S3Request { + input: Body::from(String::new()), + method: Method::GET, + uri: Uri::from_static("/rustfs/admin/v4/plugins/instances"), + headers: HeaderMap::new(), + extensions: Extensions::new(), + credentials: None, + region: None, + service: None, + trailing_headers: None, + }; + + for err in [ + super::authorize_plugin_instance_request(&credential_less_request()) + .await + .expect_err("a request without credentials must be rejected"), + super::authorize_plugin_instance_write_request(&credential_less_request()) + .await + .expect_err("a request without credentials must be rejected"), + ] { + assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidRequest); + assert_eq!(err.message(), Some("authentication required")); + } + } + #[test] fn configured_instance_without_runtime_appears_offline() { let config = Config(HashMap::from([( From b97fb021803308382ca31ad2a209ca9618597bdf Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 19 Aug 2026 13:51:23 +0800 Subject: [PATCH 14/27] docs(operations): land the heal/scanner MinIO audit baseline with closure results (backlog#1862) (#6258) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs(operations): land the heal/scanner MinIO audit baseline with closure results Move the comprehensive heal/scanner vs MinIO analysis (2026-08-16) into docs/operations/ so it finally enters the tree — the docs/ root is ignored by the gitignore whitelist, which is why the baseline the audit issue referenced as "to be merged with a PR" never landed. Append §9 closure results: all 14 backlog sub-issues (#1865-#1878) closed with the per-item PR map, two further misjudgment corrections (HS-17 was already implemented; HS-14's MinIO idle semantics drifted upstream), HS-12/HS-18 audit conclusions, and the registered follow-ups. Backlog issue: rustfs/backlog#1862 Co-authored-by: heihutu --- ...minio-comprehensive-analysis-2026-08-16.md | 566 ++++++++++++++++++ 1 file changed, 566 insertions(+) create mode 100644 docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md diff --git a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md new file mode 100644 index 000000000..598cd8988 --- /dev/null +++ b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md @@ -0,0 +1,566 @@ +# RustFS heal / scanner 全量功能分析与 MinIO 对标（v2） + +- 日期：2026-08-16（基于 main 分支当日代码，审计时 HEAD ≈ `a118d7e4f`） +- 范围：`crates/heal`（src 19,560 行 + tests 2,274 行）、`crates/scanner`（src 约 26,000 行 + tests）、`crates/data-usage`、`crates/ecstore` 中 heal/heal_walk/bitrot_self_verify 与 config、`crates/common/src/heal_channel.rs`、`crates/madmin`（heal/scanner wire 类型）、`rustfs/src`（startup wiring、admin handlers、集群 RPC） +- 对标基线：minio/minio master（HEAD `7aac2a2c5b`，仓库已进入维护模式，master 冻结，即最终态） +- 方法：四路并行审计（heal crate / scanner crate / ecstore 集成层 / MinIO 源码研究），关键结论逐条人工抽验（文内标注"已亲验"处为一手验证） +- 本文档取代 `docs/rustfs-heal-scanner-vs-minio-parity-assessment.md`（2026-06-15 v1）。v1 之后 heal/scanner 相关提交超过 80 个（换盘自动修复全链路、resume 状态机、usage 收敛权威化、集群级 heal 协调、ILM restore 语义等），v1 的功能清单与差距判断已全面过时；v1 中"bloom filter 缺失"等结论经本次核实为**误判**（详见 §5.4）。 + +--- + +## 0. 结论摘要 + +1. **总体判断：heal 与 scanner 的核心功能链路已经完整**。对象级 heal（quorum 仲裁 + ETag 兜底 + bitrot Deep 校验 + dangling 处理）、erasure set 深扫（per-set disk-walk 并集枚举）、按版本断点续扫（schema 化持久层 + CAS 原子发布 + 崩溃窗口补齐）、换盘自动修复（readiness 校验 + 身份围栏 + durable intent + completion proof）、scanner 周期循环（leader lock + 持久化 leader-epoch 围栏）、data usage 统计（桶级/集群级、主+备+观测快照、epoch/cycle 防回退）、ILM 全动作（expiry/transition/noncurrent/free-version/delete-marker 清理）、admin Start/Query/Cancel 协议（clientToken 语义对齐 madmin）——以上均有实现且带回归测试。两个 crate 内**没有空实现/早退桩**，异常路径全部有日志 + 指标 + 错误语义。 +2. **主要缺口集中在"入口与观测面"，而不是修复算法本身**：MRF/ECDecode/Metadata 三类任务执行体已实现但无生产触发入口（`HealEvent` 完全未接线）；`CheckAbandonedParts` 在 ecstore 三层全部 `NotImplemented`；heal/scanner trace 通道缺失；scanner 超限 S3 事件缺失；madmin 客户端方法缺失（只有 wire 类型）；heal 字节级进度/ETA 未实现。 +3. **与 v1 认知的重要修正**：bloom filter 在 MinIO 当前 master **已删除**（`.bloomcycle.bin` 只存 cycle 计数），RustFS 现状与 MinIO 一致；MinIO scanner 同样是**集群级 leader 单例**，RustFS 的 leader.lock 模型与 MinIO 同型；RustFS 的 ETag 多数派兜底仲裁已实现（`crates/ecstore/src/set_disk/ops/heal.rs:525-567,679`，已亲验），v1 担心的仲裁缺口不存在。 +4. **RustFS 在多处超出 MinIO**：remote_scanner RPC 协议（远端 peer 本地扫描而非 leader 跨网读远盘）、持久化 leader-epoch CAS 围栏、周期预算与 per-set/per-disk 并发闸、pending-heal 账本、durable replacement intent + completion proof 状态机、前台压力门控（mainline throttle）、集群 heal control coordinator + envelope 重放防护。 +5. 差距分级统计：P1（行为/运维对齐缺口）8 项，P2（完善性）9 项，P3（清理/低风险）3 项，"按设计不追平"7 项。完整清单见 §6。 + +--- + +## 1. 架构总览 + +### 1.1 RustFS 三层架构 + +RustFS 把 MinIO 在 `cmd/` 内单体的 heal/scanner 拆成三层 + 两个独立 crate： + +| 层 | 位置 | 职责 | +|---|---|---| +| 原语层 | `crates/ecstore/src/set_disk/ops/heal.rs`（~3,240 行）、`ops/heal_walk.rs`、`ops/bitrot_self_verify.rs`；上层封装 `store/heal.rs`、`store/heal_walk.rs`、`core/sets.rs` | 对象/桶/format/替换盘格式修复、disk-walk 并集枚举、写入路径 bitrot 自校验；由 `SetDisks`/`Sets`/`ECStore` 实现 `rustfs_storage_api::HealOperations` 契约（`crates/storage-api/src/object.rs:503-519`） | +| heal 运行时 | `crates/heal` | 进程级 HealManager（优先级队列/调度器/auto disk scanner/断点续传 resume）、HealChannelProcessor（消费全局 heal channel）、换盘替换恢复状态机 | +| scanner 运行时 | `crates/scanner` | 数据使用扫描、ILM 评估与入队、heal 候选生产、复制用量统计、remote scanner RPC | +| 共享协议 | `crates/common/src/heal_channel.rs`（~776 行） | Start/Query/Cancel 命令通道、`HealOpts`/`HealScanMode`/`HealRequestSource`/`HealAdmission*` 共享类型、`HealResultItem`（madmin） | +| 共享数据 | `crates/data-usage` | `DataUsageEntry/Info`、直方图、`hash_path`；scanner 产生、ecstore/admin 消费 | + +启动链路（已亲验 wiring）： + +1. `rustfs/src/startup_services.rs:93` → `init_background_service_runtime(store)`。 +2. `rustfs/src/startup_background.rs:41-81`：创建全局 heal 服务取消令牌；读 `RUSTFS_SCANNER_ENABLED`（别名 `RUSTFS_ENABLE_SCANNER`，默认 true）与 `RUSTFS_HEAL_ENABLED`（别名 `RUSTFS_ENABLE_HEAL`，默认 true）；**只要 heal 或 scanner 任一开启就初始化 heal manager**（scanner 产生的 heal 候选需要消费端；两者都关时 heal channel 不初始化，`send_heal_request` 报 "Heal channel not initialized"）。 +3. `crates/heal/src/lib.rs:142-216`：owned task 内原子初始化（caller 取消不会遗留半初始化 manager，`lib.rs:123-131`；`GLOBAL_HEAL_RUNTIME_INIT` 互斥单飞）→ `HealManager::start()` → `rustfs_common::heal_channel::init_heal_channels()` → spawn `HealChannelProcessor::start_with_receipts`。 +4. `crates/heal/src/heal/manager.rs:1301-1356` `HealManager::start`：`start_scheduler()`（`manager.rs:2394-2461`，interval 默认 10s + `Notify` 事件驱动唤醒）→ `process_unclean_shutdown()`（`manager.rs:1362-1695`）→ `enable_auto_heal`（默认 true）时 `start_auto_disk_scanner()`（`manager.rs:2464-2999`）。 +5. server ready 后 `rustfs/src/startup_lifecycle.rs:150-152`：`enable_scanner` 时 `init_data_scanner(token, store)`（`crates/scanner/src/scanner.rs:1293-1372`）。 +6. 优雅停机：`rustfs/src/startup_shutdown.rs:308` `shutdown_ahm_services()`（取消令牌）；`:414` `clear_unclean_shutdown_markers()`。 + +### 1.2 MinIO 对应结构（master 最终态） + +| MinIO 文件 | 职责 | +|---|---| +| `cmd/admin-heal-ops.go` | 手动 admin heal 序列（healSequence、clientToken/forceStart/forceStop） | +| `cmd/global-heal.go` | 常驻后台 heal 队列（newBgHealSequence，token 固定 `0000-…`，永不结束）+ `healErasureSet`（逐 set 全量对象 heal） | +| `cmd/background-heal-ops.go` | healRoutine worker 池（`_MINIO_HEAL_WORKERS`，默认 GOMAXPROCS/2）消费 healTask | +| `cmd/mrf.go` | MRF（Most Recent Fail）队列（容量 100,000），进程退出时持久化 `.minio.sys/buckets/.heal/mrf/list.bin` 并启动回放 | +| `cmd/background-newdisks-heal-ops.go` | 新盘/换盘自动 resync（monitorLocalDisksAndHeal 10s 轮询 + healFreshDisk + healingTracker） | +| `cmd/erasure-healing.go` / `erasure-healing-common.go` | 对象级 heal 核心（~800 行）、listAndHeal | +| `cmd/data-scanner.go` | scanner 循环（globalLeaderLock 集群单例）+ folderScanner + applyActions | +| `cmd/erasure.go`（nsScanner）/ `erasure-server-pool.go` | NSScanner 三层结构 | +| `cmd/bucket-lifecycle.go` | ILM 执行器（expiry/transition worker 池） | +| `cmd/xl-storage.go` | DiskInfo.Healing、CheckParts/VerifyFile、CleanAbandonedData、RenameData healing 分支 | +| `cmd/prepare-storage.go` | waitForFormatErasure 新盘启动握手 | + +### 1.3 架构级差异（设计取舍，非缺陷） + +1. **heal 队列模型**：MinIO 所有 heal（scanner 抽样/MRF/admin/新盘 resync）汇入单 channel + 固定 worker 池（新盘 resync 另有 per-drive worker 池）；RustFS 是优先级堆 + 去重合并 + 容量分级丢弃 + per-set bulkhead + 前台压力门控的多策略调度器（`manager.rs:3003-3420`）。RustFS 表达力更强，代价是"重复请求被合并"的可观测性问题（v1 已指出，现有 `HealAdmissionReceipt` canonical task_id + alias 机制回应了它，`manager.rs:1759-1846`）。 +2. **scanner 远端盘访问**：MinIO leader 通过磁盘抽象层透明读写远端节点磁盘；RustFS leader 通过 remote_scanner RPC 把扫描执行下放到远端 peer 本地进行（`crates/scanner/src/remote_scanner.rs`），只回传结果与进度心跳。两者都是集群单 leader。RustFS 方案省 leader↔远端的元数据读放大，代价是需要维护独立 RPC 协议（HMAC 逐帧认证、会话重放缓存、fence 复验，`remote_scanner.rs:52-61,405-496,1024-1065`）。 +3. **heal 状态持久化**：MinIO 用单文件 `.healing.bin`（msgp healingTracker，diskID 不匹配即重置）；RustFS 用 schema 化多文件（resume/checkpoint/intent/seal/proof 各自 CAS 发布，`resume.rs:38-61`），崩溃窗口显式补齐（`erasure_healer.rs:389-402`、`resume.rs:1027-1057`）。 +4. **写路径自保护**：MinIO 写入后靠后台 heal 收敛；RustFS 在 PutObject/CompleteMultipartUpload 提交 rename 后主动检查 `convergence.needs_heal()` 并立即入队对象 heal（`set_disk/ops/object.rs:2291-2306`、`ops/multipart.rs:2574-2589`），另有读修复 read repair（`io_primitives.rs:1040-1160`）。 + +--- + +## 2. Heal 已实现功能全景 + +### 2.1 任务类型（`HealType`，`crates/heal/src/heal/task.rs:85-111`） + +| 类型 | 语义 | 执行体 | 生产触发方 | +|---|---|---|---| +| `Cluster` | 所有 bucket 依次 heal（结构 + 可选递归对象），批内重试 ≤3 | `heal_cluster` task.rs:1420-1490 | channel：bucket 为空即 Cluster（channel.rs:576-577） | +| `Object{bucket,object,version_id}` | 单对象/版本；不存在时按 `recreate_missing` 重建或报错 | `heal_object` task.rs:855-1146 | admin、scanner、read-repair、写路径收敛、add_partial | +| `Bucket{bucket}` | 桶元数据/结构；`recursive` 再遍历全部对象版本 | `heal_bucket` task.rs:1284-1418 + `heal_bucket_objects` task.rs:1508-1698 | admin（POST /v3/heal/{bucket}）、scanner `build_bucket_heal_request` | +| `Prefix{bucket,prefix}` | 按前缀递归 | `heal_prefix` task.rs:1492-1506 | channel：`recursive && prefix` 非空（channel.rs:578-585） | +| `ErasureSet{buckets,set_disk_id}` | format 修复 + healing 标记 + 逐桶预处理 + 可恢复逐版本深扫 | `heal_erasure_set` task.rs:2158-2642 | admin（pool/set 参数）、auto disk scanner、unclean shutdown、renew_disk、durable replacement 恢复 | +| `Metadata{bucket,object}` | 仅元数据（Deep、不重建数据） | `heal_metadata` task.rs:1700-1859 | **无生产触发方**（§6 HS-01） | +| `MRF{meta_path}` | 失败路径驱动的 Deep 修复（recursive+update_parity） | `heal_mrf` task.rs:1861-1992 | **无生产触发方**（仅 `HealEvent` 可生成，未接线） | +| `ECDecode{bucket,object,version_id}` | EC 解码重建（Deep+recreate+update_parity），Urgent 优先级 | `heal_ec_decode` task.rs:1994-2156 | **无生产触发方**（仅 `HealEvent` 可生成，未接线） | + +优先级 `Low/Normal/High/Urgent`（task.rs:168-179）；状态机 `Pending/Running/Retrying/Completed/Failed/Cancelled/Timeout`（task.rs:225-241）。 + +### 2.2 触发路径全景（admin 之外） + +| 通道 | source | 优先级 | 证据 | +|---|---|---|---| +| Scanner 周期抽样（1/1024，`RUSTFS_HEAL_OBJECT_SELECT_PROB`） | Scanner | Low | `scanner_folder.rs:2117-2136`、`:1150`；`remove_corrupted=HEAL_DELETE_DANGLING(true)`、`recreate_missing=false`（`common/heal_channel.rs:24`、`scanner_folder.rs:510-511`） | +| Scanner 元数据损坏（get_size 失败分类 HealMetadata） | Scanner | High | `scanner_folder.rs:2147-2208`、`:1244-1260` | +| Scanner abandoned children（缓存有、盘上无，list_path_raw quorum 核查） | Scanner | High（桶级+对象级） | `scanner_folder.rs:2528-2792` | +| Scanner pending-heal 账本重试（heal 通道满被拒后持久化，每桶每轮 ≤128 条、上限 10k） | Scanner | 原优先级 | `scanner_folder.rs:1721-1763`、`:99-100` | +| auto disk scanner（unformatted 盘经 replacement_readiness 确认 / `runtime_state=="returning"` 盘 / durable intent 重入） | AutoHeal | Low | `manager.rs:2464-2999` | +| unclean shutdown 恢复（启动读 `unclean-shutdown` 标记 → 全部本地 set ErasureSet heal） | AutoHeal | Low | `manager.rs:1362-1695` | +| 写路径收敛（PutObject/CompleteMultipartUpload 后 `convergence.needs_heal()`） | Internal | Normal | `set_disk/ops/object.rs:2291-2306`、`ops/multipart.rs:2574-2589` | +| 部分对象 heal（add_partial） | Internal | Normal | `set_disk/ops/object.rs:5808-5825` | +| 旧数据目录清理残留 enqueue | Internal | Normal | `set_disk/core/io_primitives.rs:3880-3907` | +| 读修复（metadata_read_error / missing_shards / decode_error，TTL 去重缓存） | ReadRepair | Low | `set_disk/read.rs:407,995,1079` → `submit_read_repair_heal`（`io_primitives.rs:1105-1160`），`recreate_missing=true` | +| 盘重连遇 UnformattedDisk → send_heal_disk | AutoHeal | Normal | `set_disk/ops/locking.rs:339-347` | +| Admin API（含集群 coordinator 路由） | Admin | High | `rustfs/src/admin/handlers/heal.rs:174-212`、`:771-930` | +| 集群 RPC heal（peer 调用） | — | — | `rustfs/src/storage/rpc/node_service/heal.rs`、`ecstore/src/cluster/rpc/peer_s3_client.rs:296,1209` | + +注意：MinIO 的 MRF 通道（读路径检出 part 缺失/损坏即时投递 + 队列持久化 + shutdown 回放，`cmd/mrf.go`、`erasure-object.go:395-410,800-812`）在 RustFS 由 read-repair + 写路径收敛**部分替代**；`HealType::MRF`/`ECDecode`/`Metadata` 三个执行体没有生产入口（详见 §6 HS-01）。 + +### 2.3 对象级 heal 语义（ecstore `set_disk/ops/heal.rs`） + +流程（`heal_object_with_explicit_version_regen` :426 起）： + +1. 取对象写锁（除非 `no_lock`）；`object` 以 `/` 结尾走对象目录 heal（`heal_object_dir_locked` :1587-1717：dangling 判定 + `remove` 删除 + 缺 volume 重建）。 +2. `read_all_fileinfo` 全盘读 xl.meta，全部 not-found 视为已删除返回。 +3. **quorum 仲裁 + ETag 兜底**（已亲验）：`list_online_disks` 以 mod-time quorum 为准；quorum 失效时回退 ETag 多数派仲裁（`:525-567` `filter_by_etag`/`quorum_etag`）；`pick_valid_fileinfo` 选 canonical 元数据；"meta 坏盘数 > parity" 的 cannotHeal 判定在 ETag 全盘一致时豁免（`:679`）。与 MinIO `filterDisksByETag` 双仲裁一致。 +4. `disks_with_all_parts`（:562-572）按 `scan_mode` 校验 part：**Normal 仅 stat（CheckParts 语义），Deep 做全量 bitrot 校验（VerifyFile 语义）**；Normal 扫描检出 `FileCorrupt` 自动升级 Deep 重试一次（`:2022-2031`，与 MinIO erasure-healing.go:1101-1106 同型）；无 parity 对象（EC:0）bitrot 失败判不可恢复（`:700-726`）。 +5. `should_heal_object_on_disk`（:606-650）逐盘分类 missing/corrupt/offline/outdated → 重建：per-part bitrot reader/writer（用 per-part checksum + 算法）、写临时卷后 rename 提交（`HEAL_RENAME_INCOMPLETE` 重试语义 :24）；dangling 删除安全检查 `dangling_delete_safety`（:1488）；**孤儿数据目录回收 `reclaim_orphan_data_dirs_best_effort`（:1428）**——这部分覆盖了 MinIO `CleanAbandonedData` 的主场景（但无独立 `CheckAbandonedParts` API，见 §6 HS-02）。 +6. 版本化对象：枚举"每个版本"（`storage.rs:1494-1530`）；delete-marker 路径由 `latest_meta.deleted` 决定（`storage.rs:262-277` 注释）；回归测试 `tests/heal_b5_versioned_regression_test.rs:282,334`。 +7. 显式版本重建 `try_regenerate_explicit_version_meta`（:1318）；transitioned 对象本地残留清理。 +8. 写入路径另有 shard 级 bitrot 自校验 `verify_written_bitrot_shards`（`ops/bitrot_self_verify.rs:45-129`，HighwayHash256S，最终 rename 前校验刚写出的 shard，服务 EC:0 无 parity 场景）——**注意这不是后台 bitrot 巡检**；后台巡检由 scanner bitrot_cycle 驱动 Deep heal 承担。 + +heal crate 侧包装（`task.rs:855-1146`）：存在性检查（瞬时错误转 `TransientSkip` 不误判失败 :551-569）；scanner 合成目录规范化（:1148-1180）；`recreate_missing` 重建（:1183-1282）；data-usage-cache 对象锁超时豁免（:571-653）；not-found → treated_as_deleted 成功（:1012-1029）；结果 `HealResultItem` 保留至多 1024 条 + truncated 标志（:50,845-852）。 + +递归遍历（`heal_bucket_objects` task.rs:1508-1698）：分页枚举全部版本含 delete marker、瞬时错误指数退避重试 ≤3（2^n + 抖动 :620-627）、失败样本日志截断 ≤5 条、聚合 `BatchHealFailure`。 + +### 2.4 erasure set heal 与断点续扫 + +`heal_erasure_set`（task.rs:2158-2642）四阶段（4 步进度跟踪）： + +1. **替换意图与恢复盘选择**（仅 AutoHeal + heal_endpoints 非空）：复用 durable intent 所在盘 / 排除目标端点选幸存盘；已完成代（CleanupPending）幂等收尾。 +2. **格式修复**：`heal_replacement_format(dry_run, pool, set, targets)`（`storage.rs:1372-1384`，trait 默认实现 fail-closed）；逐目标盘结果必须全 ok（`erasure_healer.rs:97-102`）+ 身份围栏复核（task.rs:2410-2420）。 +3. **healing 标记**：对目标盘写 owner CAS 标记 `{set_disk_id}:{task_id}`（`mod.rs:80-229`，CAS + 回滚 + 并发唯一 owner），使 `DiskInfo.healing` 为真（已亲验赋值链 `set_disk/mod.rs:4988`）。 +4. **逐桶预处理 + 可恢复深扫**：`ErasureSetHealer::heal_erasure_set`（`erasure_healer.rs:242-278`）。 + +`ErasureSetHealer` 扫描细节（对标 MinIO `healErasureSet`，`heal_walk.rs:15-23` 模块注释明确引用 MinIO `global-heal.go` 的 listPathRaw + objQuorum=1 + mergeXLV2Versions）： + +- **枚举器选择（backlog#920）**：Deep 或 AutoHeal → per-set **disk-walk 并集枚举** `list_versions_for_heal_page_disk_walk`（"任意盘上存在"即 sub-quorum 可重建；`storage.rs:1559-1644`，页界 1000 对象/10,000 版本，`dw1:` cursor）；普通请求走 read-quorum `list_object_versions`。 +- **续扫游标**：权威 cursor 为 opaque continuation token（`v1:`=marker JSON、`dw1:`=disk-walk key，两命名空间互斥防误读，`storage.rs:81-260`）；每完成一页先持久化 cursor 再清 dedup 集合（`erasure_healer.rs:922-927`）。 +- **页内并发**：FuturesUnordered + Semaphore，默认 `RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY=8`，Deep/AutoHeal 强制 1（`erasure_healer.rs:105-142`）。 +- **per-version dedup**：`compose_key` 长度前缀注入编码（`resume.rs:281-288`）。 +- **错误分类**：真缺席（FileNotFound 等）→ Absent（计成功）；基础设施瞬时（quorum/DiskNotFound/SlowDown 等）→ Transient（计 skipped）；其余 Failed（`erasure_healer.rs:148-182`，注释引 backlog#856/#799 B7：离线盘不得记 healed/absent）。 +- **防死循环**：空页 truncated 或页尾版本身份不前进即中止（:933-949）。 +- **完成判定**：failed/skipped/failed_buckets 任一 >0 不标记完成，`schedule_retry()` 复位 resume+checkpoint 两层（:561-626，backlog#855/B6/#1033：skip 轮不得标记完成）。 +- **替换盘提交证据**：目标端点物理回读 `replacement_targets_have_version`（`ops/heal.rs:340-412`），未确认 → transient skip。 + +### 2.5 换盘自动修复（replacement recovery） + +- **识别**（`replacement_readiness.rs:25-73`）：`replacement_mount_lease_root()` 存在、canonicalize 成功、是挂载点、物理设备 id 非空、与根设备不相交、不与兄弟盘共享物理设备（Linux 用 /proc/self/mountinfo mount-id+dev+ino）。非 root 挂载检查有回归测试（`manager.rs:3549`）。 +- **状态机**（`resume.rs:63-73`）：`Intent → Rebuilding →（写 proof）Verified → CleanupPending → 清理`；`Abandoned` 终态；跨状态迁移先写持久层再变更（`save_state_strict`）。 +- **持久化**（`resume.rs:38-61`，schema ResumeState=5/Checkpoint=5/proof=1）：`{task_id}_ahm_resume_state.json`、`_ahm_checkpoint.json`、`buckets/ahm-replacement/` 命名空间下 intent/seal/completion_proof；torn write + 无 seal 可识别并原子重建（:1316-1338）；CAS 发布、拒绝覆盖并发有效 proof（:1512-1585）。 +- **恢复**：unclean shutdown 与周期扫描都从幸存盘恢复未完成/待清理替换代（`manager.rs:1435-1640,2663-2815`）；多代冲突/校验失败 → 冻结该 set（`replacement_recovery_blocked_sets`，`manager.rs:69-87,2782-2815`）。 +- **对外快照**：`current_replacement_recovery_snapshot`（`lib.rs:262-333`）合并本地幸存盘记录，冲突 → Unknown/非 definitive；admin `GET /v4/heal/replacement-recovery`。 + +### 2.6 调度器（manager.rs） + +- 优先级堆 + 同优先级 FIFO（:148-191,330-347）；dedup key 按类型（:469-506）；入队三态查重 active→queued→retrying（:1759-1785）；重复默认 Merged 并返回 canonical task_id（`HealAdmissionReceipt`，:1821-1846）+ client token alias（:1219-1246）。 +- 容量：队列满时 best-effort 来源（Scanner/AutoHeal/ReadRepair）或低优先级被 Dropped(QueueFull)；Admin/Internal 可驱逐低优先级排队项（`push_displacing_lower_priority` :353-396）；80%/95% 压力分级（:885-909）。 +- 并发：全局 `max_concurrent_heals`（默认 4）+ per-set bulkhead `max_concurrent_per_set`（默认 1）（:3040-3073,3434-3447）。 +- 前台压力门控 mainline throttle：前台读/写 permit 利用率 ≥80% 时延迟 best-effort 任务（:919-1009,2999-3020）。 +- 超时：任务级聚合超时（默认 300s），跨重试保留剩余预算（task.rs:444-451，PR #6101）。 +- 可恢复重试：`is_recoverable_heal()`（error.rs:83-136）≤3 次、2^n 退避封顶 30s；retry 在独立 backoff task 中持有所有权（:3235-3382）。 +- 完成态保留 10 分钟供查询（:42）。 + +### 2.7 Admin API 与集群协调 + +- 路由（`rustfs/src/admin/handlers/heal.rs:174-212`）：`POST /rustfs/admin/v3/heal/`、`/heal/{bucket}`、`/heal/{bucket}/{prefix}`（同一 POST 按 query `clientToken/forceStart/forceStop` 区分 start/query/cancel，与 mc admin heal 语义对齐）；`POST /v3/background-heal/status`；`GET /v4/heal/replacement-recovery`。权限 `HealAdminAction`（route_policy.rs:334-341）。 +- 集群协调（heal.rs:771-930 + `node_service.rs:514-606`）：`heal_topology_fingerprint` + 按拓扑确定性选 coordinator 节点 + coordinator epoch；envelope 校验 + SHA256 digest 重放缓防重放；coordinator 非本机走 peer gRPC `heal_control`；`probe_heal_control` 能力探测（滚动升级场景）。 +- 请求：body 为 `HealOpts`（`recursive/dryRun/remove/recreate/scanMode(0/1/2)/updateParity/nolock/pool/set`，serde camelCase，与 madmin.HealOpts 字段对齐）；根 heal start 需 `recursive=true` 或 `pool+set` 成对；body 上限 1MB。 +- 响应：`HealStartSuccess{clientToken, clientAddress, startTime}`；`HealTaskStatus{summary, detail, startTime, settings, items, truncated, progress}`（summary ∈ running/finished/stopped/notFound）；`BackgroundHealStatus`（bitrot 起始时间/周期/当前模式 + `disabled/uninitialized/idle/active/degraded` 状态——peer 不可达显式 degraded 不冒充 idle，issue #5850 + `healOperations` 按优先级×来源矩阵 + 集群进度）。 +- `HealResultItem`/`HealDriveInfo`/`HealItemType`/DriveState 枚举与 madmin JSON 兼容（`crates/madmin/src/heal_commands.rs:19-65`）。 +- 状态 payload 超 8MiB 对折截断（channel.rs:37,73-104）；path-token 校验（错误 token 拒绝，空 path 仅匹配 Cluster）。 + +### 2.8 heal 指标与日志 + +指标：`rustfs_heal_admission_total{source,result,reason,context}`、`rustfs_heal_task_start_total`、`rustfs_heal_task_running{type,set}`、`rustfs_heal_queue_delay_seconds`、`rustfs_heal_scheduler_skip_total`、`rustfs_heal_mainline_throttle_total`、`rustfs_heal_page_concurrency_current{set}`、`rustfs_heal_candidate_enqueue/merge/drop/priority_reject_total`、`rustfs_heal_read_repair_dedup_total{reason}` 等。日志全部结构化 event style（PR #5720）；per-object 日志降级防风暴（`demote_to_debug_when!`，#5716/#5719/#5727）。 + +--- + +## 3. Scanner 已实现功能全景 + +### 3.1 循环、leader、立即触发 + +- **集群单 leader**：分布式 ns 写锁 `leader.lock`（`scanner.rs:3156-3207`，超时默认 5s）+ **持久化 leader-epoch CAS 围栏**：leader 用 ETag 前置条件向 `.bloomcycle.bin` 写 `RSCYC001` 编码的 (cycle, leader_epoch)（`scanner.rs:118,1850-1861,2177-2334`）；usage 快照再打 epoch fence（:2087-2153）。锁丢失 → 取消当前周期，30s 收敛（:108-111,2623-2642）。 +- 抢锁后立即执行一轮；周期 = `RUSTFS_SCANNER_CYCLE` > config cycle > start_delay > 部署默认 > 速度档位（±10% 抖动、下限 1s）。 +- **clean-idle 指数退避**：连续完整无脏周期间隔 ×2（封顶 24h；bitrot 周期压缩上限；桶有 lifecycle/replication 活动规则禁用，:383-456,1382-1512）。 +- **superseded/deferred 退避**：5s 起指数退避封顶 30min（:105-106,3432-3438）；维护探测失败独立退避（:459-505）。 +- **立即唤醒**：① dirty-usage 快路径——写路径 put/delete/multipart/bucket 操作调用 `record_dirty_usage_bucket`（`scanner_io.rs:222-235`；调用点 `rustfs/src/app/object_usecase.rs:6221` 等），自增 generation 并 Notify 唤醒 leader，脏桶优先排队（`scanner_io.rs:462-488`）；② 维护配置变更（lifecycle/replication 设置时 `record_scanner_maintenance_change`）；③ 运行时配置热更 generation+Notify；④ 集群活动快照变化。 +- **集群协调**：`probe_scanner_activity` 汇集本机+peer 的 `ScannerNodeActivity`（instance_id/namespace_generation/maintenance_generation/protocol_version/topology_digest/data_movement_active/dirty usage），拓扑摘要覆盖 pools/sets/drives URL，协议版本不齐拒绝共享缓存锁（`scanner.rs:970-1068`）；**数据迁移（rebalance/decommission）期间推迟周期**（`scanner_io.rs:2226-2374`）；周期结束逐 peer RPC 确认 dirty-usage ack（`scanner.rs:2925-2952`）。 + +### 3.2 遍历模型 + +- 主遍历是**全量目录 walk**（tokio::fs::read_dir 递归，`scanner_folder.rs:1915-2234`），不走 metacache；metacache/`list_path_raw` 仅用于 abandoned children 跨盘核查（:2528-2792）。 +- 三级并发：leader → per-set（信号量默认 4）→ per-disk 桶扫描（默认 4）→ 单盘递归；每桶每 set 缓存锁 `.scanner-cycle.lock.pool-N.set-M`（锁丢失取消该桶扫描，锁竞争重排队）；每盘单扫描准入（本地盘也走信号量，`scanner_io.rs:3246-3274`）。 +- 桶顺序：shuffle 后按 dirty → 未缓存 → 已缓存重排（`scanner_io.rs:2947-2949,462-488`）；目录内按名字排序 + resume 提示旋转（`scanner_folder.rs:333-359`）。 +- **断点续扫**：`DataUsageScanCheckpoint{version,resume_after,reason}` 持久于缓存 info（`data_usage_define.rs:68,293-307`）；预算耗尽/取消写入，恢复有 Used/Stale/NoHint 指标；续扫单位是目录（无跨周期对象级分页）。 +- erasure 语义：发现 `xl.meta` 即对象边界不下钻；UUID data-dir 候选最多探测 64 entry；有数据无元数据 → 记 failed + 高优 heal；symlink 目录忽略/环跳过。 +- 协作让出：每 N 对象（默认 128）`yield_now`。 + +### 3.3 大桶跳过策略（对标 MinIO compaction） + +1. 缓存当前性复用：桶与扫描计划未变（name/source/snapshot_complete/plan digest/next_cycle/leader_epoch/cache_key_format 全匹配）整桶跳过（`scanner_io.rs:1062-1109`）。 +2. compacted 目录 16 周期轮换窗口：`hash mod (next_cycle, 16)` 命中才重扫，否则从旧缓存拷贝（`scanner_folder.rs:74,2429-2442`）。 +3. compaction 阈值：子项 <500 或纯对象叶子压缩为单 entry；子文件夹 ≥2500（根 10000）预压缩；children ≥10000 归约（:75-78,2314-2340,2846-2887）。 +4. 失败对象 TTL 跳过：86400s/最多 10000 条（:88-91,1354-1381）。 + +与 MinIO master 对比：MinIO 的跳过策略同样是 hash-mod-16 周期 + compaction 阈值树（500/10000/2500），**bloom filter 已从 master 删除**。RustFS 的常量与结构与 MinIO 现状同源（MinIO 未采用跨盘 dirty-generation 优先，RustFS 额外多两层跳过——plan digest 与缓存当前性校验）。 + +### 3.4 data usage 统计 + +- 维度：每目录 entry（size/objects/versions/delete_markers/大小直方图/版本直方图/复制统计/failed_objects/per-tier stats/children/compacted，`data-usage/src/data_usage.rs:661-679`）；每对象 SizeSummary（含 per-ARN 复制目标统计、tier 统计，tier 分类：transitioned 完成记入其 tier 否则按 storage class，free version 不计）；桶级 `BucketUsageInfo`；集群级 `DataUsageInfo`（含 scanner_cycle/scanner_epoch 围栏 + usage_snapshot_complete）。 +- 存储：每桶每 set `{bucket}/.usage-cache.bin`（主 + `.bkp` 备份 + CAS 重试）；权威集群快照 `buckets/data-usage/data-usage.json`（每 10 周期同步 `.bkp`，legacy 路径兼容）；陈旧快照拒绝写入（epoch/cycle/last_update 三重判定）；被竞争 superseded 的观测快照另存 `data-usage-observed.json`。 +- 消费：`replace_bucket_usage_memory_from_info` 刷新桶用量内存 + 两层缓存失效（`scanner.rs:4142-4152`）→ bucket stats/quota/admin account_info/system；写路径内存实时叠加 overlay；启动读快照判断冷缓存跳过启动延迟。 +- 未完成 multipart 不参与统计（与 MinIO 一致，MinIO 也不扫 multipart 桶）。 + +### 3.5 ILM 集成 + +- 每对象 `ScannerItem::apply_actions`（`scanner_folder.rs:747-1032`）：`Evaluator::new(lifecycle).with_lock_retention(...).with_replication_config(...).eval()` 批量评估。 +- 已实现动作（IlmAction 全集，`common/src/metrics.rs:34-45`）：expiry 删除（Delete/DeleteRestored/DeleteRestoredVersion）、全版本删除（DeleteAllVersions/DelMarkerDeleteAllVersions，处理后停止后续版本）、transition（Transition/TransitionVersion，tier 列表运行时读取）、noncurrent 批量（DeleteVersionAction → `enqueue_by_newer_noncurrent`）、free-version 清理（`enqueue_free_version`）、object-lock retention 约束。**与 MinIO 的 9 个 ILM 动作一一对应**。 +- 执行模型：scanner 是"发现与入队"角色（expiry 队列/transition 队列在 ecstore `bucket_lifecycle_ops.rs`），动作由 worker 池消费——与 MinIO globalExpiryState/globalTransitionState 同型。 +- AbortIncompleteMultipartUpload 不在 scanner/ILM 内执行（MinIO 同样不在：`internal/bucket/lifecycle/rule.go` 有 FIXME，实际由 `erasureSets.cleanupStaleUploads` 全局例程承担）；RustFS 由 ecstore 独立后台任务 `init_background_stale_multipart_upload_cleanup`（`bucket_lifecycle_ops.rs:3289-3320`）+ 桶删除时 on-demand。 +- 集成测试覆盖：transition+restore、free-version、noncurrent、delete-marker、0-day、后台扫描过期（`scanner/tests/lifecycle_integration_test.rs:1071-2095`）。 + +### 3.6 heal 候选生产（scanner 侧） + +- 抽样：`hash mod_alt(next_cycle/prob_div, 1024/prob_div)`，进入 compacted 分支重扫时 prob_div=16 等效概率 ×16（与 MinIO 同款补偿，`scanner_folder.rs:125-127,2117-2122`）。 +- deep/normal：周期级 `get_cycle_scan_mode`（bitrot_cycle 默认 30d，`scanner.rs:1626-1657`）→ 对象级带 `HealScanMode::Deep`；新鲜对象（60s 内修改）降级 Normal（:146-155）；状态持久 `.background-heal.json`（`BackgroundHealInfo{bitrot_start_time,bitrot_start_cycle,current_scan_mode}`，与 MinIO 同路径同结构）。 +- scanner 只入队不内联执行（内联 heal 已移除，兼容旗标仅告警，`scanner_folder.rs:411-427`）；`HealScanMode::Deep` 只是标记，bitrot 校验读发生在 heal 消费端（ecstore Deep 路径）。 +- 元数据损坏 → 高优 heal（`classify_get_size_failure` → HealMetadata）；abandoned children → list_path_raw quorum 核查 + 桶级/对象级高优 heal；healing 盘粘性跳过（`should_heal` :1628-1648）。 +- pending-heal 账本：heal 通道满被拒持久化到缓存 info，下轮重试。 +- 复制 heal：`queue_replication_heal` → replication 队列（走 replication 通道而非 heal channel）；per-ARN 复制用量统计。 + +### 3.7 remote_scanner RPC 协议（RustFS 特有） + +请求 ≤16KB msgpack（version/request_id/server_epoch/session_id/session_sequence/bucket/next_cycle/leader_epoch/scan_plan_digest/skip_healing/scan_mode/budget）；帧 ≤2MB、HMAC-SHA256 逐帧认证（域 `rustfs-ns-scanner-frame-v3`）；进度心跳 1s（预算模式 250ms）；阶段播报 Scanning→Persisting；RPC 生命周期上限 24h、断连宽限 2min；防重放 session+sequence 缓存（容量 65536）；服务端校验 leader fence 与持久化 cycle 一致 + 每 5s fence 复验；结果 Complete/Partial/NamespaceNotFound/CycleAhead；不支持 v4 协议的远端盘回退 leader 本地扫描（`remote_scanner.rs` 全文件；`scanner_io.rs:2750-2812`）。 + +### 3.8 限速/预算/热更/观测 + +- DynamicSleeper 比例退避（速度档 fastest/fast/default/slow/slowest，同 MinIO 五档参数）；idle_mode 总闸；前台 S3 读流量每请求 10ms 封顶 250ms 额外退避。 +- 周期预算 ScannerCycleBudget：max_duration/max_objects/max_directories（默认 0=不限），partial 周期仍推进 cycle 计数。 +- runtime_config 三层来源（env > config > default）逐字段来源标记（Env/Config/ScannerCompatConfig/Default），admin `PUT /v3/config` 热更 → generation+Notify 即时生效；`GET /v3/scanner/status` 返回 enabled/freshness(fresh/stale/unknown)/metrics/cycle_schedule/runtime_config；`GET /v3/ilm/expiry/status` 返回 expiry 队列/worker/missed/blocked。 +- 指标：leader lock、周期 complete/partial/deferred/superseded、versions scanned、per-source（Usage/Lifecycle/BucketReplication/SiteReplication/Heal/Bitrot/Alerts）checked/executed/queued/missed、checkpoint set/used/stale、当前路径（per-disk+bucket 实时）、缓存 save 系列、并发系列、告警（excess versions/version size/folders）。 + +--- + +## 4. 与 MinIO 逐项对标 + +### 4.1 heal 触发通道对照 + +| MinIO 通道 | RustFS 对应 | 状态 | +|---|---|---| +| A. 手动 admin heal（healSequence，clientToken/forceStart/forceStop） | heal channel Start/Query/Cancel + 集群 coordinator + envelope 重放防护 | ✅ 等价且增强（集群路由）；序列语义差异见 §6 HS-06 | +| B. 常驻后台 heal 队列（newBgHealSequence + healRoutine worker 池） | HealManager 常驻调度器 + 优先级队列 + bulkhead | ✅ 等价且增强 | +| C. 新盘/换盘自动 resync（monitorLocalDisksAndHeal 10s + healFreshDisk + healingTracker + waitForFormatErasure 握手） | auto disk scanner（10s）+ replacement_readiness + durable intent/proof 状态机 + heal_replacement_format | ✅ 等价且增强（identity fence + completion proof；MinIO 的 tracker 面向对外可见性更强，见 §6 HS-07） | +| D. MRF（队列 100k + 持久化 list.bin + shutdown 回放 + 读路径 corrupt 投递） | read-repair（Low+TTL 去重）+ 写路径 convergence heal 部分承担；`HealType::MRF` 执行体无生产入口 | ⚠️ 部分等价（§6 HS-01） | +| E. Scanner 抽样 heal（1/1024 + compacted ×16 补偿）+ abandoned children | 同款抽样 + ×16 补偿 + abandoned children + pending-heal 账本 | ✅ 等价且增强（账本） | +| F. 读路径内联触发 → MRF（GetObject part 缺失/损坏、元数据重建 missingBlocks>0） | read repair（missing_shards/decode_error/metadata_read_error 三入口） | ✅ 等价（入 heal 队列而非 MRF 队列） | + +### 4.2 对象级 heal 语义对照 + +| 特性 | MinIO | RustFS | 状态 | +|---|---|---|---| +| mod-time quorum 仲裁 | listOnlineDisks | 同 | ✅ | +| ETag 多数派兜底（时钟漂移） | filterDisksByETag | `filter_by_etag`/`quorum_etag`（heal.rs:525-567） | ✅ 已亲验 | +| cannotHeal 的 ETag 豁免 | ETag 全一致豁免重试 | heal.rs:679 | ✅ | +| Normal=CheckParts（stat）/ Deep=VerifyFile（bitrot） | 是 | `disks_with_all_parts` 按 scan_mode（ops/heal.rs:562-572,978-1024） | ✅ | +| Normal 检出 corrupt 自动升 Deep 重试一次 | erasure-healing.go:1101-1106 | ops/heal.rs:2022-2031 | ✅ | +| dangling 判定（not-found > parity）+ 删除审计 | isObjectDangling/deleteIfDangling | `dangling_delete_safety`（:1488）+ scanner HEAL_DELETE_DANGLING | ✅（审计 tags 细节有差异） | +| 孤儿 data-dir/inline 清理（CleanAbandonedData） | CheckAbandonedParts（scanner 抽中 + admin Remove 时显式调用） | heal 路径内 `reclaim_orphan_data_dirs_best_effort`（:1428）；独立 API 三层 NotImplemented | ⚠️ 部分等价（§6 HS-02） | +| 版本化/delete-marker heal | HealObject versionID；nullVersionID 特判 | 逐版本枚举 + delete-marker latest heal（B5 回归） | ✅ | +| 对象级 healing 元数据标记（x-minio-healing，RenameData 跳过版本清理） | 有 | 无对象级标记；依赖盘级 healing.bin + NSLock + rename 语义 | ⚠️ 评估项（§6 HS-12） | +| Distribution/Index 一致性三处防线 | 有（manual modification 拒绝） | 目标盘格式结果全 ok 校验 + 身份围栏 | ✅（粒度不同） | +| 无 parity（EC:0）对象 | bitrot 不可恢复处理 | 判不可恢复（:700-726）+ 写入自校验 | ✅ 增强（写路径自校验） | +| 三层分布不一致拒绝 heal | 有 | heal_walk 归一化 + 页界防御 | ✅（实现方式不同） | +| multipart 孤儿对账 | CheckAbandonedParts 承担 | 显式 NotImplemented（由 lifecycle 清理承担） | ⚠️ §6 HS-02 | +| suspended/decommissioned pool 处理 | IsSuspended 跳过 | deferral 语义（store/heal.rs:192-207，PR #5876） | ✅ | +| heal 与并发删除互斥 | NSLock + healing 标记 | NSLock + 写锁 | ✅ | + +### 4.3 新盘 resync 对照 + +| MinIO | RustFS | 状态 | +|---|---|---| +| waitForFormatErasure 四类可恢复错误无限等待握手 | startup 盘解析 + renew_disk 重连路径 | ✅（模型不同：RustFS 不在启动时阻塞等待 format） | +| HealFormat NSLock + errNoHealRequired + refFormat 不一致拒绝 | `heal_format`/`heal_replacement_format` fail-closed + 目标槽位限定（PR #1787 语义） | ✅ 增强 | +| per (pool,set) 分布式锁防并发 resync | set 级队列去重 + bulkhead（manager.rs:2854-2889） | ✅ | +| 全新集群检测（待 heal 盘数==总盘数不触发） | replacement_readiness（独立挂载点/物理设备校验，非 root） | ✅ 增强 | +| healingTracker（.healing.bin：Bytes/Items 计数、QueuedBuckets/HealedBuckets、Resume 快照、RetryAttempts ≤4、HealID 联动、diskID 变更重置） | resume/checkpoint schema 化持久层 + durable intent/proof（per-task 文件，CAS） | ✅ 等价且增强（崩溃窗口补齐）；但**对外快照可见性**弱于 MinIO（§6 HS-07） | +| 跳过 heal 开始后新写入版本（ModTime > Started） | 无同款过滤 | ⚠️ §6 HS-13 | +| 跳过 ILM 已过期版本（filterLifecycle） | 无同款过滤 | ⚠️ §6 HS-13 | +| worker 数 max(GOMAXPROCS,NR)/4 下限 4，heal:drive_workers 覆盖 | 页内并发 8（Deep/AutoHeal 强制 1）+ per-set bulkhead | ✅（参数模型不同） | +| 每 entry waitForLowHTTPReq 让路 | mainline throttle（前台利用率门控） | ✅ 增强 | +| heal 范围含 `.minio.sys/config`、`.minio.sys/buckets` 两个伪桶；最新桶优先 | ErasureSet 任务逐 bucket 预处理（含 meta bucket 语义由 heal_bucket 承担） | ✅（顺序无"最新优先"） | +| 失败整体重试 ≤4 次（resetHealing + errRetryHealing） | schedule_retry 复位双层 + 可恢复重试 ≤3 | ✅ | + +### 4.4 scanner 对照 + +| MinIO | RustFS | 状态 | +|---|---|---| +| 集群单 leader（globalLeaderLock） | leader.lock + 持久化 leader-epoch CAS 围栏 | ✅ 增强（epoch 围栏防脑裂，MinIO 无持久化 epoch） | +| `.bloomcycle.bin` 只存 cycle（bloom 已删除） | 同路径存 cycle+leader_epoch（RSCYC001） | ✅ 对齐（v1 误判已修正） | +| folderScanner hash-mod-16 + compaction（500/10000/2500） | 同款常量 + plan digest + 缓存当前性校验 + dirty 优先 | ✅ 增强 | +| 每盘扫描并行 ≤GOMAXPROCS；healing 盘排除 | per-set/per-disk 信号量 + healing 盘粘性跳过 | ✅ | +| scannerSleeper（factor 2/max 1s，speed 档热更） | DynamicSleeper 同款 + idle_mode + 前台读退避 | ✅ 增强 | +| idle 语义：`scanner:idle_speed=on`（空闲时段才节流，忙时全速） | `RUSTFS_SCANNER_IDLE_MODE=true`（启用限速总闸） | ⚠️ 语义方向相反，§6 HS-14 | +| applyActions 顺序（heal→ILM→复制→告警） | apply_actions 同序（heal 候选→ILM→复制 heal→告警） | ✅ | +| ILM 9 动作 + 批量评估 + DeletePrefixObject 优化 | 同 9 动作 + 批量评估 + expiry 队列 | ✅（DeleteAllVersions 是否单调用优化未逐行核） | +| abandoned children（listPathRaw minDisks=N/2 发现漏写盘） | list_path_raw + quorum 核查 + 高优 heal | ✅ | +| incomplete multipart 独立例程（6h 间隔/24h 过期，rename 进 .trash） | ecstore 独立后台任务（可配间隔/过期） | ✅（trash 二段清理细节差异，§6 HS-18） | +| usage 维度（size/objects/versions/DM/直方图/复制/tier/bucket 级） | 全覆盖 + 集群快照三重防回退 | ✅ 增强 | +| prefix 级 usage（loadPrefixUsageFromBackend，console 消费） | 缓存内有目录树但仅 flatten 桶级 | ❌ §6 HS-08 | +| 超限事件 s3:ObjectManyVersions/LargeVersions/PrefixManyFolders + 审计 | 仅指标 alert_excess_*（默认 100/1TiB/65538 vs MinIO 100/1TB/50000） | ⚠️ §6 HS-04/HS-17 | +| scanner 指标 v3（bucket_scans/directories/objects/versions/last_activity） | rustfs_scanner_* 全套 + freshness | ✅（命名体系不同） | +| TraceScanner / realtime metrics（mc admin scanner status/trace） | 无 trace 通道；/v3/scanner/status 自有结构 | ⚠️ §6 HS-03 | + +### 4.5 admin/CLI/API 面对照 + +| MinIO | RustFS | 状态 | +|---|---|---| +| `POST /minio/admin/v3/heal/...` start/status/cancel | `POST /rustfs/admin/v3/heal/...` 同三态 | ✅（路径前缀不同属预期） | +| `HealStartSuccess`/`HealTaskStatus`/`HealResultItem`/DriveState | 同名字段 JSON 兼容 | ✅ | +| `POST /v3/background-heal/status`（BgHealState 聚合） | 同路径 + degraded 语义 + operations 矩阵 | ✅ 增强（MRF per-endpoint 子状态无，因无 MRF） | +| `GET /v3/healthinfo` 每 drive `HealInfo *HealingDisk` | 无同款 healthinfo heal 字段（replacement-recovery v4 承担部分） | ⚠️ §6 HS-07 | +| madmin 客户端 HealStart/HealStatus/BackgroundHealStatus/ScannerStatus 方法 | 仅 wire 类型，无客户端方法 | ❌ §6 HS-05 | +| mc admin heal --pool/--set、--scan-mode、--force-start/stop | HealOpts 全字段支持（pool/set/scanMode/forceStart/forceStop） | ✅（服务端就绪；缺 mc 侧入口，HS-05） | +| ErrHealAlreadyRunning / ErrHealOverlappingPaths 类型化错误 | 去重合并 + 驱逐语义；无类型化重叠拒绝 | ⚠️ §6 HS-06 | +| 结果 backpressure（maxUnconsumedItems=1000、10s 保活流式、24h 未消费 abort） | 快照式查询（1024 条 + 8MiB 截断 + 10min 保留） | ⚠️ §6 HS-06 | +| `mc support inspect`/healing-bin 离线 dump | 无（inspect.rs 存在但 healing dump 未确认） | ⚠️ P3 | + +### 4.6 观测面对照 + +| 维度 | MinIO | RustFS | 状态 | +|---|---|---|---| +| heal 指标 | minio_heal_objects_total/heal_total/errors_total/time_last_activity + v3 drive_health 2=healing | rustfs_heal_* 全套（admission/queue delay/running/throttle/page concurrency） | ✅（RustFS 缺 drive_health=healing 单一 gauge 等价物；DiskInfo.healing 已赋值） | +| scanner 指标 | v3 6 个 + realtime 18 项 | rustfs_scanner_* 全套 + per-source 维度 | ✅ | +| ILM 指标 | v3 5 个（expiry/transition pending/active/missed + versions_scanned） | ilm expiry status API + scanner per-source | ✅（指标与 API 形态不同） | +| trace | TraceHealing/TraceScanner 两通道 | 无 | ❌ §6 HS-03 | +| 审计 | HealObject 事件、dangling 删除审计、scanner:manyversions 等 | 结构化日志（event style）+ 指标；无 audit log 事件 | ⚠️ §6 HS-04 | +| 进度 | healingTracker Bytes/Items/QueuedBuckets/当前对象 + usage-cache 总量基线 | HealProgress{scanned/healed/failed/bytes/current_object/percentage}；bytes_processed 注释为 0、estimated_completion_time 恒 None | ⚠️ §6 HS-07 | + +### 4.7 配置面对照（默认值） + +| MinIO | RustFS | 备注 | +|---|---|---| +| `heal:bitrotscan`（默认 off；on=每轮；Nm=N×30×24h） | `heal.bitrot_cycle` / `RUSTFS_SCANNER_BITROT_CYCLE_SECS`（默认 30d=2592000s；0/on=每轮 Deep，off=禁用） | ✅ 同语义（RustFS 默认 30d，MinIO 默认 off——**默认值不同**，RustFS 更激进） | +| `heal:max_io=100`/`max_sleep=250ms`（waitForLowIO） | mainline throttle 阈值 80%/80%、max_sleep 250ms | ✅ 同型（阈值模型不同） | +| `heal:drive_workers`（默认 -1 自动） | 页内并发 8 + per-set 1 | ✅ 同型 | +| `_MINIO_HEAL_WORKERS`（GOMAXPROCS/2） | `RUSTFS_HEAL_MAX_CONCURRENT_HEALS=4` + `_MAX_CONCURRENT_PER_SET=1` | ✅ | +| `_MINIO_AUTO_DRIVE_HEALING`（on） | `RUSTFS_HEAL_AUTO_HEAL_ENABLE=true` | ✅ | +| `_MINIO_SCANNER`（on） | `RUSTFS_SCANNER_ENABLED=true` | ✅ | +| `scanner:speed` 五档（default=2x/1s/1m） | 同五档同名同参数 | ✅ | +| `scanner:idle_speed`（on） | `RUSTFS_SCANNER_IDLE_MODE`（true） | ⚠️ 语义方向（HS-14） | +| `scanner:alert_excess_versions=100` | 100 | ✅ | +| `scanner:alert_excess_folders=50000` | 65538（兼容 PBS 布局） | ⚠️ HS-17 | +| `ilm:expiration_workers=100`/`transition_workers=100` | ecstore expiry/transition worker 池（键见 ilm 子系统） | ✅（默认值未逐项核对） | +| `api:stale_upload_cleanup_interval=6h`/`expiry=24h` | ecstore 后台任务 env 可配 | ✅（默认值未逐项核对） | +| —（无） | `RUSTFS_HEAL_QUEUE_SIZE=10000`、`_TASK_TIMEOUT_SECS=300`、`_INTERVAL_SECS=10`、`_LOW_PRIORITY_MERGE/DROP`、`_PAGE_*`、`_SET_BULKHEAD`、`_MAINLINE_*`、`RUSTFS_SCANNER_CYCLE_MAX_*` 预算、`_MAX_CONCURRENT_SET/DISK_SCANS=4`、`_YIELD_EVERY_N_OBJECTS=128` 等 | RustFS 特有（更细粒度） | + +### 4.8 RustFS 超出 MinIO 的部分 + +1. remote_scanner RPC（扫描执行下放远端 peer 本地，含 HMAC 认证/重放缓存/fence 复验/断连宽限）。 +2. 持久化 leader-epoch CAS 围栏 + usage 快照 epoch/cycle 防回退（MinIO 仅锁，无持久 epoch）。 +3. 周期预算（max_duration/objects/directories）+ partial 周期推进语义。 +4. per-set/per-disk 扫描并发闸 + 每桶每 set 缓存锁。 +5. pending-heal 账本（heal 通道满不丢候选）。 +6. 换盘 durable intent + completion proof 状态机 + 身份围栏（MinIO healingTracker 无 proof）。 +7. mainline throttle 前台压力门控（permit 利用率驱动）。 +8. 集群 heal control coordinator + envelope 重放防护 + degraded 显式降级。 +9. 写路径 shard bitrot 自校验（EC:0 场景）。 +10. dirty-usage 快路径唤醒（写路径即时通知 + 脏桶优先）。 +11. heal 运行时可观测矩阵（优先级×来源 operations snapshot）。 +12. workload admission 联动（heal 调度器读前台压力快照）。 + +--- + +## 5. 差距与改进清单 + +分级定义：P1=行为/运维对齐缺口（影响生产运维或工具链兼容）；P2=完善性（功能在但缺一角）；P3=清理/低风险。每项含现状证据、MinIO 行为、影响、建议、验收方式。 + +### P1（8 项） + +**HS-01 MRF/ECDecode/Metadata 三类 heal 任务无生产触发入口，HealEvent 未接线** +- 现状：`HealType::MRF/ECDecode/Metadata` 执行体完整（task.rs:1700-2156）但全仓库无生产触发方；`HealEvent`/`HealEventHandler`（event.rs:50-367）crate 外零引用（已亲验 grep）；channel 转换只产生 Cluster/Object/Bucket/Prefix/ErasureSet（channel.rs:566-601）。 +- MinIO：mrf.go 独立 MRF 队列（容量 100k，满丢弃计数）、进程退出 msgp 持久化 `.heal/mrf/list.bin` + 启动回放、入队 <1s 延迟 1s（等网络恢复）、healSleeper 限速；读路径 GetObject part 缺失/损坏、元数据重建 missingBlocks>0、Put 部分成功、DeleteObject、multipart、peer client 共 7+ 投递点。 +- 影响：RustFS 的 read-repair + 写路径收敛覆盖了主场景，但缺少：① 事件驱动的 Urgent ECDecode 重建入口（ecstore 解码失败时目前仅 Low read-repair）；② Metadata-only heal 入口（scanner HealMetadata 分类存在但走普通对象 heal）；③ MRF 队列持久化（重启丢未消费修复意图——scanner pending-heal 账本部分缓解）。 +- 建议：三选一决策——(a) 接线 HealEvent（在 ecstore 解码失败/metadata 损坏点发事件）+ 实现持久化重试账本；(b) 删除 MRF/ECDecode/Metadata 死代码只保留文档说明；(c) 保留执行体、把 HealEvent 降级为内部 API。推荐 (a) 但需先量化 read-repair 是否已覆盖解码失败场景的响应时间要求。 +- 验收：解码失败 → Urgent heal 请求链路 e2e；重启后 pending 修复意图回放；HealEvent 环形缓冲指标。 + +**HS-02 CheckAbandonedParts 三层 NotImplemented（abandoned data 独立对账入口缺失）** +- 现状：`set_disk/ops/heal.rs:2052-2056`、`core/sets.rs:1144-1148`、`store/heal.rs:258-266` 三层显式 `Err(NotImplemented)`（已亲验），注释"intentionally retained above the set layer until there is a concrete caller"。 +- MinIO：`CheckAbandonedParts` → 每盘 `CleanAbandonedData`：读 xl.meta → 列 UUID data-dir + inline entries → 与 getDataDirs 差集 → 删多余 data-dir/inline 并重写 xl.meta；由 scanner 抽中 heal 与 admin heal Remove 时显式调用。 +- 影响：RustFS heal 路径内 `reclaim_orphan_data_dirs_best_effort`（:1428）覆盖"heal 时回收孤儿目录"，但 ① 无独立触发点（MinIO 在对象未到 heal 阈值时也能清 abandoned data）；② inline data 孤儿条目清理未确认；③ multipart 孤儿对账明确不做（设计决定，由 lifecycle 承担）。 +- 建议：评估把 `reclaim_orphan_data_dirs_best_effort` 提升为 heal_object 固定步骤（若尚非）+ 实现 HealOperations::check_abandoned_parts 真实现（调用同一回收逻辑），或明确文档化"由 lifecycle 承担"并关闭 API 面。 +- 验收：构造 data-dir/inline 孤儿 → scanner 抽样/admin heal 后被清理；三层 API 返回成功或显式 NotSupported 文档化。 + +**HS-03 heal/scanner trace 通道缺失** +- 现状：TraceHealing/TraceScanner 零命中（已亲验 grep 全仓库）。 +- MinIO：`madmin.TraceHealing`（mc admin trace --healing，FuncName=heal.Bucket/heal.Object/heal.CheckAbandonedParts，带 dry/remove/mode/version-id/disks/bytes）、`TraceScanner`（mc admin scanner trace，支持 --filter-size/--response-duration）。 +- 影响：无法实时观测单个 heal/scanner 动作的耗时与参数；排障只能靠指标聚合与日志。 +- 建议：在 heal channel 执行与 scanner folder/item 处理埋点，接入现有 admin trace 订阅面（若 rustfs 已有 trace 基建则复用，无则按 madmin TraceType 扩展）。 +- 验收：mc 等价工具能订阅 heal/scanner trace 流。 + +**HS-04 scanner 超限 S3 事件与审计缺失** +- 现状：仅 `rustfs_scanner_excess_*_total` 指标（versions 100/version size 1TiB/folders 65538）。 +- MinIO：发 `s3:ObjectManyVersions`（>100 版本）、`s3:ObjectLargeVersions`（累计 >1TB）、`s3:PrefixManyFolders`（>50000 子目录）事件（UserAgent: Scanner）+ scanner:manyversions/largeversions/manyprefixes 审计。 +- 影响：依赖事件订阅做容量治理的用户（console/外部审计）收不到告警。 +- 建议：scanner_folder 告警点接入 notify 事件发布（复用 lifecycle 事件通道语义）。 +- 验收：配置桶通知后超限对象触发事件。 + +**HS-05 madmin 客户端方法缺失** +- 现状：`crates/madmin/src/heal_commands.rs` 只有 wire 类型（HealDriveInfo/Infos/HealResultItem）；无 HealStart/HealStatus/BackgroundHealStatus/ScannerStatus 客户端方法。 +- MinIO：madmin-go 提供完整客户端；mc admin heal/scanner/status/trace 都建立在上面。 +- 影响：mc 等管理工具无法直接对接 RustFS heal/scanner 管理面；自动化运维只能手写 HTTP。 +- 建议：按 madmin-go 接口形状补客户端（服务端已就绪，纯客户端工作）。 +- 验收：用 madmin 客户端完成 start→query→cancel 全流程。 + +**HS-06 admin heal 序列语义与 MinIO 差异** +- 现状：重复/重叠请求被去重合并（返回 canonical task_id）或驱逐；无 ErrHealAlreadyRunning/ErrHealOverlappingPaths 类型化错误（已亲验：manager.rs:1309 的 already_running 是幂等启动保护，非 admin 语义）；结果为快照式查询（1024 条/8MiB 截断/10min 保留），非 MinIO 的流式增量（clientToken 拉增量 + maxUnconsumedItems=1000 backpressure + 10s 保活 + 24h 未消费 abort）。 +- 影响：mc admin heal 的交互模型（长连接拉增量）对 RustFS 表现为多次快照轮询；自动化脚本难以区分"已合并"与"新启动"。 +- 建议：① 增量语义：channel query 支持自上次 clientToken 起的 items 增量（或 cursor）；② 重叠请求返回类型化错误码（或 receipt 中显式 merged_into 字段——现有 alias 机制已有基础）；③ forceStart 先停旧再启新语义核对。 +- 验收：madmin 兼容客户端按 MinIO 模式轮询能取得全量 items。 + +**HS-07 healing 进度与盘级 healing 状态对外可见性不足** +- 现状：bytes 恢复进度 `progress.bytes_processed = 0 // set to 0 for now`（erasure_healer.rs:967）；`HealProgress::estimated_completion_time` 恒 None、`HealStatistics::add_healed_objects` 未写入（progress.rs:38,135-139 零调用）；healthinfo 无每盘 HealInfo 等价（MinIO HealingDisk：BytesDone/Failed/Skipped、ObjectsTotal 基线、QueuedBuckets/HealedBuckets、Resume 快照、当前 object）；v3 指标无 drive_health=2(healing) 单一 gauge 等价。 +- 影响：换盘重建（可能数小时~天）期间运维无法回答"进行到哪/还剩多少/预计何时完成"。 +- 建议：① erasure set heal 统计 bytes（heal_object 返回对象大小已可得）；② 从 usage-cache 读对象总量基线（MinIO 同款做法）；③ admin healthinfo/背景状态暴露每盘 healing 快照（DiskInfo.healing 已有，补聚合暴露）；④ ETA 由基线+速率推导。 +- 验收：换盘重建中 admin 可见 bytes 进度与 ETA；mc info 等价输出 Healing 标志。 + +**HS-08 prefix 级 usage 未暴露** +- 现状：DataUsageCache 内目录树 entry 存在（hash_path 组织），但 `dui()` 只 flatten 到桶名（data_usage_define.rs:858-915）。 +- MinIO：`loadPrefixUsageFromBackend`（30s cache）从每 set `.usage-cache.bin` 聚合 prefix usage，console 桶前缀统计消费。 +- 影响：console/前端无法展示前缀级用量；大桶定位"哪个前缀占空间"无 API。 +- 建议：实现 flatten 前缀查询 API（数据已在缓存内，纯聚合与暴露工作）。 +- 验收：ListBuckets/PrefixUsage API 返回与前缀过滤匹配的统计。 + +### P2（9 项） + +**HS-09 get_disk_status 恒返回 Ok（唯一 TODO）**：`crates/heal/src/heal/storage.rs:930-943`（已亲验）。当前无生产调用方（低风险）。建议：删除该方法或接 ecstore disk 状态真实现（DiskStatus 枚举已定义）。 + +**HS-10 HealStorageAPI 约 1/3 方法为死代码**：get_object_meta/get_object_data/put_object_data/delete_object/verify_object_integrity/ec_decode_rebuild/get_disk_status/format_disk/heal_bucket_metadata/get_object_size/get_object_checksum/list_objects_for_heal（非分页版，自带 memory_heavy 警告）均 0 调用方。建议：随 HS-01 决策一并清理或接线（死接口误导后续维护者以为存在调用路径）。 + +**HS-11 bitrot 自检缺失**：MinIO 启动时 bitrotSelfTest 对四算法已知向量自检失败即 Fatal（防静默数据损坏）。RustFS 无等价（已亲验 grep）。建议：启动时对 HighwayHash256S 等在用算法做已知向量自检（低成本高价值）。 + +**HS-12 对象级 healing 元数据标记评估**：MinIO heal 期间对象打 `x-minio-healing:true`，RenameData 据此跳过版本清理/legacy purge（漏掉会导致 heal 与并发删除互毁）。RustFS 无对象级标记（已亲验 grep object.rs 无 healing 分支），依赖 NSLock + rename 语义。建议：审计 RustFS rename 提交路径是否存在"heal 提交与并发 delete/version 清理竞争"窗口；若无则文档化差异，若有则补标记等价机制。 + +**HS-13 erasure set heal 无"跳过新写入/ILM 已过期版本"过滤**：MinIO resync 跳过 ModTime>tracker.Started 的版本（避免 heal 追新写入尾巴）与 ILM 已过期版本（避免白做）。RustFS erasure_healer 未实现同款过滤（按版本 dedup 有，时间/ILM 过滤无）。影响：重建尾部长尾（持续写入的桶 heal 完成判定被新版本推迟）与无效 heal 工作量。建议：disk-walk 枚举处加 started_at 时间过滤 + evaluator 预检。 + +**HS-14 scanner idle 语义方向与 MinIO 相反**：MinIO `scanner:idle_speed=on`（默认）= 集群空闲时才节流、忙时全速；RustFS `RUSTFS_SCANNER_IDLE_MODE=true`（默认）= 限速总闸（false=完全不休眠）。两者默认行为可能相近（都限速）但参数语义不可互换，迁移文档需显式说明；若追求 mc config 兼容需重命名/重语义。建议：先文档化差异，评估是否对齐语义。 + +**HS-15 alert_excess_folders 默认值差异**：RustFS 65538（兼容 PBS/Proxmox 布局，scanner_folder.rs:79）vs MinIO 50000。行为差异默认即触发阈值不同。建议：文档化（保留 65538 有本地理由）。 + +**HS-16 单机默认周期钩子未启用**：`single_disk_default_cycle_secs(_features) -> None` 恒空（scanner.rs:1428-1430），单机部署无专属默认周期覆盖。建议：决定单机默认周期策略后启用或删除钩子。 + +**HS-17 DeleteAllVersions 批量优化核对**：MinIO 用 DeletePrefix+DeletePrefixObject 单调用代替逐版本 fan-out。RustFS expiry 队列路径是否同款优化未逐行核实（集成测试覆盖行为正确性）。建议：核对 `apply_expiry_rule` 全版本删除路径，若无前缀单调用优化则评估补齐。 + +### P3（3 项） + +**HS-18 trash/临时目录二段清理细节核对**：MinIO `.minio.sys/tmp/.trash` 清理（delete_cleanup_interval 默认 5m + deleteCleanupSleeper）与 stale uploads rename-into-trash 二段式。RustFS 有 delete_tail_activity.rs 与 stale multipart 任务，二段语义是否完整对齐未逐行核实。建议：对照补齐或文档化。 + +**HS-19 root heal 直连死路径清理**：`should_handle_root_heal_directly` 恒 false（admin/handlers/heal.rs:1200-1202，测试锁定），store.heal_format 直连分支不可达。建议：删除死分支或恢复直连路径作为集群协调失败的降级。 + +**HS-20 兼容旗标与死指标清理**：`RUSTFS_SCANNER_INLINE_HEAL_ENABLE`（开启仅告警）+ `rustfs_scanner_inline_heal_total` 死指标 + `rustfs_common::metrics` 中 scanner 域代码分层迁移（backlog #1843 已登记）。建议：随分层迁移一并清理。 + +### 按设计不追平（7 项，记录以防后续误判为缺口） + +1. **bloom filter**：MinIO master 已删除；RustFS `.bloomcycle.bin` 复用为 cycle/epoch 围栏与 MinIO 现状一致。 +2. **scanner 集群单 leader**：双方一致；RustFS 额外有 epoch 围栏。 +3. **heal 不发 S3 bucket notification**：双方一致（heal 结果走 admin status）。 +4. **incomplete multipart 不在 scanner/ILM 内执行**：双方一致（独立后台例程）。 +5. **内联 heal 移除**：RustFS 有意为之（scanner 只入队），MinIO 的 applyHealing 内联路径不做对标。 +6. **heal 序列常驻保活（10s 空白回写）**：RustFS 快照式查询模型不同，按 HS-06 处理增量语义即可，不复制流式保活。 +7. **`.trash`/`tmp-old` 路径名兼容**：RustFS 布局常量独立，不逐字对齐 MinIO 路径。 + +--- + +## 6. 配置默认值总表（RustFS） + +heal（env 前缀 `RUSTFS_HEAL_`，`crates/config/src/constants/heal.rs`，消费于 `manager.rs:724-800`）： + +| 配置 | 默认 | 热更新 | +|---|---|---| +| AUTO_HEAL_ENABLE | true | 否 | +| QUEUE_SIZE | 10000 | 否 | +| INTERVAL_SECS | 10 | 否（启动时固定） | +| TASK_TIMEOUT_SECS | 300 | 否 | +| MAX_CONCURRENT_HEALS | 4 | 否 | +| MAX_CONCURRENT_PER_SET | 1（≤min(全局,值)） | 否 | +| LOW_PRIORITY_MERGE_ENABLE | true | 否 | +| LOW_PRIORITY_DROP_WHEN_FULL | true | 否 | +| PAGE_OBJECT_CONCURRENCY | 8（Deep/AutoHeal 强制 1） | 否 | +| EVENT_DRIVEN_SCHEDULER_ENABLE | true | 否 | +| SET_BULKHEAD_ENABLE | true | 否 | +| PAGE_PARALLEL_ENABLE | true | 否 | +| MAINLINE_THROTTLE_ENABLE | true | 否 | +| MAINLINE_READ/WRITE_UTILIZATION_HIGH_PERCENT | 80/80 | 否 | +| MAINLINE_MAX_SLEEP_MS | 250 | 否 | +| （总开关）RUSTFS_HEAL_ENABLED | true | 否 | +| admin 子系统 heal.bitrot_cycle | 30d | 是（经 scanner runtime config） | + +scanner（admin 子系统 `scanner`，`crates/config/src/constants/scanner.rs` + `ecstore/src/config/scanner.rs` + `runtime_config.rs:527-673`）： + +| 键 | env | 默认 | +|---|---|---| +| speed | RUSTFS_SCANNER_SPEED | default（2x/1s/60s） | +| delay / max_wait / cycle / start_delay | RUSTFS_SCANNER_* | 派生/空 | +| cycle_max_duration/objects/directories | …_MAX_* | 0（不限） | +| bitrot_cycle | …_BITROT_CYCLE_SECS | 2592000（30d；0/on=每轮，off=禁用） | +| idle_mode | …_IDLE_MODE | true | +| cache_save_timeout | …_CACHE_SAVE_TIMEOUT_SECS | 30s | +| max_concurrent_set_scans / disk_scans | …_MAX_CONCURRENT_* | 4/4 | +| yield_every_n_objects | …_YIELD_EVERY_N_OBJECTS | 128 | +| alert_excess_versions / version_size / folders | …_ALERT_* | 100 / 1TiB / 65538 | + +scanner 内部 env：`RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=16`、`RUSTFS_HEAL_OBJECT_SELECT_PROB=1024`、`RUSTFS_SCANNER_DEEP_VERIFY_COOLDOWN_SECS=60`、`RUSTFS_DATA_USAGE_FAILED_OBJECT_TTL_SECS=86400`/`_MAX=10000`、`RUSTFS_LOCK_ACQUIRE_TIMEOUT=5s`、`RUSTFS_SCANNER_ENABLED=true`、`RUSTFS_SCANNER_INLINE_HEAL_ENABLE=false`（兼容告警）。 + +全部 17 个 scanner 键支持 env > config 双通道 + admin PUT 热更（generation+Notify 即时生效）；heal 运行时参数目前仅 env（无 admin 热更入口，`Arc>` 结构已预留）。 + +--- + +## 7. 相关 backlog / 历史索引 + +- 换盘自动修复系列（已闭环）：backlog #1786（冗余假绿算法）、#1787（目标槽位限定）、#1789（resume 与 healing marker 绑定 replacement 实例）、#1791（黑白盒验收矩阵）。 +- #801 DiskInfo.healing 从未赋值（已修复闭环，现 `set_disk/mod.rs:4988` 有赋值链）。 +- #1651 Scanner 指标节点/source/bucket-drive 维度（OPEN，本分析 §3.8/§4.6 相关）。 +- #1843 crates/common 83% scanner/heal 域代码分层迁移（OPEN，含 HS-20）。 +- 代码注释引用的历史缺陷（现已有防护与回归测试）：#856/#799 B7（离线盘误记 healed）、#855/B6/#1033（skip 不得标记完成）、#920（sub-quorum 并集枚举）、#856 B5（按版本续扫）、#5173（bitrot trailing bytes）、#5029（回归节点 stale 版本合并）。 +- v1 对标文档：`docs/rustfs-heal-scanner-vs-minio-parity-assessment.md`（本文取代）、落地手册 `docs/rustfs-heal-scanner-vs-minio-improvement-playbook.md`（部分条目已被后续实现超越）。 +- 换盘深度分析：`docs/new-disk-replacement-and-healing-deep-analysis-zh.md`、`docs/node-disk-identity-and-healing-analysis-zh.md`。 + +## 8. 审计方法与局限 + +- 四路并行审计（heal crate 逐文件、scanner crate 逐文件、ecstore 集成层 wiring、MinIO master 源码研究）+ 主会话对关键"缺失"结论逐条亲验（get_disk_status TODO、HealEvent 零外部引用、.bloomcycle.bin 无 bloom 实现、check_abandoned_parts 三层 NotImplemented、ETag 兜底已实现、trace 通道零命中、already_running 语义）。 +- 未逐行核实的点（已在文中标注"未确认/未逐行核"）：DeleteAllVersions 前缀单调用优化（HS-17）、trash 二段清理细节（HS-18）、ilm worker 默认值对照、stale multipart 默认值对照、mc CLI flag 逐字拼写（MinIO 侧）。其中 HS-17 与 HS-18 已于 2026-08-19 完成逐行核实，结论见 §9.2/§9.3。 +- MinIO 侧引用以其 master `7aac2a2c5b` 为准；RustFS 侧行号以 2026-08-16 工作区为准，后续演进请以符号名检索为准。 + +## 9. 落地结果（2026-08-19 更新） + +本审计衍生的 14 个子 issue（backlog #1865~#1878）已全部闭环。本节为差距清单 HS-01~HS-20 的最终处置记录，也是下一轮对标重审的增量基线。 + +### 9.1 已落地（PR 均已合并 main） + +- HS-01 MRF 接线 + 持久化修复账本（#1865，PR #6189）：决策选 (a)。common MRF channel（bounded 8192、try_send 永不阻塞）+ heal mrf_queue（100k 条 / 8MiB 双限环形）+ `buckets/.heal/mrf/journal.bin` CRC 持久化回放（torn tail 截断、回放后删除）+ 三投递点（read decode_error→Urgent ECDecode、scanner 元数据损坏→High Metadata、add_partial→Normal）+ `RUSTFS_HEAL_MRF_ENABLE` 一键回退。 +- HS-02 abandoned parts/data-dir 对账（#1866，PR #6179）：接通 abandoned 检查入口，保留 dry-run / reclaim 计数。 +- HS-03 heal/scanner trace 通道（#1867，PR #6179）：进程内 trace bus + `/v3/trace` admin 流式订阅 + heal task / abandoned-parts / scanner folder / ILM / heal-candidate trace producer。 +- HS-04 scanner 超限 S3 事件（#1868，PR #6176）：`s3:Scanner:ManyVersions/LargeVersions/BigPrefix` 三事件 + 24h 边沿冷却；HS-15 阈值差异文档化（`docs/operations/scanner-excess-alerts.md`）。 +- HS-05 madmin 客户端一期（#1869，PR #6166）：SigV4 admin 客户端 heal/scanner 方法；增量消费方法待 follow-up（协议已由 HS-06 并入）。 +- HS-06 admin heal 增量语义与类型化重叠（#1870，PR #6206）：`sinceSeq/nextSeq/minSeq` 增量游标（wire additive、缺省=全量快照）+ `RUSTFS_HEAL_OVERLAP_POLICY`（默认 merge 不变；minio_error 下 AlreadyRunning/OverlappingPaths 类型化拒绝）+ forceStart 先停旧再启新。 +- HS-07 healing 进度可见性（#1871，PR #6179）：data-usage 总量基线 + baseline/current/healed 计数。 +- HS-08 prefix usage（#1872，PR #6171）：`GET /v3/usage/{bucket}`。 +- HS-11 bitrot 启动自检（#1873，PR #6165）。 +- HS-13 heal 跳过过滤（#1875，PR #6179）：过滤命中版本不再计为失败。 +- HS-16 单机周期钩子（#1878，PR #6250）：删恒 None 钩子，决策记录见 `docs/operations/heal-scanner-parity-notes-zh.md`。 +- HS-09/10/19/20 死代码清理批（#1877，PR #6256）：净 −911 行零行为变更；`get_disk_status` TODO（全仓库唯一产品 TODO）清零；HS-01 联动的 `ec_decode_rebuild`/`get_object_meta` 保留并加 Reserved 注释（MRF 当前经 `heal_object` 执行）。 + +### 9.2 核对后确认"已实现 / 非缺口"（审计期误判修正，累计四例） + +- bloom filter（§0 已修正）：MinIO master 已删除，双方现状一致。 +- ETag 兜底仲裁（§0 已修正）：RustFS 已有实现（`set_disk/ops/heal.rs`）。 +- HS-17（#1876，2026-08-19 逐行核实后关闭）：DeleteAllVersions 前缀单调用优化 RustFS 已完整实现——`apply_expiry_on_non_transitioned_objects` 对 `delete_all()` 两 action 设 `delete_prefix + delete_prefix_object` 后单次 `delete_object`（`bucket_lifecycle_ops.rs:5047-5056`），SetDisks 分支一次写锁 + 一次全版本 quorum 读 + 内联逐版本 object-lock 检查（`set_disk/ops/object.rs:5566-5612`），与 MinIO `expire.go` 的 `applyExpiryOnNonTransitionedObjects` 逐行对齐。§8 原列"未逐行核实"的本项已有结论：现状即优化路径，无需实现。 +- HS-14（#1878，PR #6250 附带核对）：MinIO"idle=空闲才节流"是 2024-01 minio/minio#18734 之前的行为（`scannerIdleMode` 现为静态配置，`idle_speed=on` 默认即始终按速度档节流，"idle"命名是历史残留）；RustFS `RUSTFS_SCANNER_IDLE_MODE` 与 MinIO 当前语义方向一致，且另有 MinIO 没有的前台读退避下限。真实迁移陷阱（变量须 `RUSTFS_` 前缀、`on/off` vs `true/false` 词表、`false` 连前台保护一起关）已文档化于 `docs/operations/heal-scanner-parity-notes-zh.md`。 + +### 9.3 审计型结论（无需改代码） + +- HS-12（#1874，PR #6183）：不存在 MinIO 用 `x-minio-healing` 防御的那类竞争——所有同 (bucket, object) 提交面在同一把对象级 ns 写锁互斥，heal 锁 guard 覆盖 rename 提交全程；交付 2 个并发不变量回归测试 + `docs/operations/heal-concurrency-safety-notes-zh.md` 交点矩阵。 +- HS-18（#1878，2026-08-19 逐行核实）：trash/tmp 三段清理全对齐——stale multipart 隔离-清理等价且更安全（`delete_all_with_quorum` 逐盘递归删即 `move_to_trash` rename 进 `.rustfs.sys/tmp/.trash`，另有锁 + fence）、trash 排空基本等价（无逐条 sleeper 节流，5m 周期天然限频）、tmp 非 trash 24h 回收等价（RustFS 5m 比 MinIO 6h 更及时）；周期默认 24h/6h/5m 三项全对齐。§8 原列"未逐行核实"的本项已有结论。 + +### 9.4 移交 follow-up（汇总于 backlog#1862 评论区） + +HS-01 bitrot GET→MRF 全链路 e2e、kill -9 journal 回放 e2e、队列满压测 RSS（≤ 预算+10%）；HS-05/06 madmin 增量消费方法 + wire 单一来源化 + embedded e2e + 多轮轮询 soak；HS-08 多盘 scanner 周期 e2e；HS-04 超限审计条目；HS-18 低于 quorum 的 stale-multipart 崩溃残留窗口（扇出中途崩溃且已清盘数 > parity 时 FileNotFound 不在忽略集导致不自然收敛，修复需专用 quorum 变体）。 + +下一轮重审建议：跟随 heal/scanner 下一个大特性落地后触发，以本节为增量基线。 From 6f3f2f5f62763e90ea09c8b2084d556ea9091e57 Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 19 Aug 2026 14:06:38 +0800 Subject: [PATCH 15/27] test(lifecycle): cover noncurrent marker cleanup cascade (#6252) --- crates/e2e_test/src/reliant/lifecycle.rs | 170 +++++++++++++++++++++++ 1 file changed, 170 insertions(+) diff --git a/crates/e2e_test/src/reliant/lifecycle.rs b/crates/e2e_test/src/reliant/lifecycle.rs index 3c475bf88..9de87dff4 100644 --- a/crates/e2e_test/src/reliant/lifecycle.rs +++ b/crates/e2e_test/src/reliant/lifecycle.rs @@ -168,6 +168,24 @@ async fn wait_for_version_expired( } } +async fn wait_for_key_versions_empty(client: &Client, bucket: &str, key: &str, deadline: StdDuration) -> TestResult { + let start = std::time::Instant::now(); + loop { + let listing = client.list_object_versions().bucket(bucket).prefix(key).send().await?; + if listing.versions().is_empty() && listing.delete_markers().is_empty() { + return Ok(()); + } + if start.elapsed() >= deadline { + return Err(format!( + "object {bucket}/{key} still had versions or delete markers after {}s: {listing:?}", + deadline.as_secs() + ) + .into()); + } + tokio::time::sleep(StdDuration::from_millis(500)).await; + } +} + /// Build a prefix-scoped `Days`-based expiration rule. fn expiration_rule(id: &str, prefix: &str, days: i32) -> Result> { let rule = LifecycleRule::builder() @@ -193,6 +211,21 @@ fn noncurrent_expiration_rule( Ok(rule) } +fn noncurrent_expiration_with_delete_marker_cleanup_rule( + id: &str, + prefix: &str, + days: i32, +) -> Result> { + let rule = LifecycleRule::builder() + .id(id) + .filter(LifecycleRuleFilter::builder().prefix(prefix).build()) + .expiration(LifecycleExpiration::builder().expired_object_delete_marker(true).build()) + .noncurrent_version_expiration(NoncurrentVersionExpiration::builder().noncurrent_days(days).build()) + .status(ExpirationStatus::Enabled) + .build()?; + Ok(rule) +} + async fn put_expiration_config(client: &Client, bucket: &str, rule: LifecycleRule) -> TestResult { let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?; client @@ -412,6 +445,143 @@ async fn test_lifecycle_noncurrent_version_expiry_removes_only_old_version() -> Ok(()) } +/// A combined `NoncurrentDays=1` and `ExpiredObjectDeleteMarker=true` rule +/// must remove a noncurrent data version and then its sole latest delete +/// marker, without expiring current-only objects. A second prefix with only +/// noncurrent expiry proves that marker cleanup comes from EODM. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_lifecycle_noncurrent_expiry_then_cleans_expired_delete_marker() -> TestResult { + let mut env = RustFSTestEnvironment::new().await?; + let mut extra_env = fast_lifecycle_env(); + extra_env.push(("RUSTFS_ILM_DEBUG_DAY_SECS", "2")); + env.start_rustfs_server_with_env(vec![], &extra_env).await?; + + let client = env.create_s3_client(); + let bucket = "ilm-expired-delete-marker"; + client.create_bucket().bucket(bucket).send().await?; + client + .put_bucket_versioning() + .bucket(bucket) + .versioning_configuration( + VersioningConfiguration::builder() + .status(BucketVersioningStatus::Enabled) + .build(), + ) + .send() + .await?; + + let cascade_key = "cascade/deleted.txt"; + let cascade_put = client + .put_object() + .bucket(bucket) + .key(cascade_key) + .body(ByteStream::from_static(b"cascade payload")) + .send() + .await?; + let cascade_data_version = cascade_put + .version_id() + .map(str::to_string) + .expect("cascade PUT returns a version id"); + let cascade_delete = client.delete_object().bucket(bucket).key(cascade_key).send().await?; + let cascade_marker_version = cascade_delete + .version_id() + .map(str::to_string) + .expect("cascade DELETE returns a marker version id"); + assert_eq!(cascade_delete.delete_marker(), Some(true)); + + let survivor_key = "cascade/current-only.txt"; + client + .put_object() + .bucket(bucket) + .key(survivor_key) + .body(ByteStream::from_static(b"current payload")) + .send() + .await?; + let survivor_before = client.get_object().bucket(bucket).key(survivor_key).send().await?; + assert_eq!(survivor_before.body.collect().await?.into_bytes().as_ref(), b"current payload"); + + let control_key = "nve-only/deleted.txt"; + let control_put = client + .put_object() + .bucket(bucket) + .key(control_key) + .body(ByteStream::from_static(b"control payload")) + .send() + .await?; + let control_data_version = control_put + .version_id() + .map(str::to_string) + .expect("control PUT returns a version id"); + let control_delete = client.delete_object().bucket(bucket).key(control_key).send().await?; + let control_marker_version = control_delete + .version_id() + .map(str::to_string) + .expect("control DELETE returns a marker version id"); + assert_eq!(control_delete.delete_marker(), Some(true)); + + let cascade_before = client + .list_object_versions() + .bucket(bucket) + .prefix(cascade_key) + .send() + .await?; + assert!( + cascade_before + .versions() + .iter() + .any(|version| version.version_id() == Some(cascade_data_version.as_str())), + "cascade data version must exist before lifecycle is installed: {cascade_before:?}" + ); + assert!( + cascade_before + .delete_markers() + .iter() + .any(|marker| { marker.version_id() == Some(cascade_marker_version.as_str()) && marker.is_latest() == Some(true) }), + "cascade latest delete marker must exist before lifecycle is installed: {cascade_before:?}" + ); + + let lifecycle = BucketLifecycleConfiguration::builder() + .rules(noncurrent_expiration_with_delete_marker_cleanup_rule( + "expire-and-clean-marker", + "cascade/", + 1, + )?) + .rules(noncurrent_expiration_rule("expire-only", "nve-only/", 1)?) + .build()?; + client + .put_bucket_lifecycle_configuration() + .bucket(bucket) + .lifecycle_configuration(lifecycle) + .send() + .await?; + + wait_for_key_versions_empty(&client, bucket, cascade_key, StdDuration::from_secs(90)).await?; + wait_for_version_expired(&client, bucket, control_key, &control_data_version, StdDuration::from_secs(90)).await?; + + let survivor = client.get_object().bucket(bucket).key(survivor_key).send().await?; + assert_eq!(survivor.body.collect().await?.into_bytes().as_ref(), b"current payload"); + + let control_after = client + .list_object_versions() + .bucket(bucket) + .prefix(control_key) + .send() + .await?; + assert!( + control_after.versions().is_empty(), + "NVE-only control must remove its data version: {control_after:?}" + ); + assert!( + control_after + .delete_markers() + .iter() + .any(|marker| { marker.version_id() == Some(control_marker_version.as_str()) && marker.is_latest() == Some(true) }), + "NVE-only control must preserve its latest delete marker: {control_after:?}" + ); + + Ok(()) +} + /// `Days=0` expiration is invalid per S3 semantics (`Days` must be a positive /// integer >= 1). A `PutBucketLifecycleConfiguration` carrying a zero-day rule /// must be rejected with `InvalidArgument` (HTTP 400) - see crates/lifecycle From 05e6dc5f4a51a166fa9bceac7495b57d68b313a3 Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 19 Aug 2026 14:10:34 +0800 Subject: [PATCH 16/27] docs(operations): add an English counterpart of the heal/scanner audit baseline (#6261) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs(operations): land the heal/scanner MinIO audit baseline with closure results Move the comprehensive heal/scanner vs MinIO analysis (2026-08-16) into docs/operations/ so it finally enters the tree — the docs/ root is ignored by the gitignore whitelist, which is why the baseline the audit issue referenced as "to be merged with a PR" never landed. Append §9 closure results: all 14 backlog sub-issues (#1865-#1878) closed with the per-item PR map, two further misjudgment corrections (HS-17 was already implemented; HS-14's MinIO idle semantics drifted upstream), HS-12/HS-18 audit conclusions, and the registered follow-ups. Backlog issue: rustfs/backlog#1862 Co-Authored-By: heihutu * docs(operations): add an English counterpart of the audit baseline Rename the Chinese analysis to *_zh.md (matching the repo's bilingual convention of scanner-excess-alerts.md / _zh.md) and add a full English translation at the original path, cross-linked at the top of both files. Co-Authored-By: heihutu --------- Signed-off-by: houseme Co-authored-by: heihutu --- ...minio-comprehensive-analysis-2026-08-16.md | 804 ++++++++---------- ...io-comprehensive-analysis-2026-08-16_zh.md | 568 +++++++++++++ 2 files changed, 939 insertions(+), 433 deletions(-) create mode 100644 docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md diff --git a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md index 598cd8988..d696b6871 100644 --- a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md +++ b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md @@ -1,566 +1,504 @@ -# RustFS heal / scanner 全量功能分析与 MinIO 对标（v2） +# RustFS heal & scanner vs MinIO — comprehensive parity analysis (v2, 2026-08-16) -- 日期：2026-08-16（基于 main 分支当日代码，审计时 HEAD ≈ `a118d7e4f`） -- 范围：`crates/heal`（src 19,560 行 + tests 2,274 行）、`crates/scanner`（src 约 26,000 行 + tests）、`crates/data-usage`、`crates/ecstore` 中 heal/heal_walk/bitrot_self_verify 与 config、`crates/common/src/heal_channel.rs`、`crates/madmin`（heal/scanner wire 类型）、`rustfs/src`（startup wiring、admin handlers、集群 RPC） -- 对标基线：minio/minio master（HEAD `7aac2a2c5b`，仓库已进入维护模式，master 冻结，即最终态） -- 方法：四路并行审计（heal crate / scanner crate / ecstore 集成层 / MinIO 源码研究），关键结论逐条人工抽验（文内标注"已亲验"处为一手验证） -- 本文档取代 `docs/rustfs-heal-scanner-vs-minio-parity-assessment.md`（2026-06-15 v1）。v1 之后 heal/scanner 相关提交超过 80 个（换盘自动修复全链路、resume 状态机、usage 收敛权威化、集群级 heal 协调、ILM restore 语义等），v1 的功能清单与差距判断已全面过时；v1 中"bloom filter 缺失"等结论经本次核实为**误判**（详见 §5.4）。 +> English | [中文版](rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md) + +- Date: 2026-08-16 (based on that day's `main` code; audit HEAD ≈ `a118d7e4f`) +- Scope: `crates/heal` (src 19,560 lines + tests 2,274 lines), `crates/scanner` (src ~26,000 lines + tests), `crates/data-usage`, the heal/heal_walk/bitrot_self_verify and config parts of `crates/ecstore`, `crates/common/src/heal_channel.rs`, `crates/madmin` (heal/scanner wire types), `rustfs/src` (startup wiring, admin handlers, cluster RPC) +- Parity baseline: minio/minio master (HEAD `7aac2a2c5b`; the repo has entered maintenance mode with master frozen, i.e. its final state) +- Method: four parallel audit tracks (heal crate / scanner crate / ecstore integration layer / MinIO source study), with key conclusions verified by hand one by one (points marked "verified first-hand" below were checked against the source directly) +- This document supersedes `docs/rustfs-heal-scanner-vs-minio-parity-assessment.md` (2026-06-15, v1). Since v1 there have been more than 80 heal/scanner commits (the full automatic drive-replacement healing chain, the resume state machine, making usage convergence authoritative, cluster-level heal coordination, ILM restore semantics, etc.), so v1's feature inventory and gap judgments are comprehensively outdated; v1 conclusions such as "bloom filter missing" were verified this round to be **misjudgments** (see §5.4). --- -## 0. 结论摘要 +## 0. Conclusion summary -1. **总体判断：heal 与 scanner 的核心功能链路已经完整**。对象级 heal（quorum 仲裁 + ETag 兜底 + bitrot Deep 校验 + dangling 处理）、erasure set 深扫（per-set disk-walk 并集枚举）、按版本断点续扫（schema 化持久层 + CAS 原子发布 + 崩溃窗口补齐）、换盘自动修复（readiness 校验 + 身份围栏 + durable intent + completion proof）、scanner 周期循环（leader lock + 持久化 leader-epoch 围栏）、data usage 统计（桶级/集群级、主+备+观测快照、epoch/cycle 防回退）、ILM 全动作（expiry/transition/noncurrent/free-version/delete-marker 清理）、admin Start/Query/Cancel 协议（clientToken 语义对齐 madmin）——以上均有实现且带回归测试。两个 crate 内**没有空实现/早退桩**，异常路径全部有日志 + 指标 + 错误语义。 -2. **主要缺口集中在"入口与观测面"，而不是修复算法本身**：MRF/ECDecode/Metadata 三类任务执行体已实现但无生产触发入口（`HealEvent` 完全未接线）；`CheckAbandonedParts` 在 ecstore 三层全部 `NotImplemented`；heal/scanner trace 通道缺失；scanner 超限 S3 事件缺失；madmin 客户端方法缺失（只有 wire 类型）；heal 字节级进度/ETA 未实现。 -3. **与 v1 认知的重要修正**：bloom filter 在 MinIO 当前 master **已删除**（`.bloomcycle.bin` 只存 cycle 计数），RustFS 现状与 MinIO 一致；MinIO scanner 同样是**集群级 leader 单例**，RustFS 的 leader.lock 模型与 MinIO 同型；RustFS 的 ETag 多数派兜底仲裁已实现（`crates/ecstore/src/set_disk/ops/heal.rs:525-567,679`，已亲验），v1 担心的仲裁缺口不存在。 -4. **RustFS 在多处超出 MinIO**：remote_scanner RPC 协议（远端 peer 本地扫描而非 leader 跨网读远盘）、持久化 leader-epoch CAS 围栏、周期预算与 per-set/per-disk 并发闸、pending-heal 账本、durable replacement intent + completion proof 状态机、前台压力门控（mainline throttle）、集群 heal control coordinator + envelope 重放防护。 -5. 差距分级统计：P1（行为/运维对齐缺口）8 项，P2（完善性）9 项，P3（清理/低风险）3 项，"按设计不追平"7 项。完整清单见 §6。 +1. **Overall verdict: the core functional chains of heal and scanner are complete.** Object-level heal (quorum arbitration + ETag fallback + bitrot Deep verification + dangling handling), erasure set deep scans (per-set disk-walk union enumeration), per-version resumable scans (schema'd persistence layer + CAS atomic publish + crash-window backfill), automatic drive-replacement healing (readiness validation + identity fencing + durable intent + completion proof), the scanner cycle loop (leader lock + persisted leader-epoch fence), data usage statistics (bucket-level/cluster-level, primary + backup + observed snapshots, epoch/cycle anti-rollback), the full ILM action set (expiry/transition/noncurrent/free-version/delete-marker cleanup), and the admin Start/Query/Cancel protocol (clientToken semantics aligned with madmin) — all of these are implemented and carry regression tests. There are **no empty implementations / early-return stubs** inside the two crates; every exceptional path has logs + metrics + error semantics. +2. **The main gaps concentrate on "entry points and the observability surface", not on the repair algorithms themselves**: the MRF/ECDecode/Metadata task executors are implemented but have no production trigger entry (`HealEvent` is entirely unwired); `CheckAbandonedParts` is `NotImplemented` at all three ecstore layers; the heal/scanner trace channels are missing; scanner excess S3 events are missing; madmin client methods are missing (only wire types exist); heal byte-level progress/ETA is not implemented. +3. **Important corrections to the v1 understanding**: the bloom filter has been **removed** from current MinIO master (`.bloomcycle.bin` stores only a cycle count), so RustFS's current state matches MinIO; the MinIO scanner is likewise a **cluster-level leader singleton**, and RustFS's leader.lock model is the same shape as MinIO's; RustFS's ETag majority-fallback arbitration is already implemented (`crates/ecstore/src/set_disk/ops/heal.rs:525-567,679`, verified first-hand) — the arbitration gap v1 worried about does not exist. +4. **RustFS exceeds MinIO in several places**: the remote_scanner RPC protocol (remote peers scan locally instead of the leader reading remote drives across the network), the persisted leader-epoch CAS fence, cycle budgets and per-set/per-disk concurrency gates, the pending-heal ledger, the durable replacement intent + completion proof state machine, foreground pressure gating (mainline throttle), and the cluster heal control coordinator + envelope replay protection. +5. Gap severity tally: 8 P1 items (behavioral/operational alignment gaps), 9 P2 items (completeness), 3 P3 items (cleanup/low risk), and 7 items of "not pursuing parity by design". Full list in §6. --- -## 1. 架构总览 +## 1. Architecture overview -### 1.1 RustFS 三层架构 +### 1.1 RustFS's three-layer architecture -RustFS 把 MinIO 在 `cmd/` 内单体的 heal/scanner 拆成三层 + 两个独立 crate： +RustFS splits the heal/scanner functionality that MinIO keeps inside the `cmd/` monolith into three layers plus two standalone crates: -| 层 | 位置 | 职责 | +| Layer | Location | Responsibilities | |---|---|---| -| 原语层 | `crates/ecstore/src/set_disk/ops/heal.rs`（~3,240 行）、`ops/heal_walk.rs`、`ops/bitrot_self_verify.rs`；上层封装 `store/heal.rs`、`store/heal_walk.rs`、`core/sets.rs` | 对象/桶/format/替换盘格式修复、disk-walk 并集枚举、写入路径 bitrot 自校验；由 `SetDisks`/`Sets`/`ECStore` 实现 `rustfs_storage_api::HealOperations` 契约（`crates/storage-api/src/object.rs:503-519`） | -| heal 运行时 | `crates/heal` | 进程级 HealManager（优先级队列/调度器/auto disk scanner/断点续传 resume）、HealChannelProcessor（消费全局 heal channel）、换盘替换恢复状态机 | -| scanner 运行时 | `crates/scanner` | 数据使用扫描、ILM 评估与入队、heal 候选生产、复制用量统计、remote scanner RPC | -| 共享协议 | `crates/common/src/heal_channel.rs`（~776 行） | Start/Query/Cancel 命令通道、`HealOpts`/`HealScanMode`/`HealRequestSource`/`HealAdmission*` 共享类型、`HealResultItem`（madmin） | -| 共享数据 | `crates/data-usage` | `DataUsageEntry/Info`、直方图、`hash_path`；scanner 产生、ecstore/admin 消费 | +| Primitives layer | `crates/ecstore/src/set_disk/ops/heal.rs` (~3,240 lines), `ops/heal_walk.rs`, `ops/bitrot_self_verify.rs`; upper wrappers `store/heal.rs`, `store/heal_walk.rs`, `core/sets.rs` | Object/bucket/format/replacement-drive format repair, disk-walk union enumeration, write-path bitrot self-verification; the `rustfs_storage_api::HealOperations` contract is implemented by `SetDisks`/`Sets`/`ECStore` (`crates/storage-api/src/object.rs:503-519`) | +| heal runtime | `crates/heal` | Process-level HealManager (priority queue/scheduler/auto disk scanner/resumable resume), HealChannelProcessor (consumes the global heal channel), drive-replacement recovery state machine | +| scanner runtime | `crates/scanner` | Data usage scanning, ILM evaluation and enqueueing, heal candidate production, replication usage statistics, remote scanner RPC | +| Shared protocol | `crates/common/src/heal_channel.rs` (~776 lines) | Start/Query/Cancel command channel, `HealOpts`/`HealScanMode`/`HealRequestSource`/`HealAdmission*` shared types, `HealResultItem` (madmin) | +| Shared data | `crates/data-usage` | `DataUsageEntry/Info`, histograms, `hash_path`; produced by the scanner, consumed by ecstore/admin | -启动链路（已亲验 wiring）： +Startup chain (wiring verified first-hand): -1. `rustfs/src/startup_services.rs:93` → `init_background_service_runtime(store)`。 -2. `rustfs/src/startup_background.rs:41-81`：创建全局 heal 服务取消令牌；读 `RUSTFS_SCANNER_ENABLED`（别名 `RUSTFS_ENABLE_SCANNER`，默认 true）与 `RUSTFS_HEAL_ENABLED`（别名 `RUSTFS_ENABLE_HEAL`，默认 true）；**只要 heal 或 scanner 任一开启就初始化 heal manager**（scanner 产生的 heal 候选需要消费端；两者都关时 heal channel 不初始化，`send_heal_request` 报 "Heal channel not initialized"）。 -3. `crates/heal/src/lib.rs:142-216`：owned task 内原子初始化（caller 取消不会遗留半初始化 manager，`lib.rs:123-131`；`GLOBAL_HEAL_RUNTIME_INIT` 互斥单飞）→ `HealManager::start()` → `rustfs_common::heal_channel::init_heal_channels()` → spawn `HealChannelProcessor::start_with_receipts`。 -4. `crates/heal/src/heal/manager.rs:1301-1356` `HealManager::start`：`start_scheduler()`（`manager.rs:2394-2461`，interval 默认 10s + `Notify` 事件驱动唤醒）→ `process_unclean_shutdown()`（`manager.rs:1362-1695`）→ `enable_auto_heal`（默认 true）时 `start_auto_disk_scanner()`（`manager.rs:2464-2999`）。 -5. server ready 后 `rustfs/src/startup_lifecycle.rs:150-152`：`enable_scanner` 时 `init_data_scanner(token, store)`（`crates/scanner/src/scanner.rs:1293-1372`）。 -6. 优雅停机：`rustfs/src/startup_shutdown.rs:308` `shutdown_ahm_services()`（取消令牌）；`:414` `clear_unclean_shutdown_markers()`。 +1. `rustfs/src/startup_services.rs:93` → `init_background_service_runtime(store)`. +2. `rustfs/src/startup_background.rs:41-81`: create the global heal service cancel token; read `RUSTFS_SCANNER_ENABLED` (alias `RUSTFS_ENABLE_SCANNER`, default true) and `RUSTFS_HEAL_ENABLED` (alias `RUSTFS_ENABLE_HEAL`, default true); **the heal manager is initialized whenever either heal or scanner is enabled** (heal candidates produced by the scanner need a consumer; with both off, the heal channel is not initialized and `send_heal_request` reports "Heal channel not initialized"). +3. `crates/heal/src/lib.rs:142-216`: atomic initialization inside an owned task (a caller cancel cannot leave a half-initialized manager behind, `lib.rs:123-131`; `GLOBAL_HEAL_RUNTIME_INIT` mutex single-flight) → `HealManager::start()` → `rustfs_common::heal_channel::init_heal_channels()` → spawn `HealChannelProcessor::start_with_receipts`. +4. `crates/heal/src/heal/manager.rs:1301-1356` `HealManager::start`: `start_scheduler()` (`manager.rs:2394-2461`, interval default 10s + `Notify` event-driven wakeup) → `process_unclean_shutdown()` (`manager.rs:1362-1695`) → when `enable_auto_heal` (default true), `start_auto_disk_scanner()` (`manager.rs:2464-2999`). +5. After the server is ready, `rustfs/src/startup_lifecycle.rs:150-152`: when `enable_scanner`, `init_data_scanner(token, store)` (`crates/scanner/src/scanner.rs:1293-1372`). +6. Graceful shutdown: `rustfs/src/startup_shutdown.rs:308` `shutdown_ahm_services()` (cancel token); `:414` `clear_unclean_shutdown_markers()`. -### 1.2 MinIO 对应结构（master 最终态） +### 1.2 MinIO's corresponding structure (final master state) -| MinIO 文件 | 职责 | +| MinIO file | Responsibilities | |---|---| -| `cmd/admin-heal-ops.go` | 手动 admin heal 序列（healSequence、clientToken/forceStart/forceStop） | -| `cmd/global-heal.go` | 常驻后台 heal 队列（newBgHealSequence，token 固定 `0000-…`，永不结束）+ `healErasureSet`（逐 set 全量对象 heal） | -| `cmd/background-heal-ops.go` | healRoutine worker 池（`_MINIO_HEAL_WORKERS`，默认 GOMAXPROCS/2）消费 healTask | -| `cmd/mrf.go` | MRF（Most Recent Fail）队列（容量 100,000），进程退出时持久化 `.minio.sys/buckets/.heal/mrf/list.bin` 并启动回放 | -| `cmd/background-newdisks-heal-ops.go` | 新盘/换盘自动 resync（monitorLocalDisksAndHeal 10s 轮询 + healFreshDisk + healingTracker） | -| `cmd/erasure-healing.go` / `erasure-healing-common.go` | 对象级 heal 核心（~800 行）、listAndHeal | -| `cmd/data-scanner.go` | scanner 循环（globalLeaderLock 集群单例）+ folderScanner + applyActions | -| `cmd/erasure.go`（nsScanner）/ `erasure-server-pool.go` | NSScanner 三层结构 | -| `cmd/bucket-lifecycle.go` | ILM 执行器（expiry/transition worker 池） | -| `cmd/xl-storage.go` | DiskInfo.Healing、CheckParts/VerifyFile、CleanAbandonedData、RenameData healing 分支 | -| `cmd/prepare-storage.go` | waitForFormatErasure 新盘启动握手 | +| `cmd/admin-heal-ops.go` | Manual admin heal sequence (healSequence, clientToken/forceStart/forceStop) | +| `cmd/global-heal.go` | Resident background heal queue (newBgHealSequence, token fixed `0000-…`, never ends) + `healErasureSet` (full-object heal per set) | +| `cmd/background-heal-ops.go` | healRoutine worker pool (`_MINIO_HEAL_WORKERS`, default GOMAXPROCS/2) consuming healTask | +| `cmd/mrf.go` | MRF (Most Recent Fail) queue (capacity 100,000), persisted at process exit to `.minio.sys/buckets/.heal/mrf/list.bin` with startup replay | +| `cmd/background-newdisks-heal-ops.go` | Automatic resync for new/replaced drives (monitorLocalDisksAndHeal 10s polling + healFreshDisk + healingTracker) | +| `cmd/erasure-healing.go` / `erasure-healing-common.go` | Object-level heal core (~800 lines), listAndHeal | +| `cmd/data-scanner.go` | Scanner loop (globalLeaderLock cluster singleton) + folderScanner + applyActions | +| `cmd/erasure.go` (nsScanner) / `erasure-server-pool.go` | NSScanner three-layer structure | +| `cmd/bucket-lifecycle.go` | ILM executor (expiry/transition worker pools) | +| `cmd/xl-storage.go` | DiskInfo.Healing, CheckParts/VerifyFile, CleanAbandonedData, RenameData healing branch | +| `cmd/prepare-storage.go` | waitForFormatErasure new-drive startup handshake | -### 1.3 架构级差异（设计取舍，非缺陷） +### 1.3 Architecture-level differences (design trade-offs, not defects) -1. **heal 队列模型**：MinIO 所有 heal（scanner 抽样/MRF/admin/新盘 resync）汇入单 channel + 固定 worker 池（新盘 resync 另有 per-drive worker 池）；RustFS 是优先级堆 + 去重合并 + 容量分级丢弃 + per-set bulkhead + 前台压力门控的多策略调度器（`manager.rs:3003-3420`）。RustFS 表达力更强，代价是"重复请求被合并"的可观测性问题（v1 已指出，现有 `HealAdmissionReceipt` canonical task_id + alias 机制回应了它，`manager.rs:1759-1846`）。 -2. **scanner 远端盘访问**：MinIO leader 通过磁盘抽象层透明读写远端节点磁盘；RustFS leader 通过 remote_scanner RPC 把扫描执行下放到远端 peer 本地进行（`crates/scanner/src/remote_scanner.rs`），只回传结果与进度心跳。两者都是集群单 leader。RustFS 方案省 leader↔远端的元数据读放大，代价是需要维护独立 RPC 协议（HMAC 逐帧认证、会话重放缓存、fence 复验，`remote_scanner.rs:52-61,405-496,1024-1065`）。 -3. **heal 状态持久化**：MinIO 用单文件 `.healing.bin`（msgp healingTracker，diskID 不匹配即重置）；RustFS 用 schema 化多文件（resume/checkpoint/intent/seal/proof 各自 CAS 发布，`resume.rs:38-61`），崩溃窗口显式补齐（`erasure_healer.rs:389-402`、`resume.rs:1027-1057`）。 -4. **写路径自保护**：MinIO 写入后靠后台 heal 收敛；RustFS 在 PutObject/CompleteMultipartUpload 提交 rename 后主动检查 `convergence.needs_heal()` 并立即入队对象 heal（`set_disk/ops/object.rs:2291-2306`、`ops/multipart.rs:2574-2589`），另有读修复 read repair（`io_primitives.rs:1040-1160`）。 +1. **heal queue model**: MinIO funnels every heal (scanner sampling/MRF/admin/new-disk resync) into a single channel + a fixed worker pool (new-disk resync additionally has a per-drive worker pool); RustFS is a multi-policy scheduler built from a priority heap + dedup-merge + capacity-tiered dropping + per-set bulkhead + foreground pressure gating (`manager.rs:3003-3420`). RustFS is more expressive, at the cost of an observability question around "duplicate requests being merged" (already pointed out in v1; the current `HealAdmissionReceipt` canonical task_id + alias mechanism answers it, `manager.rs:1759-1846`). +2. **scanner remote-drive access**: the MinIO leader transparently reads and writes remote-node drives through the disk abstraction layer; the RustFS leader pushes scan execution down to the remote peer to run locally via the remote_scanner RPC (`crates/scanner/src/remote_scanner.rs`), with only results and progress heartbeats sent back. Both are cluster single-leader. RustFS's approach saves the leader↔remote metadata read amplification, at the cost of maintaining a separate RPC protocol (HMAC per-frame authentication, session replay cache, fence re-validation, `remote_scanner.rs:52-61,405-496,1024-1065`). +3. **heal state persistence**: MinIO uses a single file `.healing.bin` (msgp healingTracker, reset whenever the diskID mismatches); RustFS uses a schema'd multi-file layout (resume/checkpoint/intent/seal/proof, each CAS-published, `resume.rs:38-61`), with the crash window explicitly backfilled (`erasure_healer.rs:389-402`, `resume.rs:1027-1057`). +4. **write-path self-protection**: MinIO relies on background heal to converge after writes; RustFS, after the commit rename in PutObject/CompleteMultipartUpload, actively checks `convergence.needs_heal()` and immediately enqueues an object heal (`set_disk/ops/object.rs:2291-2306`, `ops/multipart.rs:2574-2589`), and additionally has read repair (`io_primitives.rs:1040-1160`). --- -## 2. Heal 已实现功能全景 +## 2. Heal implemented-feature panorama -### 2.1 任务类型（`HealType`，`crates/heal/src/heal/task.rs:85-111`） +### 2.1 Task types (`HealType`, `crates/heal/src/heal/task.rs:85-111`) -| 类型 | 语义 | 执行体 | 生产触发方 | +| Type | Semantics | Executor | Production trigger | |---|---|---|---| -| `Cluster` | 所有 bucket 依次 heal（结构 + 可选递归对象），批内重试 ≤3 | `heal_cluster` task.rs:1420-1490 | channel：bucket 为空即 Cluster（channel.rs:576-577） | -| `Object{bucket,object,version_id}` | 单对象/版本；不存在时按 `recreate_missing` 重建或报错 | `heal_object` task.rs:855-1146 | admin、scanner、read-repair、写路径收敛、add_partial | -| `Bucket{bucket}` | 桶元数据/结构；`recursive` 再遍历全部对象版本 | `heal_bucket` task.rs:1284-1418 + `heal_bucket_objects` task.rs:1508-1698 | admin（POST /v3/heal/{bucket}）、scanner `build_bucket_heal_request` | -| `Prefix{bucket,prefix}` | 按前缀递归 | `heal_prefix` task.rs:1492-1506 | channel：`recursive && prefix` 非空（channel.rs:578-585） | -| `ErasureSet{buckets,set_disk_id}` | format 修复 + healing 标记 + 逐桶预处理 + 可恢复逐版本深扫 | `heal_erasure_set` task.rs:2158-2642 | admin（pool/set 参数）、auto disk scanner、unclean shutdown、renew_disk、durable replacement 恢复 | -| `Metadata{bucket,object}` | 仅元数据（Deep、不重建数据） | `heal_metadata` task.rs:1700-1859 | **无生产触发方**（§6 HS-01） | -| `MRF{meta_path}` | 失败路径驱动的 Deep 修复（recursive+update_parity） | `heal_mrf` task.rs:1861-1992 | **无生产触发方**（仅 `HealEvent` 可生成，未接线） | -| `ECDecode{bucket,object,version_id}` | EC 解码重建（Deep+recreate+update_parity），Urgent 优先级 | `heal_ec_decode` task.rs:1994-2156 | **无生产触发方**（仅 `HealEvent` 可生成，未接线） | +| `Cluster` | all buckets healed in turn (structure + optional recursive objects), in-batch retry ≤3 | `heal_cluster` task.rs:1420-1490 | channel: empty bucket means Cluster (channel.rs:576-577) | +| `Object{bucket,object,version_id}` | single object/version; when absent, rebuild per `recreate_missing` or error out | `heal_object` task.rs:855-1146 | admin, scanner, read-repair, write-path convergence, add_partial | +| `Bucket{bucket}` | bucket metadata/structure; `recursive` additionally walks all object versions | `heal_bucket` task.rs:1284-1418 + `heal_bucket_objects` task.rs:1508-1698 | admin (POST /v3/heal/{bucket}), scanner `build_bucket_heal_request` | +| `Prefix{bucket,prefix}` | recursive by prefix | `heal_prefix` task.rs:1492-1506 | channel: `recursive && prefix` non-empty (channel.rs:578-585) | +| `ErasureSet{buckets,set_disk_id}` | format repair + healing marker + per-bucket preprocessing + resumable per-version deep scan | `heal_erasure_set` task.rs:2158-2642 | admin (pool/set params), auto disk scanner, unclean shutdown, renew_disk, durable replacement recovery | +| `Metadata{bucket,object}` | metadata only (Deep, does not rebuild data) | `heal_metadata` task.rs:1700-1859 | **no production trigger** (§6 HS-01) | +| `MRF{meta_path}` | failure-path-driven Deep repair (recursive+update_parity) | `heal_mrf` task.rs:1861-1992 | **no production trigger** (only `HealEvent` can generate it, unwired) | +| `ECDecode{bucket,object,version_id}` | EC decode rebuild (Deep+recreate+update_parity), Urgent priority | `heal_ec_decode` task.rs:1994-2156 | **no production trigger** (only `HealEvent` can generate it, unwired) | -优先级 `Low/Normal/High/Urgent`（task.rs:168-179）；状态机 `Pending/Running/Retrying/Completed/Failed/Cancelled/Timeout`（task.rs:225-241）。 +Priorities `Low/Normal/High/Urgent` (task.rs:168-179); state machine `Pending/Running/Retrying/Completed/Failed/Cancelled/Timeout` (task.rs:225-241). -### 2.2 触发路径全景（admin 之外） +### 2.2 Trigger-path panorama (beyond admin) -| 通道 | source | 优先级 | 证据 | +| Channel | source | Priority | Evidence | |---|---|---|---| -| Scanner 周期抽样（1/1024，`RUSTFS_HEAL_OBJECT_SELECT_PROB`） | Scanner | Low | `scanner_folder.rs:2117-2136`、`:1150`；`remove_corrupted=HEAL_DELETE_DANGLING(true)`、`recreate_missing=false`（`common/heal_channel.rs:24`、`scanner_folder.rs:510-511`） | -| Scanner 元数据损坏（get_size 失败分类 HealMetadata） | Scanner | High | `scanner_folder.rs:2147-2208`、`:1244-1260` | -| Scanner abandoned children（缓存有、盘上无，list_path_raw quorum 核查） | Scanner | High（桶级+对象级） | `scanner_folder.rs:2528-2792` | -| Scanner pending-heal 账本重试（heal 通道满被拒后持久化，每桶每轮 ≤128 条、上限 10k） | Scanner | 原优先级 | `scanner_folder.rs:1721-1763`、`:99-100` | -| auto disk scanner（unformatted 盘经 replacement_readiness 确认 / `runtime_state=="returning"` 盘 / durable intent 重入） | AutoHeal | Low | `manager.rs:2464-2999` | -| unclean shutdown 恢复（启动读 `unclean-shutdown` 标记 → 全部本地 set ErasureSet heal） | AutoHeal | Low | `manager.rs:1362-1695` | -| 写路径收敛（PutObject/CompleteMultipartUpload 后 `convergence.needs_heal()`） | Internal | Normal | `set_disk/ops/object.rs:2291-2306`、`ops/multipart.rs:2574-2589` | -| 部分对象 heal（add_partial） | Internal | Normal | `set_disk/ops/object.rs:5808-5825` | -| 旧数据目录清理残留 enqueue | Internal | Normal | `set_disk/core/io_primitives.rs:3880-3907` | -| 读修复（metadata_read_error / missing_shards / decode_error，TTL 去重缓存） | ReadRepair | Low | `set_disk/read.rs:407,995,1079` → `submit_read_repair_heal`（`io_primitives.rs:1105-1160`），`recreate_missing=true` | -| 盘重连遇 UnformattedDisk → send_heal_disk | AutoHeal | Normal | `set_disk/ops/locking.rs:339-347` | -| Admin API（含集群 coordinator 路由） | Admin | High | `rustfs/src/admin/handlers/heal.rs:174-212`、`:771-930` | -| 集群 RPC heal（peer 调用） | — | — | `rustfs/src/storage/rpc/node_service/heal.rs`、`ecstore/src/cluster/rpc/peer_s3_client.rs:296,1209` | +| Scanner periodic sampling (1/1024, `RUSTFS_HEAL_OBJECT_SELECT_PROB`) | Scanner | Low | `scanner_folder.rs:2117-2136`, `:1150`; `remove_corrupted=HEAL_DELETE_DANGLING(true)`, `recreate_missing=false` (`common/heal_channel.rs:24`, `scanner_folder.rs:510-511`) | +| Scanner metadata corruption (get_size failure classified HealMetadata) | Scanner | High | `scanner_folder.rs:2147-2208`, `:1244-1260` | +| Scanner abandoned children (present in cache, absent on disk, list_path_raw quorum verification) | Scanner | High (bucket-level + object-level) | `scanner_folder.rs:2528-2792` | +| Scanner pending-heal ledger retry (persisted after rejection by a full heal channel, ≤128 per bucket per round, 10k cap) | Scanner | original priority | `scanner_folder.rs:1721-1763`, `:99-100` | +| auto disk scanner (unformatted drive confirmed via replacement_readiness / `runtime_state=="returning"` drive / durable-intent re-entry) | AutoHeal | Low | `manager.rs:2464-2999` | +| unclean shutdown recovery (startup reads the `unclean-shutdown` marker → ErasureSet heal for all local sets) | AutoHeal | Low | `manager.rs:1362-1695` | +| write-path convergence (after PutObject/CompleteMultipartUpload, `convergence.needs_heal()`) | Internal | Normal | `set_disk/ops/object.rs:2291-2306`, `ops/multipart.rs:2574-2589` | +| partial-object heal (add_partial) | Internal | Normal | `set_disk/ops/object.rs:5808-5825` | +| stale data-directory cleanup leftover enqueue | Internal | Normal | `set_disk/core/io_primitives.rs:3880-3907` | +| read repair (metadata_read_error / missing_shards / decode_error, TTL dedup cache) | ReadRepair | Low | `set_disk/read.rs:407,995,1079` → `submit_read_repair_heal` (`io_primitives.rs:1105-1160`), `recreate_missing=true` | +| drive reconnect hits UnformattedDisk → send_heal_disk | AutoHeal | Normal | `set_disk/ops/locking.rs:339-347` | +| Admin API (incl. cluster coordinator routing) | Admin | High | `rustfs/src/admin/handlers/heal.rs:174-212`, `:771-930` | +| cluster RPC heal (peer invocation) | — | — | `rustfs/src/storage/rpc/node_service/heal.rs`, `ecstore/src/cluster/rpc/peer_s3_client.rs:296,1209` | -注意：MinIO 的 MRF 通道（读路径检出 part 缺失/损坏即时投递 + 队列持久化 + shutdown 回放，`cmd/mrf.go`、`erasure-object.go:395-410,800-812`）在 RustFS 由 read-repair + 写路径收敛**部分替代**；`HealType::MRF`/`ECDecode`/`Metadata` 三个执行体没有生产入口（详见 §6 HS-01）。 +Note: MinIO's MRF channel (read-path immediate delivery on missing/corrupt parts + queue persistence + shutdown replay, `cmd/mrf.go`, `erasure-object.go:395-410,800-812`) is **partially replaced** in RustFS by read-repair + write-path convergence; the three executors `HealType::MRF`/`ECDecode`/`Metadata` have no production entry (see §6 HS-01 for details). -### 2.3 对象级 heal 语义（ecstore `set_disk/ops/heal.rs`） +### 2.3 Object-level heal semantics (ecstore `set_disk/ops/heal.rs`) -流程（`heal_object_with_explicit_version_regen` :426 起）： +Flow (`heal_object_with_explicit_version_regen` from :426): -1. 取对象写锁（除非 `no_lock`）；`object` 以 `/` 结尾走对象目录 heal（`heal_object_dir_locked` :1587-1717：dangling 判定 + `remove` 删除 + 缺 volume 重建）。 -2. `read_all_fileinfo` 全盘读 xl.meta，全部 not-found 视为已删除返回。 -3. **quorum 仲裁 + ETag 兜底**（已亲验）：`list_online_disks` 以 mod-time quorum 为准；quorum 失效时回退 ETag 多数派仲裁（`:525-567` `filter_by_etag`/`quorum_etag`）；`pick_valid_fileinfo` 选 canonical 元数据；"meta 坏盘数 > parity" 的 cannotHeal 判定在 ETag 全盘一致时豁免（`:679`）。与 MinIO `filterDisksByETag` 双仲裁一致。 -4. `disks_with_all_parts`（:562-572）按 `scan_mode` 校验 part：**Normal 仅 stat（CheckParts 语义），Deep 做全量 bitrot 校验（VerifyFile 语义）**；Normal 扫描检出 `FileCorrupt` 自动升级 Deep 重试一次（`:2022-2031`，与 MinIO erasure-healing.go:1101-1106 同型）；无 parity 对象（EC:0）bitrot 失败判不可恢复（`:700-726`）。 -5. `should_heal_object_on_disk`（:606-650）逐盘分类 missing/corrupt/offline/outdated → 重建：per-part bitrot reader/writer（用 per-part checksum + 算法）、写临时卷后 rename 提交（`HEAL_RENAME_INCOMPLETE` 重试语义 :24）；dangling 删除安全检查 `dangling_delete_safety`（:1488）；**孤儿数据目录回收 `reclaim_orphan_data_dirs_best_effort`（:1428）**——这部分覆盖了 MinIO `CleanAbandonedData` 的主场景（但无独立 `CheckAbandonedParts` API，见 §6 HS-02）。 -6. 版本化对象：枚举"每个版本"（`storage.rs:1494-1530`）；delete-marker 路径由 `latest_meta.deleted` 决定（`storage.rs:262-277` 注释）；回归测试 `tests/heal_b5_versioned_regression_test.rs:282,334`。 -7. 显式版本重建 `try_regenerate_explicit_version_meta`（:1318）；transitioned 对象本地残留清理。 -8. 写入路径另有 shard 级 bitrot 自校验 `verify_written_bitrot_shards`（`ops/bitrot_self_verify.rs:45-129`，HighwayHash256S，最终 rename 前校验刚写出的 shard，服务 EC:0 无 parity 场景）——**注意这不是后台 bitrot 巡检**；后台巡检由 scanner bitrot_cycle 驱动 Deep heal 承担。 +1. Take the object write lock (unless `no_lock`); an `object` ending with `/` goes through object-directory heal (`heal_object_dir_locked` :1587-1717: dangling determination + `remove` deletion + missing-volume rebuild). +2. `read_all_fileinfo` reads xl.meta from all disks; all-not-found is treated as already deleted and returns. +3. **quorum arbitration + ETag fallback** (verified first-hand): `list_online_disks` treats the mod-time quorum as authoritative; when quorum fails it falls back to ETag majority arbitration (`:525-567` `filter_by_etag`/`quorum_etag`); `pick_valid_fileinfo` picks the canonical metadata; the cannotHeal determination for "number of bad-meta disks > parity" is waived when the ETag agrees across all disks (`:679`). Matches MinIO's dual arbitration in `filterDisksByETag`. +4. `disks_with_all_parts` (:562-572) validates parts per `scan_mode`: **Normal only stats (CheckParts semantics), Deep does full bitrot verification (VerifyFile semantics)**; when a Normal scan detects `FileCorrupt` it automatically escalates to Deep and retries once (`:2022-2031`, same shape as MinIO erasure-healing.go:1101-1106); a no-parity object (EC:0) with a bitrot failure is judged unrecoverable (`:700-726`). +5. `should_heal_object_on_disk` (:606-650) classifies each disk as missing/corrupt/offline/outdated → rebuild: per-part bitrot reader/writer (using per-part checksum + algorithm), write into a temporary volume then rename to commit (`HEAL_RENAME_INCOMPLETE` retry semantics :24); dangling-deletion safety check `dangling_delete_safety` (:1488); **orphan data-directory reclamation `reclaim_orphan_data_dirs_best_effort` (:1428)** — this part covers the main scenarios of MinIO's `CleanAbandonedData` (but there is no standalone `CheckAbandonedParts` API, see §6 HS-02). +6. Versioned objects: enumerate "every version" (`storage.rs:1494-1530`); the delete-marker path is decided by `latest_meta.deleted` (`storage.rs:262-277` comment); regression tests `tests/heal_b5_versioned_regression_test.rs:282,334`. +7. Explicit-version rebuild `try_regenerate_explicit_version_meta` (:1318); cleanup of local leftovers of transitioned objects. +8. The write path additionally has shard-level bitrot self-verification `verify_written_bitrot_shards` (`ops/bitrot_self_verify.rs:45-129`, HighwayHash256S, verifying freshly written shards right before the final rename, serving the EC:0 no-parity case) — **note this is not background bitrot patrol**; background patrol is carried by scanner bitrot_cycle-driven Deep heal. -heal crate 侧包装（`task.rs:855-1146`）：存在性检查（瞬时错误转 `TransientSkip` 不误判失败 :551-569）；scanner 合成目录规范化（:1148-1180）；`recreate_missing` 重建（:1183-1282）；data-usage-cache 对象锁超时豁免（:571-653）；not-found → treated_as_deleted 成功（:1012-1029）；结果 `HealResultItem` 保留至多 1024 条 + truncated 标志（:50,845-852）。 +heal-crate-side wrapper (`task.rs:855-1146`): existence check (transient errors become `TransientSkip` to avoid false failures :551-569); scanner synthetic-directory normalization (:1148-1180); `recreate_missing` rebuild (:1183-1282); data-usage-cache object-lock timeout exemption (:571-653); not-found → treated_as_deleted success (:1012-1029); results `HealResultItem` keep at most 1024 entries + truncated flag (:50,845-852). -递归遍历（`heal_bucket_objects` task.rs:1508-1698）：分页枚举全部版本含 delete marker、瞬时错误指数退避重试 ≤3（2^n + 抖动 :620-627）、失败样本日志截断 ≤5 条、聚合 `BatchHealFailure`。 +Recursive walk (`heal_bucket_objects` task.rs:1508-1698): paginated enumeration of all versions including delete markers, transient-error exponential-backoff retry ≤3 (2^n + jitter :620-627), failure-sample log truncation ≤5 entries, aggregated `BatchHealFailure`. -### 2.4 erasure set heal 与断点续扫 +### 2.4 erasure set heal and resumable scans -`heal_erasure_set`（task.rs:2158-2642）四阶段（4 步进度跟踪）： +`heal_erasure_set` (task.rs:2158-2642) runs in four phases (4-step progress tracking): -1. **替换意图与恢复盘选择**（仅 AutoHeal + heal_endpoints 非空）：复用 durable intent 所在盘 / 排除目标端点选幸存盘；已完成代（CleanupPending）幂等收尾。 -2. **格式修复**：`heal_replacement_format(dry_run, pool, set, targets)`（`storage.rs:1372-1384`，trait 默认实现 fail-closed）；逐目标盘结果必须全 ok（`erasure_healer.rs:97-102`）+ 身份围栏复核（task.rs:2410-2420）。 -3. **healing 标记**：对目标盘写 owner CAS 标记 `{set_disk_id}:{task_id}`（`mod.rs:80-229`，CAS + 回滚 + 并发唯一 owner），使 `DiskInfo.healing` 为真（已亲验赋值链 `set_disk/mod.rs:4988`）。 -4. **逐桶预处理 + 可恢复深扫**：`ErasureSetHealer::heal_erasure_set`（`erasure_healer.rs:242-278`）。 +1. **Replacement intent and recovery-drive selection** (AutoHeal only + non-empty heal_endpoints): reuse the drive holding the durable intent / exclude the target endpoints and pick surviving drives; already-completed generations get an idempotent CleanupPending wrap-up. +2. **Format repair**: `heal_replacement_format(dry_run, pool, set, targets)` (`storage.rs:1372-1384`, trait default fail-closed); per-target-drive results must all be ok (`erasure_healer.rs:97-102`) + identity-fence re-check (task.rs:2410-2420). +3. **healing marker**: write an owner CAS marker `{set_disk_id}:{task_id}` to the target drive (`mod.rs:80-229`, CAS + rollback + unique concurrent owner), which makes `DiskInfo.healing` true (assignment chain verified first-hand `set_disk/mod.rs:4988`). +4. **Per-bucket preprocessing + resumable deep scan**: `ErasureSetHealer::heal_erasure_set` (`erasure_healer.rs:242-278`). -`ErasureSetHealer` 扫描细节（对标 MinIO `healErasureSet`，`heal_walk.rs:15-23` 模块注释明确引用 MinIO `global-heal.go` 的 listPathRaw + objQuorum=1 + mergeXLV2Versions）： +`ErasureSetHealer` scan details (benchmarked against MinIO `healErasureSet`; the `heal_walk.rs:15-23` module comment explicitly cites MinIO `global-heal.go`'s listPathRaw + objQuorum=1 + mergeXLV2Versions): -- **枚举器选择（backlog#920）**：Deep 或 AutoHeal → per-set **disk-walk 并集枚举** `list_versions_for_heal_page_disk_walk`（"任意盘上存在"即 sub-quorum 可重建；`storage.rs:1559-1644`，页界 1000 对象/10,000 版本，`dw1:` cursor）；普通请求走 read-quorum `list_object_versions`。 -- **续扫游标**：权威 cursor 为 opaque continuation token（`v1:`=marker JSON、`dw1:`=disk-walk key，两命名空间互斥防误读，`storage.rs:81-260`）；每完成一页先持久化 cursor 再清 dedup 集合（`erasure_healer.rs:922-927`）。 -- **页内并发**：FuturesUnordered + Semaphore，默认 `RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY=8`，Deep/AutoHeal 强制 1（`erasure_healer.rs:105-142`）。 -- **per-version dedup**：`compose_key` 长度前缀注入编码（`resume.rs:281-288`）。 -- **错误分类**：真缺席（FileNotFound 等）→ Absent（计成功）；基础设施瞬时（quorum/DiskNotFound/SlowDown 等）→ Transient（计 skipped）；其余 Failed（`erasure_healer.rs:148-182`，注释引 backlog#856/#799 B7：离线盘不得记 healed/absent）。 -- **防死循环**：空页 truncated 或页尾版本身份不前进即中止（:933-949）。 -- **完成判定**：failed/skipped/failed_buckets 任一 >0 不标记完成，`schedule_retry()` 复位 resume+checkpoint 两层（:561-626，backlog#855/B6/#1033：skip 轮不得标记完成）。 -- **替换盘提交证据**：目标端点物理回读 `replacement_targets_have_version`（`ops/heal.rs:340-412`），未确认 → transient skip。 +- **Enumerator choice (backlog#920)**: Deep or AutoHeal → per-set **disk-walk union enumeration** `list_versions_for_heal_page_disk_walk` ("exists on any drive" means sub-quorum reconstructible; `storage.rs:1559-1644`, page bounds 1,000 objects/10,000 versions, `dw1:` cursor); ordinary requests go through read-quorum `list_object_versions`. +- **Resume cursor**: the authoritative cursor is an opaque continuation token (`v1:` = marker JSON, `dw1:` = disk-walk key; the two namespaces are mutually exclusive against misreads, `storage.rs:81-260`); after each completed page, persist the cursor first, then clear the dedup set (`erasure_healer.rs:922-927`). +- **In-page concurrency**: FuturesUnordered + Semaphore, default `RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY=8`, Deep/AutoHeal forces 1 (`erasure_healer.rs:105-142`). +- **per-version dedup**: `compose_key` length-prefix injection encoding (`resume.rs:281-288`). +- **Error classification**: truly absent (FileNotFound etc.) → Absent (counted as success); infrastructure-transient (quorum/DiskNotFound/SlowDown etc.) → Transient (counted as skipped); everything else Failed (`erasure_healer.rs:148-182`; the comment cites backlog#856/#799 B7: offline drives must not be recorded healed/absent). +- **Loop protection**: abort when an empty page is truncated or the page-tail version identity does not advance (:933-949). +- **Completion determination**: if any of failed/skipped/failed_buckets is >0, do not mark complete; `schedule_retry()` resets both the resume and checkpoint layers (:561-626; backlog#855/B6/#1033: a skip round must not be marked complete). +- **Replacement-drive commit proof**: physical read-back on the target endpoints `replacement_targets_have_version` (`ops/heal.rs:340-412`); unconfirmed → transient skip. -### 2.5 换盘自动修复（replacement recovery） +### 2.5 Automatic drive-replacement healing (replacement recovery) -- **识别**（`replacement_readiness.rs:25-73`）：`replacement_mount_lease_root()` 存在、canonicalize 成功、是挂载点、物理设备 id 非空、与根设备不相交、不与兄弟盘共享物理设备（Linux 用 /proc/self/mountinfo mount-id+dev+ino）。非 root 挂载检查有回归测试（`manager.rs:3549`）。 -- **状态机**（`resume.rs:63-73`）：`Intent → Rebuilding →（写 proof）Verified → CleanupPending → 清理`；`Abandoned` 终态；跨状态迁移先写持久层再变更（`save_state_strict`）。 -- **持久化**（`resume.rs:38-61`，schema ResumeState=5/Checkpoint=5/proof=1）：`{task_id}_ahm_resume_state.json`、`_ahm_checkpoint.json`、`buckets/ahm-replacement/` 命名空间下 intent/seal/completion_proof；torn write + 无 seal 可识别并原子重建（:1316-1338）；CAS 发布、拒绝覆盖并发有效 proof（:1512-1585）。 -- **恢复**：unclean shutdown 与周期扫描都从幸存盘恢复未完成/待清理替换代（`manager.rs:1435-1640,2663-2815`）；多代冲突/校验失败 → 冻结该 set（`replacement_recovery_blocked_sets`，`manager.rs:69-87,2782-2815`）。 -- **对外快照**：`current_replacement_recovery_snapshot`（`lib.rs:262-333`）合并本地幸存盘记录，冲突 → Unknown/非 definitive；admin `GET /v4/heal/replacement-recovery`。 +- **Identification** (`replacement_readiness.rs:25-73`): `replacement_mount_lease_root()` exists, canonicalize succeeds, is a mount point, the physical device id is non-empty, disjoint from the root device, and shares no physical device with sibling drives (Linux uses /proc/self/mountinfo mount-id+dev+ino). The non-root mount check has a regression test (`manager.rs:3549`). +- **State machine** (`resume.rs:63-73`): `Intent → Rebuilding → (write proof) Verified → CleanupPending → cleanup`; `Abandoned` is a terminal state; state transitions write the persistence layer first, then mutate (`save_state_strict`). +- **Persistence** (`resume.rs:38-61`, schema ResumeState=5/Checkpoint=5/proof=1): `{task_id}_ahm_resume_state.json`, `_ahm_checkpoint.json`, and intent/seal/completion_proof under the `buckets/ahm-replacement/` namespace; torn write + no seal is recognizable and rebuilt atomically (:1316-1338); CAS publish, refuses to overwrite a concurrently valid proof (:1512-1585). +- **Recovery**: both unclean shutdown and the periodic scan recover unfinished/pending-cleanup replacement generations from surviving drives (`manager.rs:1435-1640,2663-2815`); multi-generation conflict / validation failure → freeze that set (`replacement_recovery_blocked_sets`, `manager.rs:69-87,2782-2815`). +- **External snapshot**: `current_replacement_recovery_snapshot` (`lib.rs:262-333`) merges local surviving-drive records; conflict → Unknown / non-definitive; admin `GET /v4/heal/replacement-recovery`. -### 2.6 调度器（manager.rs） +### 2.6 Scheduler (manager.rs) -- 优先级堆 + 同优先级 FIFO（:148-191,330-347）；dedup key 按类型（:469-506）；入队三态查重 active→queued→retrying（:1759-1785）；重复默认 Merged 并返回 canonical task_id（`HealAdmissionReceipt`，:1821-1846）+ client token alias（:1219-1246）。 -- 容量：队列满时 best-effort 来源（Scanner/AutoHeal/ReadRepair）或低优先级被 Dropped(QueueFull)；Admin/Internal 可驱逐低优先级排队项（`push_displacing_lower_priority` :353-396）；80%/95% 压力分级（:885-909）。 -- 并发：全局 `max_concurrent_heals`（默认 4）+ per-set bulkhead `max_concurrent_per_set`（默认 1）（:3040-3073,3434-3447）。 -- 前台压力门控 mainline throttle：前台读/写 permit 利用率 ≥80% 时延迟 best-effort 任务（:919-1009,2999-3020）。 -- 超时：任务级聚合超时（默认 300s），跨重试保留剩余预算（task.rs:444-451，PR #6101）。 -- 可恢复重试：`is_recoverable_heal()`（error.rs:83-136）≤3 次、2^n 退避封顶 30s；retry 在独立 backoff task 中持有所有权（:3235-3382）。 -- 完成态保留 10 分钟供查询（:42）。 +- Priority heap + FIFO within the same priority (:148-191,330-347); dedup key per type (:469-506); enqueue three-state dedup active→queued→retrying (:1759-1785); duplicates default to Merged and return the canonical task_id (`HealAdmissionReceipt`, :1821-1846) + client token alias (:1219-1246). +- Capacity: when the queue is full, best-effort sources (Scanner/AutoHeal/ReadRepair) or low-priority items get Dropped(QueueFull); Admin/Internal may evict queued lower-priority items (`push_displacing_lower_priority` :353-396); 80%/95% tiered pressure handling (:885-909). +- Concurrency: global `max_concurrent_heals` (default 4) + per-set bulkhead `max_concurrent_per_set` (default 1) (:3040-3073,3434-3447). +- Foreground pressure gating, mainline throttle: delay best-effort tasks when foreground read/write permit utilization is ≥80% (:919-1009,2999-3020). +- Timeout: task-level aggregate timeout (default 300s), remaining budget preserved across retries (task.rs:444-451, PR #6101). +- Recoverable retry: `is_recoverable_heal()` (error.rs:83-136) ≤3 attempts, 2^n backoff capped at 30s; retries hold ownership inside a standalone backoff task (:3235-3382). +- Completion states are retained for 10 minutes for querying (:42). -### 2.7 Admin API 与集群协调 +### 2.7 Admin API and cluster coordination -- 路由（`rustfs/src/admin/handlers/heal.rs:174-212`）：`POST /rustfs/admin/v3/heal/`、`/heal/{bucket}`、`/heal/{bucket}/{prefix}`（同一 POST 按 query `clientToken/forceStart/forceStop` 区分 start/query/cancel，与 mc admin heal 语义对齐）；`POST /v3/background-heal/status`；`GET /v4/heal/replacement-recovery`。权限 `HealAdminAction`（route_policy.rs:334-341）。 -- 集群协调（heal.rs:771-930 + `node_service.rs:514-606`）：`heal_topology_fingerprint` + 按拓扑确定性选 coordinator 节点 + coordinator epoch；envelope 校验 + SHA256 digest 重放缓防重放；coordinator 非本机走 peer gRPC `heal_control`；`probe_heal_control` 能力探测（滚动升级场景）。 -- 请求：body 为 `HealOpts`（`recursive/dryRun/remove/recreate/scanMode(0/1/2)/updateParity/nolock/pool/set`，serde camelCase，与 madmin.HealOpts 字段对齐）；根 heal start 需 `recursive=true` 或 `pool+set` 成对；body 上限 1MB。 -- 响应：`HealStartSuccess{clientToken, clientAddress, startTime}`；`HealTaskStatus{summary, detail, startTime, settings, items, truncated, progress}`（summary ∈ running/finished/stopped/notFound）；`BackgroundHealStatus`（bitrot 起始时间/周期/当前模式 + `disabled/uninitialized/idle/active/degraded` 状态——peer 不可达显式 degraded 不冒充 idle，issue #5850 + `healOperations` 按优先级×来源矩阵 + 集群进度）。 -- `HealResultItem`/`HealDriveInfo`/`HealItemType`/DriveState 枚举与 madmin JSON 兼容（`crates/madmin/src/heal_commands.rs:19-65`）。 -- 状态 payload 超 8MiB 对折截断（channel.rs:37,73-104）；path-token 校验（错误 token 拒绝，空 path 仅匹配 Cluster）。 +- Routes (`rustfs/src/admin/handlers/heal.rs:174-212`): `POST /rustfs/admin/v3/heal/`, `/heal/{bucket}`, `/heal/{bucket}/{prefix}` (the same POST distinguishes start/query/cancel by the query `clientToken/forceStart/forceStop`, aligned with mc admin heal semantics); `POST /v3/background-heal/status`; `GET /v4/heal/replacement-recovery`. Permission `HealAdminAction` (route_policy.rs:334-341). +- Cluster coordination (heal.rs:771-930 + `node_service.rs:514-606`): `heal_topology_fingerprint` + deterministic-by-topology coordinator-node selection + coordinator epoch; envelope validation + SHA256 digest replay protection; when the coordinator is not local, go through peer gRPC `heal_control`; `probe_heal_control` capability probe (rolling-upgrade scenario). +- Request: the body is `HealOpts` (`recursive/dryRun/remove/recreate/scanMode(0/1/2)/updateParity/nolock/pool/set`, serde camelCase, fields aligned with madmin.HealOpts); a root heal start requires `recursive=true` or a `pool+set` pair; body cap 1MB. +- Response: `HealStartSuccess{clientToken, clientAddress, startTime}`; `HealTaskStatus{summary, detail, startTime, settings, items, truncated, progress}` (summary ∈ running/finished/stopped/notFound); `BackgroundHealStatus` (bitrot start time/cycle/current mode + `disabled/uninitialized/idle/active/degraded` states — an unreachable peer is explicitly degraded rather than impersonating idle, issue #5850) + `healOperations` as a priority×source matrix + cluster progress. +- `HealResultItem`/`HealDriveInfo`/`HealItemType`/DriveState enums are JSON-compatible with madmin (`crates/madmin/src/heal_commands.rs:19-65`). +- A status payload over 8MiB is truncated by halving (channel.rs:37,73-104); path-token validation (wrong token rejected; an empty path matches Cluster only). -### 2.8 heal 指标与日志 +### 2.8 heal metrics and logs -指标：`rustfs_heal_admission_total{source,result,reason,context}`、`rustfs_heal_task_start_total`、`rustfs_heal_task_running{type,set}`、`rustfs_heal_queue_delay_seconds`、`rustfs_heal_scheduler_skip_total`、`rustfs_heal_mainline_throttle_total`、`rustfs_heal_page_concurrency_current{set}`、`rustfs_heal_candidate_enqueue/merge/drop/priority_reject_total`、`rustfs_heal_read_repair_dedup_total{reason}` 等。日志全部结构化 event style（PR #5720）；per-object 日志降级防风暴（`demote_to_debug_when!`，#5716/#5719/#5727）。 +Metrics: `rustfs_heal_admission_total{source,result,reason,context}`, `rustfs_heal_task_start_total`, `rustfs_heal_task_running{type,set}`, `rustfs_heal_queue_delay_seconds`, `rustfs_heal_scheduler_skip_total`, `rustfs_heal_mainline_throttle_total`, `rustfs_heal_page_concurrency_current{set}`, `rustfs_heal_candidate_enqueue/merge/drop/priority_reject_total`, `rustfs_heal_read_repair_dedup_total{reason}`, etc. All logs are structured event style (PR #5720); per-object logs are demoted to prevent storms (`demote_to_debug_when!`, #5716/#5719/#5727). --- -## 3. Scanner 已实现功能全景 +## 3. Scanner implemented-feature panorama -### 3.1 循环、leader、立即触发 +### 3.1 Loop, leader, immediate triggering -- **集群单 leader**：分布式 ns 写锁 `leader.lock`（`scanner.rs:3156-3207`，超时默认 5s）+ **持久化 leader-epoch CAS 围栏**：leader 用 ETag 前置条件向 `.bloomcycle.bin` 写 `RSCYC001` 编码的 (cycle, leader_epoch)（`scanner.rs:118,1850-1861,2177-2334`）；usage 快照再打 epoch fence（:2087-2153）。锁丢失 → 取消当前周期，30s 收敛（:108-111,2623-2642）。 -- 抢锁后立即执行一轮；周期 = `RUSTFS_SCANNER_CYCLE` > config cycle > start_delay > 部署默认 > 速度档位（±10% 抖动、下限 1s）。 -- **clean-idle 指数退避**：连续完整无脏周期间隔 ×2（封顶 24h；bitrot 周期压缩上限；桶有 lifecycle/replication 活动规则禁用，:383-456,1382-1512）。 -- **superseded/deferred 退避**：5s 起指数退避封顶 30min（:105-106,3432-3438）；维护探测失败独立退避（:459-505）。 -- **立即唤醒**：① dirty-usage 快路径——写路径 put/delete/multipart/bucket 操作调用 `record_dirty_usage_bucket`（`scanner_io.rs:222-235`；调用点 `rustfs/src/app/object_usecase.rs:6221` 等），自增 generation 并 Notify 唤醒 leader，脏桶优先排队（`scanner_io.rs:462-488`）；② 维护配置变更（lifecycle/replication 设置时 `record_scanner_maintenance_change`）；③ 运行时配置热更 generation+Notify；④ 集群活动快照变化。 -- **集群协调**：`probe_scanner_activity` 汇集本机+peer 的 `ScannerNodeActivity`（instance_id/namespace_generation/maintenance_generation/protocol_version/topology_digest/data_movement_active/dirty usage），拓扑摘要覆盖 pools/sets/drives URL，协议版本不齐拒绝共享缓存锁（`scanner.rs:970-1068`）；**数据迁移（rebalance/decommission）期间推迟周期**（`scanner_io.rs:2226-2374`）；周期结束逐 peer RPC 确认 dirty-usage ack（`scanner.rs:2925-2952`）。 +- **Cluster single leader**: distributed ns write lock `leader.lock` (`scanner.rs:3156-3207`, timeout default 5s) + **persisted leader-epoch CAS fence**: the leader writes (cycle, leader_epoch) encoded as `RSCYC001` into `.bloomcycle.bin` using an ETag precondition (`scanner.rs:118,1850-1861,2177-2334`); usage snapshots additionally carry an epoch fence (:2087-2153). Lock lost → cancel the current cycle, converging within 30s (:108-111,2623-2642). +- One round executes immediately after the lock is acquired; cycle = `RUSTFS_SCANNER_CYCLE` > config cycle > start_delay > deployment default > speed tier (±10% jitter, floor 1s). +- **clean-idle exponential backoff**: consecutive fully-clean idle intervals double (capped at 24h; bitrot-cycle compression cap; disabled when a bucket has active lifecycle/replication rules, :383-456,1382-1512). +- **superseded/deferred backoff**: exponential backoff from 5s capped at 30min (:105-106,3432-3438); maintenance probing failures get an independent backoff (:459-505). +- **Immediate wakeup**: ① dirty-usage fast path — write-path put/delete/multipart/bucket operations call `record_dirty_usage_bucket` (`scanner_io.rs:222-235`; call sites include `rustfs/src/app/object_usecase.rs:6221`), bump the generation and Notify-wake the leader; dirty buckets are queued first (`scanner_io.rs:462-488`); ② maintenance-config changes (lifecycle/replication settings call `record_scanner_maintenance_change`); ③ runtime-config hot updates generation+Notify; ④ cluster activity snapshot changes. +- **Cluster coordination**: `probe_scanner_activity` gathers this node's and peers' `ScannerNodeActivity` (instance_id/namespace_generation/maintenance_generation/protocol_version/topology_digest/data_movement_active/dirty usage); the topology digest covers pools/sets/drives URLs; a mismatched protocol version refuses to share the cache lock (`scanner.rs:970-1068`); **cycles are deferred during data movement (rebalance/decommission)** (`scanner_io.rs:2226-2374`); at cycle end, per-peer RPC confirms the dirty-usage ack (`scanner.rs:2925-2952`). -### 3.2 遍历模型 +### 3.2 Traversal model -- 主遍历是**全量目录 walk**（tokio::fs::read_dir 递归，`scanner_folder.rs:1915-2234`），不走 metacache；metacache/`list_path_raw` 仅用于 abandoned children 跨盘核查（:2528-2792）。 -- 三级并发：leader → per-set（信号量默认 4）→ per-disk 桶扫描（默认 4）→ 单盘递归；每桶每 set 缓存锁 `.scanner-cycle.lock.pool-N.set-M`（锁丢失取消该桶扫描，锁竞争重排队）；每盘单扫描准入（本地盘也走信号量，`scanner_io.rs:3246-3274`）。 -- 桶顺序：shuffle 后按 dirty → 未缓存 → 已缓存重排（`scanner_io.rs:2947-2949,462-488`）；目录内按名字排序 + resume 提示旋转（`scanner_folder.rs:333-359`）。 -- **断点续扫**：`DataUsageScanCheckpoint{version,resume_after,reason}` 持久于缓存 info（`data_usage_define.rs:68,293-307`）；预算耗尽/取消写入，恢复有 Used/Stale/NoHint 指标；续扫单位是目录（无跨周期对象级分页）。 -- erasure 语义：发现 `xl.meta` 即对象边界不下钻；UUID data-dir 候选最多探测 64 entry；有数据无元数据 → 记 failed + 高优 heal；symlink 目录忽略/环跳过。 -- 协作让出：每 N 对象（默认 128）`yield_now`。 +- The main traversal is a **full directory walk** (tokio::fs::read_dir recursion, `scanner_folder.rs:1915-2234`), not via metacache; metacache/`list_path_raw` is used only for the abandoned-children cross-drive verification (:2528-2792). +- Three-level concurrency: leader → per-set (semaphore default 4) → per-disk bucket scans (default 4) → single-drive recursion; a cache lock per bucket per set `.scanner-cycle.lock.pool-N.set-M` (losing the lock cancels that bucket's scan; lock contention re-queues); single-scan admission per drive (local drives also go through the semaphore, `scanner_io.rs:3246-3274`). +- Bucket ordering: after shuffle, re-ordered as dirty → uncached → cached (`scanner_io.rs:2947-2949,462-488`); entries within a directory sorted by name + resume-hint rotation (`scanner_folder.rs:333-359`). +- **Resumable scanning**: `DataUsageScanCheckpoint{version,resume_after,reason}` persisted in the cache info (`data_usage_define.rs:68,293-307`); written on budget exhaustion/cancel; resumption has Used/Stale/NoHint metrics; the resume unit is a directory (no cross-cycle object-level pagination). +- Erasure semantics: finding `xl.meta` marks an object boundary with no descent; at most 64 UUID data-dir candidate entries probed; data without metadata → record failed + high-priority heal; symlink directories ignored / cycles skipped. +- Cooperative yielding: `yield_now` every N objects (default 128). -### 3.3 大桶跳过策略（对标 MinIO compaction） +### 3.3 Large-bucket skip strategy (benchmarked against MinIO compaction) -1. 缓存当前性复用：桶与扫描计划未变（name/source/snapshot_complete/plan digest/next_cycle/leader_epoch/cache_key_format 全匹配）整桶跳过（`scanner_io.rs:1062-1109`）。 -2. compacted 目录 16 周期轮换窗口：`hash mod (next_cycle, 16)` 命中才重扫，否则从旧缓存拷贝（`scanner_folder.rs:74,2429-2442`）。 -3. compaction 阈值：子项 <500 或纯对象叶子压缩为单 entry；子文件夹 ≥2500（根 10000）预压缩；children ≥10000 归约（:75-78,2314-2340,2846-2887）。 -4. 失败对象 TTL 跳过：86400s/最多 10000 条（:88-91,1354-1381）。 +1. Cache-currency reuse: if the bucket and scan plan are unchanged (name/source/snapshot_complete/plan digest/next_cycle/leader_epoch/cache_key_format all match), the whole bucket is skipped (`scanner_io.rs:1062-1109`). +2. compacted-directory 16-cycle rotation window: rescan only when `hash mod (next_cycle, 16)` hits, otherwise copy from the old cache (`scanner_folder.rs:74,2429-2442`). +3. compaction thresholds: children <500 or pure-object leaves compress into a single entry; subfolders ≥2500 (root 10000) pre-compressed; children ≥10000 reduced (:75-78,2314-2340,2846-2887). +4. failed-object TTL skip: 86400s / at most 10,000 entries (:88-91,1354-1381). -与 MinIO master 对比：MinIO 的跳过策略同样是 hash-mod-16 周期 + compaction 阈值树（500/10000/2500），**bloom filter 已从 master 删除**。RustFS 的常量与结构与 MinIO 现状同源（MinIO 未采用跨盘 dirty-generation 优先，RustFS 额外多两层跳过——plan digest 与缓存当前性校验）。 +Compared with MinIO master: MinIO's skip strategy is likewise hash-mod-16 cycles + a compaction threshold tree (500/10000/2500), and the **bloom filter has been removed from master**. RustFS's constants and structure share the same origin as MinIO's current state (MinIO does not adopt cross-drive dirty-generation prioritization; RustFS additionally has two more skip layers — plan digest and cache-currency validation). -### 3.4 data usage 统计 +### 3.4 data usage statistics -- 维度：每目录 entry（size/objects/versions/delete_markers/大小直方图/版本直方图/复制统计/failed_objects/per-tier stats/children/compacted，`data-usage/src/data_usage.rs:661-679`）；每对象 SizeSummary（含 per-ARN 复制目标统计、tier 统计，tier 分类：transitioned 完成记入其 tier 否则按 storage class，free version 不计）；桶级 `BucketUsageInfo`；集群级 `DataUsageInfo`（含 scanner_cycle/scanner_epoch 围栏 + usage_snapshot_complete）。 -- 存储：每桶每 set `{bucket}/.usage-cache.bin`（主 + `.bkp` 备份 + CAS 重试）；权威集群快照 `buckets/data-usage/data-usage.json`（每 10 周期同步 `.bkp`，legacy 路径兼容）；陈旧快照拒绝写入（epoch/cycle/last_update 三重判定）；被竞争 superseded 的观测快照另存 `data-usage-observed.json`。 -- 消费：`replace_bucket_usage_memory_from_info` 刷新桶用量内存 + 两层缓存失效（`scanner.rs:4142-4152`）→ bucket stats/quota/admin account_info/system；写路径内存实时叠加 overlay；启动读快照判断冷缓存跳过启动延迟。 -- 未完成 multipart 不参与统计（与 MinIO 一致，MinIO 也不扫 multipart 桶）。 +- Dimensions: per-directory entry (size/objects/versions/delete_markers/size histogram/version histogram/replication stats/failed_objects/per-tier stats/children/compacted, `data-usage/src/data_usage.rs:661-679`); per-object SizeSummary (incl. per-ARN replication-target stats and tier stats; tier classification: fully transitioned counts toward its tier, otherwise by storage class; free versions not counted); bucket-level `BucketUsageInfo`; cluster-level `DataUsageInfo` (incl. scanner_cycle/scanner_epoch fence + usage_snapshot_complete). +- Storage: per bucket per set `{bucket}/.usage-cache.bin` (primary + `.bkp` backup + CAS retry); the authoritative cluster snapshot `buckets/data-usage/data-usage.json` (`.bkp` synced every 10 cycles, legacy path compatible); stale snapshots rejected on write (triple epoch/cycle/last_update determination); observation snapshots superseded by a race are stored separately as `data-usage-observed.json`. +- Consumption: `replace_bucket_usage_memory_from_info` refreshes bucket-usage memory + two-level cache invalidation (`scanner.rs:4142-4152`) → bucket stats/quota/admin account_info/system; the write path overlays memory in real time; at startup, reading the snapshot detects a cold cache and skips startup delay. +- Incomplete multipart uploads are not counted (consistent with MinIO, which also does not scan the multipart bucket). -### 3.5 ILM 集成 +### 3.5 ILM integration -- 每对象 `ScannerItem::apply_actions`（`scanner_folder.rs:747-1032`）：`Evaluator::new(lifecycle).with_lock_retention(...).with_replication_config(...).eval()` 批量评估。 -- 已实现动作（IlmAction 全集，`common/src/metrics.rs:34-45`）：expiry 删除（Delete/DeleteRestored/DeleteRestoredVersion）、全版本删除（DeleteAllVersions/DelMarkerDeleteAllVersions，处理后停止后续版本）、transition（Transition/TransitionVersion，tier 列表运行时读取）、noncurrent 批量（DeleteVersionAction → `enqueue_by_newer_noncurrent`）、free-version 清理（`enqueue_free_version`）、object-lock retention 约束。**与 MinIO 的 9 个 ILM 动作一一对应**。 -- 执行模型：scanner 是"发现与入队"角色（expiry 队列/transition 队列在 ecstore `bucket_lifecycle_ops.rs`），动作由 worker 池消费——与 MinIO globalExpiryState/globalTransitionState 同型。 -- AbortIncompleteMultipartUpload 不在 scanner/ILM 内执行（MinIO 同样不在：`internal/bucket/lifecycle/rule.go` 有 FIXME，实际由 `erasureSets.cleanupStaleUploads` 全局例程承担）；RustFS 由 ecstore 独立后台任务 `init_background_stale_multipart_upload_cleanup`（`bucket_lifecycle_ops.rs:3289-3320`）+ 桶删除时 on-demand。 -- 集成测试覆盖：transition+restore、free-version、noncurrent、delete-marker、0-day、后台扫描过期（`scanner/tests/lifecycle_integration_test.rs:1071-2095`）。 +- Per object `ScannerItem::apply_actions` (`scanner_folder.rs:747-1032`): `Evaluator::new(lifecycle).with_lock_retention(...).with_replication_config(...).eval()` batch evaluation. +- Implemented actions (the full IlmAction set, `common/src/metrics.rs:34-45`): expiry deletes (Delete/DeleteRestored/DeleteRestoredVersion), all-versions deletes (DeleteAllVersions/DelMarkerDeleteAllVersions, stop further versions after handling), transition (Transition/TransitionVersion, tier list read at runtime), noncurrent batches (DeleteVersionAction → `enqueue_by_newer_noncurrent`), free-version cleanup (`enqueue_free_version`), object-lock retention constraints. **A one-to-one mapping onto MinIO's 9 ILM actions.** +- Execution model: the scanner is the "discover and enqueue" role (the expiry/transition queues live in ecstore `bucket_lifecycle_ops.rs`); actions are consumed by worker pools — the same shape as MinIO's globalExpiryState/globalTransitionState. +- AbortIncompleteMultipartUpload is not executed inside scanner/ILM (MinIO likewise: `internal/bucket/lifecycle/rule.go` has a FIXME, and it is actually carried by the `erasureSets.cleanupStaleUploads` global routine); in RustFS it is an independent ecstore background task `init_background_stale_multipart_upload_cleanup` (`bucket_lifecycle_ops.rs:3289-3320`) + on-demand at bucket deletion. +- Integration-test coverage: transition+restore, free-version, noncurrent, delete-marker, 0-day, background-scan expiry (`scanner/tests/lifecycle_integration_test.rs:1071-2095`). -### 3.6 heal 候选生产（scanner 侧） +### 3.6 heal candidate production (scanner side) -- 抽样：`hash mod_alt(next_cycle/prob_div, 1024/prob_div)`，进入 compacted 分支重扫时 prob_div=16 等效概率 ×16（与 MinIO 同款补偿，`scanner_folder.rs:125-127,2117-2122`）。 -- deep/normal：周期级 `get_cycle_scan_mode`（bitrot_cycle 默认 30d，`scanner.rs:1626-1657`）→ 对象级带 `HealScanMode::Deep`；新鲜对象（60s 内修改）降级 Normal（:146-155）；状态持久 `.background-heal.json`（`BackgroundHealInfo{bitrot_start_time,bitrot_start_cycle,current_scan_mode}`，与 MinIO 同路径同结构）。 -- scanner 只入队不内联执行（内联 heal 已移除，兼容旗标仅告警，`scanner_folder.rs:411-427`）；`HealScanMode::Deep` 只是标记，bitrot 校验读发生在 heal 消费端（ecstore Deep 路径）。 -- 元数据损坏 → 高优 heal（`classify_get_size_failure` → HealMetadata）；abandoned children → list_path_raw quorum 核查 + 桶级/对象级高优 heal；healing 盘粘性跳过（`should_heal` :1628-1648）。 -- pending-heal 账本：heal 通道满被拒持久化到缓存 info，下轮重试。 -- 复制 heal：`queue_replication_heal` → replication 队列（走 replication 通道而非 heal channel）；per-ARN 复制用量统计。 +- Sampling: `hash mod_alt(next_cycle/prob_div, 1024/prob_div)`; when rescanning via the compacted branch, prob_div=16 gives an equivalent ×16 probability (the same compensation as MinIO, `scanner_folder.rs:125-127,2117-2122`). +- deep/normal: cycle-level `get_cycle_scan_mode` (bitrot_cycle default 30d, `scanner.rs:1626-1657`) → object-level with `HealScanMode::Deep`; fresh objects (modified within 60s) are demoted to Normal (:146-155); state persisted in `.background-heal.json` (`BackgroundHealInfo{bitrot_start_time,bitrot_start_cycle,current_scan_mode}`, same path and structure as MinIO). +- The scanner only enqueues, never executes inline (inline heal was removed; the compat flag only warns, `scanner_folder.rs:411-427`); `HealScanMode::Deep` is just a marker — the bitrot-verification read happens at the heal consumer (the ecstore Deep path). +- Metadata corruption → high-priority heal (`classify_get_size_failure` → HealMetadata); abandoned children → list_path_raw quorum verification + bucket-level/object-level high-priority heal; healing drives get sticky skipping (`should_heal` :1628-1648). +- pending-heal ledger: candidates rejected by a full heal channel are persisted into the cache info and retried next round. +- Replication heal: `queue_replication_heal` → the replication queue (going through the replication channel, not the heal channel); per-ARN replication usage statistics. -### 3.7 remote_scanner RPC 协议（RustFS 特有） +### 3.7 remote_scanner RPC protocol (RustFS-specific) -请求 ≤16KB msgpack（version/request_id/server_epoch/session_id/session_sequence/bucket/next_cycle/leader_epoch/scan_plan_digest/skip_healing/scan_mode/budget）；帧 ≤2MB、HMAC-SHA256 逐帧认证（域 `rustfs-ns-scanner-frame-v3`）；进度心跳 1s（预算模式 250ms）；阶段播报 Scanning→Persisting；RPC 生命周期上限 24h、断连宽限 2min；防重放 session+sequence 缓存（容量 65536）；服务端校验 leader fence 与持久化 cycle 一致 + 每 5s fence 复验；结果 Complete/Partial/NamespaceNotFound/CycleAhead；不支持 v4 协议的远端盘回退 leader 本地扫描（`remote_scanner.rs` 全文件；`scanner_io.rs:2750-2812`）。 +Requests ≤16KB msgpack (version/request_id/server_epoch/session_id/session_sequence/bucket/next_cycle/leader_epoch/scan_plan_digest/skip_healing/scan_mode/budget); frames ≤2MB, HMAC-SHA256 per-frame authentication (domain `rustfs-ns-scanner-frame-v3`); progress heartbeats 1s (250ms in budget mode); phase announcements Scanning→Persisting; RPC lifetime cap 24h, disconnect grace 2min; anti-replay session+sequence cache (capacity 65536); the server validates leader-fence and persisted-cycle consistency + fence re-validation every 5s; results Complete/Partial/NamespaceNotFound/CycleAhead; remote drives without v4-protocol support fall back to the leader scanning locally (`remote_scanner.rs` whole file; `scanner_io.rs:2750-2812`). -### 3.8 限速/预算/热更/观测 +### 3.8 Rate limiting / budgets / hot updates / observability -- DynamicSleeper 比例退避（速度档 fastest/fast/default/slow/slowest，同 MinIO 五档参数）；idle_mode 总闸；前台 S3 读流量每请求 10ms 封顶 250ms 额外退避。 -- 周期预算 ScannerCycleBudget：max_duration/max_objects/max_directories（默认 0=不限），partial 周期仍推进 cycle 计数。 -- runtime_config 三层来源（env > config > default）逐字段来源标记（Env/Config/ScannerCompatConfig/Default），admin `PUT /v3/config` 热更 → generation+Notify 即时生效；`GET /v3/scanner/status` 返回 enabled/freshness(fresh/stale/unknown)/metrics/cycle_schedule/runtime_config；`GET /v3/ilm/expiry/status` 返回 expiry 队列/worker/missed/blocked。 -- 指标：leader lock、周期 complete/partial/deferred/superseded、versions scanned、per-source（Usage/Lifecycle/BucketReplication/SiteReplication/Heal/Bitrot/Alerts）checked/executed/queued/missed、checkpoint set/used/stale、当前路径（per-disk+bucket 实时）、缓存 save 系列、并发系列、告警（excess versions/version size/folders）。 +- DynamicSleeper proportional backoff (speed tiers fastest/fast/default/slow/slowest, same five-tier parameters as MinIO); idle_mode master switch; an extra backoff capped at 250ms per request (10ms base) driven by foreground S3 read traffic. +- Cycle budget ScannerCycleBudget: max_duration/max_objects/max_directories (default 0 = unlimited); partial cycles still advance the cycle count. +- runtime_config with three-layer sources (env > config > default) and per-field source markers (Env/Config/ScannerCompatConfig/Default); admin `PUT /v3/config` hot update → generation+Notify takes effect immediately; `GET /v3/scanner/status` returns enabled/freshness(fresh/stale/unknown)/metrics/cycle_schedule/runtime_config; `GET /v3/ilm/expiry/status` returns expiry queue/workers/missed/blocked. +- Metrics: leader lock; cycle complete/partial/deferred/superseded; versions scanned; per-source (Usage/Lifecycle/BucketReplication/SiteReplication/Heal/Bitrot/Alerts) checked/executed/queued/missed; checkpoint set/used/stale; current path (per-disk+bucket in real time); cache save series; concurrency series; alerts (excess versions/version size/folders). --- -## 4. 与 MinIO 逐项对标 +## 4. Item-by-item parity versus MinIO -### 4.1 heal 触发通道对照 +### 4.1 heal trigger-channel comparison -| MinIO 通道 | RustFS 对应 | 状态 | +| MinIO channel | RustFS counterpart | Status | |---|---|---| -| A. 手动 admin heal（healSequence，clientToken/forceStart/forceStop） | heal channel Start/Query/Cancel + 集群 coordinator + envelope 重放防护 | ✅ 等价且增强（集群路由）；序列语义差异见 §6 HS-06 | -| B. 常驻后台 heal 队列（newBgHealSequence + healRoutine worker 池） | HealManager 常驻调度器 + 优先级队列 + bulkhead | ✅ 等价且增强 | -| C. 新盘/换盘自动 resync（monitorLocalDisksAndHeal 10s + healFreshDisk + healingTracker + waitForFormatErasure 握手） | auto disk scanner（10s）+ replacement_readiness + durable intent/proof 状态机 + heal_replacement_format | ✅ 等价且增强（identity fence + completion proof；MinIO 的 tracker 面向对外可见性更强，见 §6 HS-07） | -| D. MRF（队列 100k + 持久化 list.bin + shutdown 回放 + 读路径 corrupt 投递） | read-repair（Low+TTL 去重）+ 写路径 convergence heal 部分承担；`HealType::MRF` 执行体无生产入口 | ⚠️ 部分等价（§6 HS-01） | -| E. Scanner 抽样 heal（1/1024 + compacted ×16 补偿）+ abandoned children | 同款抽样 + ×16 补偿 + abandoned children + pending-heal 账本 | ✅ 等价且增强（账本） | -| F. 读路径内联触发 → MRF（GetObject part 缺失/损坏、元数据重建 missingBlocks>0） | read repair（missing_shards/decode_error/metadata_read_error 三入口） | ✅ 等价（入 heal 队列而非 MRF 队列） | +| A. Manual admin heal (healSequence, clientToken/forceStart/forceStop) | heal channel Start/Query/Cancel + cluster coordinator + envelope replay protection | ✅ equivalent and enhanced (cluster routing); sequence-semantics differences in §6 HS-06 | +| B. Resident background heal queue (newBgHealSequence + healRoutine worker pool) | HealManager resident scheduler + priority queue + bulkhead | ✅ equivalent and enhanced | +| C. Automatic new/replaced-drive resync (monitorLocalDisksAndHeal 10s + healFreshDisk + healingTracker + waitForFormatErasure handshake) | auto disk scanner (10s) + replacement_readiness + durable intent/proof state machine + heal_replacement_format | ✅ equivalent and enhanced (identity fence + completion proof; MinIO's tracker is stronger on external visibility, see §6 HS-07) | +| D. MRF (100k queue + persisted list.bin + shutdown replay + read-path corrupt delivery) | read-repair (Low + TTL dedup) + write-path convergence heal carry it partially; the `HealType::MRF` executor has no production entry | ⚠️ partially equivalent (§6 HS-01) | +| E. Scanner sampled heal (1/1024 + compacted ×16 compensation) + abandoned children | the same sampling + ×16 compensation + abandoned children + pending-heal ledger | ✅ equivalent and enhanced (the ledger) | +| F. Read-path inline trigger → MRF (GetObject part missing/corrupt, metadata rebuild missingBlocks>0) | read repair (three entries: missing_shards/decode_error/metadata_read_error) | ✅ equivalent (enqueued into the heal queue rather than the MRF queue) | -### 4.2 对象级 heal 语义对照 +### 4.2 Object-level heal semantics comparison -| 特性 | MinIO | RustFS | 状态 | +| Feature | MinIO | RustFS | Status | |---|---|---|---| -| mod-time quorum 仲裁 | listOnlineDisks | 同 | ✅ | -| ETag 多数派兜底（时钟漂移） | filterDisksByETag | `filter_by_etag`/`quorum_etag`（heal.rs:525-567） | ✅ 已亲验 | -| cannotHeal 的 ETag 豁免 | ETag 全一致豁免重试 | heal.rs:679 | ✅ | -| Normal=CheckParts（stat）/ Deep=VerifyFile（bitrot） | 是 | `disks_with_all_parts` 按 scan_mode（ops/heal.rs:562-572,978-1024） | ✅ | -| Normal 检出 corrupt 自动升 Deep 重试一次 | erasure-healing.go:1101-1106 | ops/heal.rs:2022-2031 | ✅ | -| dangling 判定（not-found > parity）+ 删除审计 | isObjectDangling/deleteIfDangling | `dangling_delete_safety`（:1488）+ scanner HEAL_DELETE_DANGLING | ✅（审计 tags 细节有差异） | -| 孤儿 data-dir/inline 清理（CleanAbandonedData） | CheckAbandonedParts（scanner 抽中 + admin Remove 时显式调用） | heal 路径内 `reclaim_orphan_data_dirs_best_effort`（:1428）；独立 API 三层 NotImplemented | ⚠️ 部分等价（§6 HS-02） | -| 版本化/delete-marker heal | HealObject versionID；nullVersionID 特判 | 逐版本枚举 + delete-marker latest heal（B5 回归） | ✅ | -| 对象级 healing 元数据标记（x-minio-healing，RenameData 跳过版本清理） | 有 | 无对象级标记；依赖盘级 healing.bin + NSLock + rename 语义 | ⚠️ 评估项（§6 HS-12） | -| Distribution/Index 一致性三处防线 | 有（manual modification 拒绝） | 目标盘格式结果全 ok 校验 + 身份围栏 | ✅（粒度不同） | -| 无 parity（EC:0）对象 | bitrot 不可恢复处理 | 判不可恢复（:700-726）+ 写入自校验 | ✅ 增强（写路径自校验） | -| 三层分布不一致拒绝 heal | 有 | heal_walk 归一化 + 页界防御 | ✅（实现方式不同） | -| multipart 孤儿对账 | CheckAbandonedParts 承担 | 显式 NotImplemented（由 lifecycle 清理承担） | ⚠️ §6 HS-02 | -| suspended/decommissioned pool 处理 | IsSuspended 跳过 | deferral 语义（store/heal.rs:192-207，PR #5876） | ✅ | -| heal 与并发删除互斥 | NSLock + healing 标记 | NSLock + 写锁 | ✅ | +| mod-time quorum arbitration | listOnlineDisks | same | ✅ | +| ETag majority fallback (clock drift) | filterDisksByETag | `filter_by_etag`/`quorum_etag` (heal.rs:525-567) | ✅ verified first-hand | +| cannotHeal ETag waiver | waived on all-consistent ETag retry | heal.rs:679 | ✅ | +| Normal=CheckParts (stat) / Deep=VerifyFile (bitrot) | yes | `disks_with_all_parts` by scan_mode (ops/heal.rs:562-572,978-1024) | ✅ | +| Normal detecting corrupt auto-escalates to one Deep retry | erasure-healing.go:1101-1106 | ops/heal.rs:2022-2031 | ✅ | +| dangling determination (not-found > parity) + deletion auditing | isObjectDangling/deleteIfDangling | `dangling_delete_safety` (:1488) + scanner HEAL_DELETE_DANGLING | ✅ (audit-tags details differ) | +| Orphan data-dir/inline cleanup (CleanAbandonedData) | CheckAbandonedParts (invoked explicitly on scanner sampling + admin Remove) | in-heal-path `reclaim_orphan_data_dirs_best_effort` (:1428); standalone API NotImplemented at all three layers | ⚠️ partially equivalent (§6 HS-02) | +| Versioned/delete-marker heal | HealObject versionID; nullVersionID special case | per-version enumeration + delete-marker latest heal (B5 regression) | ✅ | +| Object-level healing metadata marker (x-minio-healing, RenameData skips version cleanup) | yes | no object-level marker; relies on drive-level healing.bin + NSLock + rename semantics | ⚠️ evaluation item (§6 HS-12) | +| Distribution/Index consistency, three lines of defense | yes (manual modification rejected) | target-drive format results all-ok check + identity fence | ✅ (different granularity) | +| no-parity (EC:0) objects | bitrot treated as unrecoverable | judged unrecoverable (:700-726) + write self-verification | ✅ enhanced (write-path self-verification) | +| three-layer distribution inconsistency refuses heal | yes | heal_walk normalization + page-bound defense | ✅ (different implementation approach) | +| multipart orphan reconciliation | carried by CheckAbandonedParts | explicitly NotImplemented (carried by lifecycle cleanup) | ⚠️ §6 HS-02 | +| suspended/decommissioned pool handling | skipped via IsSuspended | deferral semantics (store/heal.rs:192-207, PR #5876) | ✅ | +| heal mutually exclusive with concurrent deletes | NSLock + healing marker | NSLock + write lock | ✅ | -### 4.3 新盘 resync 对照 +### 4.3 new-drive resync comparison -| MinIO | RustFS | 状态 | +| MinIO | RustFS | Status | |---|---|---| -| waitForFormatErasure 四类可恢复错误无限等待握手 | startup 盘解析 + renew_disk 重连路径 | ✅（模型不同：RustFS 不在启动时阻塞等待 format） | -| HealFormat NSLock + errNoHealRequired + refFormat 不一致拒绝 | `heal_format`/`heal_replacement_format` fail-closed + 目标槽位限定（PR #1787 语义） | ✅ 增强 | -| per (pool,set) 分布式锁防并发 resync | set 级队列去重 + bulkhead（manager.rs:2854-2889） | ✅ | -| 全新集群检测（待 heal 盘数==总盘数不触发） | replacement_readiness（独立挂载点/物理设备校验，非 root） | ✅ 增强 | -| healingTracker（.healing.bin：Bytes/Items 计数、QueuedBuckets/HealedBuckets、Resume 快照、RetryAttempts ≤4、HealID 联动、diskID 变更重置） | resume/checkpoint schema 化持久层 + durable intent/proof（per-task 文件，CAS） | ✅ 等价且增强（崩溃窗口补齐）；但**对外快照可见性**弱于 MinIO（§6 HS-07） | -| 跳过 heal 开始后新写入版本（ModTime > Started） | 无同款过滤 | ⚠️ §6 HS-13 | -| 跳过 ILM 已过期版本（filterLifecycle） | 无同款过滤 | ⚠️ §6 HS-13 | -| worker 数 max(GOMAXPROCS,NR)/4 下限 4，heal:drive_workers 覆盖 | 页内并发 8（Deep/AutoHeal 强制 1）+ per-set bulkhead | ✅（参数模型不同） | -| 每 entry waitForLowHTTPReq 让路 | mainline throttle（前台利用率门控） | ✅ 增强 | -| heal 范围含 `.minio.sys/config`、`.minio.sys/buckets` 两个伪桶；最新桶优先 | ErasureSet 任务逐 bucket 预处理（含 meta bucket 语义由 heal_bucket 承担） | ✅（顺序无"最新优先"） | -| 失败整体重试 ≤4 次（resetHealing + errRetryHealing） | schedule_retry 复位双层 + 可恢复重试 ≤3 | ✅ | +| waitForFormatErasure handshake waiting indefinitely on four classes of recoverable errors | startup drive resolution + renew_disk reconnect path | ✅ (different model: RustFS does not block at startup waiting for format) | +| HealFormat NSLock + errNoHealRequired + refFormat-mismatch rejection | `heal_format`/`heal_replacement_format` fail-closed + target-slot restriction (PR #1787 semantics) | ✅ enhanced | +| per (pool,set) distributed lock preventing concurrent resync | set-level queue dedup + bulkhead (manager.rs:2854-2889) | ✅ | +| brand-new-cluster detection (drives-to-heal == total drives does not trigger) | replacement_readiness (independent mount point / physical-device validation, non-root) | ✅ enhanced | +| healingTracker (.healing.bin: Bytes/Items counters, QueuedBuckets/HealedBuckets, Resume snapshot, RetryAttempts ≤4, HealID linkage, diskID-change reset) | resume/checkpoint schema'd persistence + durable intent/proof (per-task files, CAS) | ✅ equivalent and enhanced (crash-window backfill); but **external snapshot visibility** is weaker than MinIO's (§6 HS-07) | +| skip versions written after heal start (ModTime > Started) | no such filter | ⚠️ §6 HS-13 | +| skip ILM-expired versions (filterLifecycle) | no such filter | ⚠️ §6 HS-13 | +| worker count max(GOMAXPROCS,NR)/4 floor 4, heal:drive_workers override | in-page concurrency 8 (Deep/AutoHeal forced to 1) + per-set bulkhead | ✅ (different parameter model) | +| waitForLowHTTPReq yield per entry | mainline throttle (foreground-utilization gating) | ✅ enhanced | +| heal scope includes the two pseudo-buckets `.minio.sys/config` and `.minio.sys/buckets`; newest bucket first | ErasureSet task pre-processes per bucket (meta-bucket semantics carried by heal_bucket) | ✅ (no "newest first" ordering) | +| whole-failure retry ≤4 (resetHealing + errRetryHealing) | schedule_retry resets both layers + recoverable retry ≤3 | ✅ | -### 4.4 scanner 对照 +### 4.4 scanner comparison -| MinIO | RustFS | 状态 | +| MinIO | RustFS | Status | |---|---|---| -| 集群单 leader（globalLeaderLock） | leader.lock + 持久化 leader-epoch CAS 围栏 | ✅ 增强（epoch 围栏防脑裂，MinIO 无持久化 epoch） | -| `.bloomcycle.bin` 只存 cycle（bloom 已删除） | 同路径存 cycle+leader_epoch（RSCYC001） | ✅ 对齐（v1 误判已修正） | -| folderScanner hash-mod-16 + compaction（500/10000/2500） | 同款常量 + plan digest + 缓存当前性校验 + dirty 优先 | ✅ 增强 | -| 每盘扫描并行 ≤GOMAXPROCS；healing 盘排除 | per-set/per-disk 信号量 + healing 盘粘性跳过 | ✅ | -| scannerSleeper（factor 2/max 1s，speed 档热更） | DynamicSleeper 同款 + idle_mode + 前台读退避 | ✅ 增强 | -| idle 语义：`scanner:idle_speed=on`（空闲时段才节流，忙时全速） | `RUSTFS_SCANNER_IDLE_MODE=true`（启用限速总闸） | ⚠️ 语义方向相反，§6 HS-14 | -| applyActions 顺序（heal→ILM→复制→告警） | apply_actions 同序（heal 候选→ILM→复制 heal→告警） | ✅ | -| ILM 9 动作 + 批量评估 + DeletePrefixObject 优化 | 同 9 动作 + 批量评估 + expiry 队列 | ✅（DeleteAllVersions 是否单调用优化未逐行核） | -| abandoned children（listPathRaw minDisks=N/2 发现漏写盘） | list_path_raw + quorum 核查 + 高优 heal | ✅ | -| incomplete multipart 独立例程（6h 间隔/24h 过期，rename 进 .trash） | ecstore 独立后台任务（可配间隔/过期） | ✅（trash 二段清理细节差异，§6 HS-18） | -| usage 维度（size/objects/versions/DM/直方图/复制/tier/bucket 级） | 全覆盖 + 集群快照三重防回退 | ✅ 增强 | -| prefix 级 usage（loadPrefixUsageFromBackend，console 消费） | 缓存内有目录树但仅 flatten 桶级 | ❌ §6 HS-08 | -| 超限事件 s3:ObjectManyVersions/LargeVersions/PrefixManyFolders + 审计 | 仅指标 alert_excess_*（默认 100/1TiB/65538 vs MinIO 100/1TB/50000） | ⚠️ §6 HS-04/HS-17 | -| scanner 指标 v3（bucket_scans/directories/objects/versions/last_activity） | rustfs_scanner_* 全套 + freshness | ✅（命名体系不同） | -| TraceScanner / realtime metrics（mc admin scanner status/trace） | 无 trace 通道；/v3/scanner/status 自有结构 | ⚠️ §6 HS-03 | +| cluster single leader (globalLeaderLock) | leader.lock + persisted leader-epoch CAS fence | ✅ enhanced (epoch fence against split-brain; MinIO has no persisted epoch) | +| `.bloomcycle.bin` stores only the cycle (bloom removed) | same path stores cycle+leader_epoch (RSCYC001) | ✅ aligned (v1 misjudgment corrected) | +| folderScanner hash-mod-16 + compaction (500/10000/2500) | same constants + plan digest + cache-currency validation + dirty-first | ✅ enhanced | +| ≤GOMAXPROCS parallel scans per drive; healing drives excluded | per-set/per-disk semaphores + sticky skip of healing drives | ✅ | +| scannerSleeper (factor 2/max 1s, speed tiers hot-swapped) | DynamicSleeper same + idle_mode + foreground-read backoff | ✅ enhanced | +| idle semantics: `scanner:idle_speed=on` (throttle only in idle windows, full speed when busy) | `RUSTFS_SCANNER_IDLE_MODE=true` (master switch for rate limiting) | ⚠️ opposite semantic direction, §6 HS-14 | +| applyActions order (heal→ILM→replication→alerts) | apply_actions same order (heal candidates→ILM→replication heal→alerts) | ✅ | +| ILM 9 actions + batch evaluation + DeletePrefixObject optimization | same 9 actions + batch evaluation + expiry queue | ✅ (whether DeleteAllVersions has the single-call optimization was not checked line by line) | +| abandoned children (listPathRaw minDisks=N/2 detects under-written drives) | list_path_raw + quorum verification + high-priority heal | ✅ | +| incomplete multipart independent routine (6h interval/24h expiry, rename into .trash) | ecstore independent background task (configurable interval/expiry) | ✅ (trash two-stage cleanup detail differences, §6 HS-18) | +| usage dimensions (size/objects/versions/DM/histograms/replication/tier/bucket level) | full coverage + cluster snapshot with triple anti-rollback | ✅ enhanced | +| prefix-level usage (loadPrefixUsageFromBackend, consumed by console) | the cache holds the directory tree but flattens only to bucket level | ❌ §6 HS-08 | +| excess events s3:ObjectManyVersions/LargeVersions/PrefixManyFolders + auditing | metrics alert_excess_* only (defaults 100/1TiB/65538 vs MinIO 100/1TB/50000) | ⚠️ §6 HS-04/HS-17 | +| scanner metrics v3 (bucket_scans/directories/objects/versions/last_activity) | full rustfs_scanner_* suite + freshness | ✅ (different naming scheme) | +| TraceScanner / realtime metrics (mc admin scanner status/trace) | no trace channel; /v3/scanner/status has its own structure | ⚠️ §6 HS-03 | -### 4.5 admin/CLI/API 面对照 +### 4.5 admin/CLI/API surface comparison -| MinIO | RustFS | 状态 | +| MinIO | RustFS | Status | |---|---|---| -| `POST /minio/admin/v3/heal/...` start/status/cancel | `POST /rustfs/admin/v3/heal/...` 同三态 | ✅（路径前缀不同属预期） | -| `HealStartSuccess`/`HealTaskStatus`/`HealResultItem`/DriveState | 同名字段 JSON 兼容 | ✅ | -| `POST /v3/background-heal/status`（BgHealState 聚合） | 同路径 + degraded 语义 + operations 矩阵 | ✅ 增强（MRF per-endpoint 子状态无，因无 MRF） | -| `GET /v3/healthinfo` 每 drive `HealInfo *HealingDisk` | 无同款 healthinfo heal 字段（replacement-recovery v4 承担部分） | ⚠️ §6 HS-07 | -| madmin 客户端 HealStart/HealStatus/BackgroundHealStatus/ScannerStatus 方法 | 仅 wire 类型，无客户端方法 | ❌ §6 HS-05 | -| mc admin heal --pool/--set、--scan-mode、--force-start/stop | HealOpts 全字段支持（pool/set/scanMode/forceStart/forceStop） | ✅（服务端就绪；缺 mc 侧入口，HS-05） | -| ErrHealAlreadyRunning / ErrHealOverlappingPaths 类型化错误 | 去重合并 + 驱逐语义；无类型化重叠拒绝 | ⚠️ §6 HS-06 | -| 结果 backpressure（maxUnconsumedItems=1000、10s 保活流式、24h 未消费 abort） | 快照式查询（1024 条 + 8MiB 截断 + 10min 保留） | ⚠️ §6 HS-06 | -| `mc support inspect`/healing-bin 离线 dump | 无（inspect.rs 存在但 healing dump 未确认） | ⚠️ P3 | +| `POST /minio/admin/v3/heal/...` start/status/cancel | `POST /rustfs/admin/v3/heal/...` same three states | ✅ (different path prefix is expected) | +| `HealStartSuccess`/`HealTaskStatus`/`HealResultItem`/DriveState | same-named fields JSON-compatible | ✅ | +| `POST /v3/background-heal/status` (BgHealState aggregate) | same path + degraded semantics + operations matrix | ✅ enhanced (no MRF per-endpoint sub-state, because there is no MRF) | +| `GET /v3/healthinfo` per-drive `HealInfo *HealingDisk` | no equivalent healthinfo heal field (replacement-recovery v4 covers part of it) | ⚠️ §6 HS-07 | +| madmin client HealStart/HealStatus/BackgroundHealStatus/ScannerStatus methods | wire types only, no client methods | ❌ §6 HS-05 | +| mc admin heal --pool/--set, --scan-mode, --force-start/stop | HealOpts full field support (pool/set/scanMode/forceStart/forceStop) | ✅ (server-side ready; missing the mc-side entry, HS-05) | +| ErrHealAlreadyRunning / ErrHealOverlappingPaths typed errors | dedup-merge + eviction semantics; no typed overlap rejection | ⚠️ §6 HS-06 | +| result backpressure (maxUnconsumedItems=1000, 10s keep-alive streaming, 24h unconsumed abort) | snapshot-style query (1024 entries + 8MiB truncation + 10min retention) | ⚠️ §6 HS-06 | +| `mc support inspect`/healing-bin offline dump | none (inspect.rs exists but the healing dump is unconfirmed) | ⚠️ P3 | -### 4.6 观测面对照 +### 4.6 observability surface comparison -| 维度 | MinIO | RustFS | 状态 | +| Dimension | MinIO | RustFS | Status | |---|---|---|---| -| heal 指标 | minio_heal_objects_total/heal_total/errors_total/time_last_activity + v3 drive_health 2=healing | rustfs_heal_* 全套（admission/queue delay/running/throttle/page concurrency） | ✅（RustFS 缺 drive_health=healing 单一 gauge 等价物；DiskInfo.healing 已赋值） | -| scanner 指标 | v3 6 个 + realtime 18 项 | rustfs_scanner_* 全套 + per-source 维度 | ✅ | -| ILM 指标 | v3 5 个（expiry/transition pending/active/missed + versions_scanned） | ilm expiry status API + scanner per-source | ✅（指标与 API 形态不同） | -| trace | TraceHealing/TraceScanner 两通道 | 无 | ❌ §6 HS-03 | -| 审计 | HealObject 事件、dangling 删除审计、scanner:manyversions 等 | 结构化日志（event style）+ 指标；无 audit log 事件 | ⚠️ §6 HS-04 | -| 进度 | healingTracker Bytes/Items/QueuedBuckets/当前对象 + usage-cache 总量基线 | HealProgress{scanned/healed/failed/bytes/current_object/percentage}；bytes_processed 注释为 0、estimated_completion_time 恒 None | ⚠️ §6 HS-07 | +| heal metrics | minio_heal_objects_total/heal_total/errors_total/time_last_activity + v3 drive_health 2=healing | full rustfs_heal_* suite (admission/queue delay/running/throttle/page concurrency) | ✅ (RustFS lacks an equivalent of the single drive_health=healing gauge; DiskInfo.healing is already assigned) | +| scanner metrics | v3 6 + realtime 18 items | full rustfs_scanner_* suite + per-source dimensions | ✅ | +| ILM metrics | v3 5 (expiry/transition pending/active/missed + versions_scanned) | ilm expiry status API + scanner per-source | ✅ (different metrics and API shape) | +| trace | TraceHealing/TraceScanner channels | none | ❌ §6 HS-03 | +| auditing | HealObject events, dangling-deletion audit, scanner:manyversions etc. | structured logs (event style) + metrics; no audit-log events | ⚠️ §6 HS-04 | +| progress | healingTracker Bytes/Items/QueuedBuckets/current object + usage-cache total baseline | HealProgress{scanned/healed/failed/bytes/current_object/percentage}; bytes_processed annotated as 0, estimated_completion_time always None | ⚠️ §6 HS-07 | -### 4.7 配置面对照（默认值） +### 4.7 configuration surface comparison (defaults) -| MinIO | RustFS | 备注 | +| MinIO | RustFS | Notes | |---|---|---| -| `heal:bitrotscan`（默认 off；on=每轮；Nm=N×30×24h） | `heal.bitrot_cycle` / `RUSTFS_SCANNER_BITROT_CYCLE_SECS`（默认 30d=2592000s；0/on=每轮 Deep，off=禁用） | ✅ 同语义（RustFS 默认 30d，MinIO 默认 off——**默认值不同**，RustFS 更激进） | -| `heal:max_io=100`/`max_sleep=250ms`（waitForLowIO） | mainline throttle 阈值 80%/80%、max_sleep 250ms | ✅ 同型（阈值模型不同） | -| `heal:drive_workers`（默认 -1 自动） | 页内并发 8 + per-set 1 | ✅ 同型 | -| `_MINIO_HEAL_WORKERS`（GOMAXPROCS/2） | `RUSTFS_HEAL_MAX_CONCURRENT_HEALS=4` + `_MAX_CONCURRENT_PER_SET=1` | ✅ | -| `_MINIO_AUTO_DRIVE_HEALING`（on） | `RUSTFS_HEAL_AUTO_HEAL_ENABLE=true` | ✅ | -| `_MINIO_SCANNER`（on） | `RUSTFS_SCANNER_ENABLED=true` | ✅ | -| `scanner:speed` 五档（default=2x/1s/1m） | 同五档同名同参数 | ✅ | -| `scanner:idle_speed`（on） | `RUSTFS_SCANNER_IDLE_MODE`（true） | ⚠️ 语义方向（HS-14） | +| `heal:bitrotscan` (default off; on=every cycle; Nm=N×30×24h) | `heal.bitrot_cycle` / `RUSTFS_SCANNER_BITROT_CYCLE_SECS` (default 30d=2592000s; 0/on=Deep every cycle, off=disabled) | ✅ same semantics (RustFS default 30d, MinIO default off — **different defaults**, RustFS more aggressive) | +| `heal:max_io=100`/`max_sleep=250ms` (waitForLowIO) | mainline throttle thresholds 80%/80%, max_sleep 250ms | ✅ same shape (different threshold model) | +| `heal:drive_workers` (default -1 auto) | in-page concurrency 8 + per-set 1 | ✅ same shape | +| `_MINIO_HEAL_WORKERS` (GOMAXPROCS/2) | `RUSTFS_HEAL_MAX_CONCURRENT_HEALS=4` + `_MAX_CONCURRENT_PER_SET=1` | ✅ | +| `_MINIO_AUTO_DRIVE_HEALING` (on) | `RUSTFS_HEAL_AUTO_HEAL_ENABLE=true` | ✅ | +| `_MINIO_SCANNER` (on) | `RUSTFS_SCANNER_ENABLED=true` | ✅ | +| `scanner:speed` five tiers (default=2x/1s/1m) | same five tiers, same names, same parameters | ✅ | +| `scanner:idle_speed` (on) | `RUSTFS_SCANNER_IDLE_MODE` (true) | ⚠️ semantic direction (HS-14) | | `scanner:alert_excess_versions=100` | 100 | ✅ | -| `scanner:alert_excess_folders=50000` | 65538（兼容 PBS 布局） | ⚠️ HS-17 | -| `ilm:expiration_workers=100`/`transition_workers=100` | ecstore expiry/transition worker 池（键见 ilm 子系统） | ✅（默认值未逐项核对） | -| `api:stale_upload_cleanup_interval=6h`/`expiry=24h` | ecstore 后台任务 env 可配 | ✅（默认值未逐项核对） | -| —（无） | `RUSTFS_HEAL_QUEUE_SIZE=10000`、`_TASK_TIMEOUT_SECS=300`、`_INTERVAL_SECS=10`、`_LOW_PRIORITY_MERGE/DROP`、`_PAGE_*`、`_SET_BULKHEAD`、`_MAINLINE_*`、`RUSTFS_SCANNER_CYCLE_MAX_*` 预算、`_MAX_CONCURRENT_SET/DISK_SCANS=4`、`_YIELD_EVERY_N_OBJECTS=128` 等 | RustFS 特有（更细粒度） | +| `scanner:alert_excess_folders=50000` | 65538 (compatible with the PBS layout) | ⚠️ HS-17 | +| `ilm:expiration_workers=100`/`transition_workers=100` | ecstore expiry/transition worker pools (keys under the ilm subsystem) | ✅ (defaults not checked item by item) | +| `api:stale_upload_cleanup_interval=6h`/`expiry=24h` | ecstore background task, configurable via env | ✅ (defaults not checked item by item) | +| — (none) | `RUSTFS_HEAL_QUEUE_SIZE=10000`, `_TASK_TIMEOUT_SECS=300`, `_INTERVAL_SECS=10`, `_LOW_PRIORITY_MERGE/DROP`, `_PAGE_*`, `_SET_BULKHEAD`, `_MAINLINE_*`, `RUSTFS_SCANNER_CYCLE_MAX_*` budgets, `_MAX_CONCURRENT_SET/DISK_SCANS=4`, `_YIELD_EVERY_N_OBJECTS=128`, etc. | RustFS-specific (finer-grained) | -### 4.8 RustFS 超出 MinIO 的部分 +### 4.8 Where RustFS exceeds MinIO -1. remote_scanner RPC（扫描执行下放远端 peer 本地，含 HMAC 认证/重放缓存/fence 复验/断连宽限）。 -2. 持久化 leader-epoch CAS 围栏 + usage 快照 epoch/cycle 防回退（MinIO 仅锁，无持久 epoch）。 -3. 周期预算（max_duration/objects/directories）+ partial 周期推进语义。 -4. per-set/per-disk 扫描并发闸 + 每桶每 set 缓存锁。 -5. pending-heal 账本（heal 通道满不丢候选）。 -6. 换盘 durable intent + completion proof 状态机 + 身份围栏（MinIO healingTracker 无 proof）。 -7. mainline throttle 前台压力门控（permit 利用率驱动）。 -8. 集群 heal control coordinator + envelope 重放防护 + degraded 显式降级。 -9. 写路径 shard bitrot 自校验（EC:0 场景）。 -10. dirty-usage 快路径唤醒（写路径即时通知 + 脏桶优先）。 -11. heal 运行时可观测矩阵（优先级×来源 operations snapshot）。 -12. workload admission 联动（heal 调度器读前台压力快照）。 +1. remote_scanner RPC (scan execution pushed down to the remote peer locally, with HMAC authentication/replay cache/fence re-validation/disconnect grace). +2. Persisted leader-epoch CAS fence + usage-snapshot epoch/cycle anti-rollback (MinIO has only the lock, no persisted epoch). +3. Cycle budgets (max_duration/objects/directories) + partial-cycle advancement semantics. +4. per-set/per-disk scan concurrency gates + a cache lock per bucket per set. +5. pending-heal ledger (heal candidates are not lost when the heal channel is full). +6. Drive-replacement durable intent + completion proof state machine + identity fence (MinIO's healingTracker has no proof). +7. mainline throttle foreground pressure gating (driven by permit utilization). +8. Cluster heal control coordinator + envelope replay protection + explicit degraded fallback. +9. Write-path shard bitrot self-verification (the EC:0 case). +10. dirty-usage fast-path wakeup (immediate write-path notification + dirty buckets first). +11. heal runtime observability matrix (priority×source operations snapshot). +12. workload admission integration (the heal scheduler reads the foreground pressure snapshot). --- -## 5. 差距与改进清单 +## 5. Gap and improvement list -分级定义：P1=行为/运维对齐缺口（影响生产运维或工具链兼容）；P2=完善性（功能在但缺一角）；P3=清理/低风险。每项含现状证据、MinIO 行为、影响、建议、验收方式。 +Severity definitions: P1 = behavioral/operational alignment gap (affects production operations or toolchain compatibility); P2 = completeness (the feature exists but is missing a corner); P3 = cleanup/low risk. Each item includes current-state evidence, MinIO behavior, impact, recommendation, and acceptance. -### P1（8 项） +### P1 (8 items) -**HS-01 MRF/ECDecode/Metadata 三类 heal 任务无生产触发入口，HealEvent 未接线** -- 现状：`HealType::MRF/ECDecode/Metadata` 执行体完整（task.rs:1700-2156）但全仓库无生产触发方；`HealEvent`/`HealEventHandler`（event.rs:50-367）crate 外零引用（已亲验 grep）；channel 转换只产生 Cluster/Object/Bucket/Prefix/ErasureSet（channel.rs:566-601）。 -- MinIO：mrf.go 独立 MRF 队列（容量 100k，满丢弃计数）、进程退出 msgp 持久化 `.heal/mrf/list.bin` + 启动回放、入队 <1s 延迟 1s（等网络恢复）、healSleeper 限速；读路径 GetObject part 缺失/损坏、元数据重建 missingBlocks>0、Put 部分成功、DeleteObject、multipart、peer client 共 7+ 投递点。 -- 影响：RustFS 的 read-repair + 写路径收敛覆盖了主场景，但缺少：① 事件驱动的 Urgent ECDecode 重建入口（ecstore 解码失败时目前仅 Low read-repair）；② Metadata-only heal 入口（scanner HealMetadata 分类存在但走普通对象 heal）；③ MRF 队列持久化（重启丢未消费修复意图——scanner pending-heal 账本部分缓解）。 -- 建议：三选一决策——(a) 接线 HealEvent（在 ecstore 解码失败/metadata 损坏点发事件）+ 实现持久化重试账本；(b) 删除 MRF/ECDecode/Metadata 死代码只保留文档说明；(c) 保留执行体、把 HealEvent 降级为内部 API。推荐 (a) 但需先量化 read-repair 是否已覆盖解码失败场景的响应时间要求。 -- 验收：解码失败 → Urgent heal 请求链路 e2e；重启后 pending 修复意图回放；HealEvent 环形缓冲指标。 +**HS-01 The MRF/ECDecode/Metadata heal task types have no production trigger; HealEvent unwired** +- Current state: the `HealType::MRF/ECDecode/Metadata` executors are complete (task.rs:1700-2156) but have no production trigger anywhere in the repo; `HealEvent`/`HealEventHandler` (event.rs:50-367) has zero references outside the crate (verified first-hand by grep); channel conversion produces only Cluster/Object/Bucket/Prefix/ErasureSet (channel.rs:566-601). +- MinIO: mrf.go has a standalone MRF queue (capacity 100k, drop-and-count when full), msgp persistence to `.heal/mrf/list.bin` at process exit + startup replay, 1s delay for enqueues <1s (waiting for network recovery), healSleeper rate limiting; on the read path, GetObject part missing/corrupt, metadata rebuild missingBlocks>0, partial Put success, DeleteObject, multipart, and the peer client add up to 7+ delivery points. +- Impact: RustFS's read-repair + write-path convergence covers the main scenarios, but lacks: ① an event-driven Urgent ECDecode rebuild entry (on ecstore decode failure there is currently only Low read-repair); ② a metadata-only heal entry (the scanner's HealMetadata classification exists but goes through ordinary object heal); ③ MRF queue persistence (unconsumed repair intents are lost on restart — partially mitigated by the scanner's pending-heal ledger). +- Recommendation: a pick-one-of-three decision — (a) wire HealEvent (emit events at ecstore decode-failure/metadata-corruption points) + implement a persistent retry ledger; (b) delete the MRF/ECDecode/Metadata dead code and keep only a documentation note; (c) keep the executors and demote HealEvent to an internal API. (a) is recommended, but first quantify whether read-repair already meets the response-time requirements for decode-failure scenarios. +- Acceptance: an e2e decode-failure → Urgent heal-request chain; replay of pending repair intents after restart; HealEvent ring-buffer metrics. -**HS-02 CheckAbandonedParts 三层 NotImplemented（abandoned data 独立对账入口缺失）** -- 现状：`set_disk/ops/heal.rs:2052-2056`、`core/sets.rs:1144-1148`、`store/heal.rs:258-266` 三层显式 `Err(NotImplemented)`（已亲验），注释"intentionally retained above the set layer until there is a concrete caller"。 -- MinIO：`CheckAbandonedParts` → 每盘 `CleanAbandonedData`：读 xl.meta → 列 UUID data-dir + inline entries → 与 getDataDirs 差集 → 删多余 data-dir/inline 并重写 xl.meta；由 scanner 抽中 heal 与 admin heal Remove 时显式调用。 -- 影响：RustFS heal 路径内 `reclaim_orphan_data_dirs_best_effort`（:1428）覆盖"heal 时回收孤儿目录"，但 ① 无独立触发点（MinIO 在对象未到 heal 阈值时也能清 abandoned data）；② inline data 孤儿条目清理未确认；③ multipart 孤儿对账明确不做（设计决定，由 lifecycle 承担）。 -- 建议：评估把 `reclaim_orphan_data_dirs_best_effort` 提升为 heal_object 固定步骤（若尚非）+ 实现 HealOperations::check_abandoned_parts 真实现（调用同一回收逻辑），或明确文档化"由 lifecycle 承担"并关闭 API 面。 -- 验收：构造 data-dir/inline 孤儿 → scanner 抽样/admin heal 后被清理；三层 API 返回成功或显式 NotSupported 文档化。 +**HS-02 CheckAbandonedParts NotImplemented at all three layers (missing standalone abandoned-data reconciliation entry)** +- Current state: `set_disk/ops/heal.rs:2052-2056`, `core/sets.rs:1144-1148`, `store/heal.rs:258-266` explicitly return `Err(NotImplemented)` at all three layers (verified first-hand); the comment reads "intentionally retained above the set layer until there is a concrete caller". +- MinIO: `CheckAbandonedParts` → per-drive `CleanAbandonedData`: read xl.meta → list UUID data-dirs + inline entries → diff against getDataDirs → delete surplus data-dirs/inline entries and rewrite xl.meta; invoked explicitly on scanner-sampled heals and admin heal Remove. +- Impact: RustFS's in-heal-path `reclaim_orphan_data_dirs_best_effort` (:1428) covers "reclaim orphan directories while healing", but ① there is no standalone trigger point (MinIO can also clean abandoned data before an object reaches the heal threshold); ② orphan inline-data entry cleanup is unconfirmed; ③ multipart orphan reconciliation is explicitly out of scope (a design decision, carried by lifecycle). +- Recommendation: evaluate promoting `reclaim_orphan_data_dirs_best_effort` to a fixed step of heal_object (if it is not already) + implement a real HealOperations::check_abandoned_parts (calling the same reclamation logic), or explicitly document "carried by lifecycle" and close the API surface. +- Acceptance: construct data-dir/inline orphans → cleaned after scanner sampling/admin heal; the three-layer API returns success or an explicitly documented NotSupported. -**HS-03 heal/scanner trace 通道缺失** -- 现状：TraceHealing/TraceScanner 零命中（已亲验 grep 全仓库）。 -- MinIO：`madmin.TraceHealing`（mc admin trace --healing，FuncName=heal.Bucket/heal.Object/heal.CheckAbandonedParts，带 dry/remove/mode/version-id/disks/bytes）、`TraceScanner`（mc admin scanner trace，支持 --filter-size/--response-duration）。 -- 影响：无法实时观测单个 heal/scanner 动作的耗时与参数；排障只能靠指标聚合与日志。 -- 建议：在 heal channel 执行与 scanner folder/item 处理埋点，接入现有 admin trace 订阅面（若 rustfs 已有 trace 基建则复用，无则按 madmin TraceType 扩展）。 -- 验收：mc 等价工具能订阅 heal/scanner trace 流。 +**HS-03 heal/scanner trace channels missing** +- Current state: zero hits for TraceHealing/TraceScanner (verified first-hand by grepping the whole repo). +- MinIO: `madmin.TraceHealing` (mc admin trace --healing, FuncName=heal.Bucket/heal.Object/heal.CheckAbandonedParts, with dry/remove/mode/version-id/disks/bytes), `TraceScanner` (mc admin scanner trace, supports --filter-size/--response-duration). +- Impact: no way to observe in real time the latency and parameters of individual heal/scanner actions; troubleshooting can rely only on aggregated metrics and logs. +- Recommendation: instrument heal-channel execution and scanner folder/item handling, and hook them into the existing admin trace subscription surface (reuse the rustfs trace infrastructure if it exists; otherwise extend it per madmin TraceType). +- Acceptance: an mc-equivalent tool can subscribe to the heal/scanner trace stream. -**HS-04 scanner 超限 S3 事件与审计缺失** -- 现状：仅 `rustfs_scanner_excess_*_total` 指标（versions 100/version size 1TiB/folders 65538）。 -- MinIO：发 `s3:ObjectManyVersions`（>100 版本）、`s3:ObjectLargeVersions`（累计 >1TB）、`s3:PrefixManyFolders`（>50000 子目录）事件（UserAgent: Scanner）+ scanner:manyversions/largeversions/manyprefixes 审计。 -- 影响：依赖事件订阅做容量治理的用户（console/外部审计）收不到告警。 -- 建议：scanner_folder 告警点接入 notify 事件发布（复用 lifecycle 事件通道语义）。 -- 验收：配置桶通知后超限对象触发事件。 +**HS-04 Scanner excess S3 events and auditing missing** +- Current state: only `rustfs_scanner_excess_*_total` metrics (versions 100 / version size 1TiB / folders 65538). +- MinIO: emits `s3:ObjectManyVersions` (>100 versions), `s3:ObjectLargeVersions` (cumulative >1TB), `s3:PrefixManyFolders` (>50000 subdirectories) events (UserAgent: Scanner) + scanner:manyversions/largeversions/manyprefixes auditing. +- Impact: users relying on event subscriptions for capacity governance (console/external auditing) receive no alerts. +- Recommendation: hook the scanner_folder alert points into notify event publishing (reusing the lifecycle event-channel semantics). +- Acceptance: after configuring bucket notifications, an over-threshold object triggers an event. -**HS-05 madmin 客户端方法缺失** -- 现状：`crates/madmin/src/heal_commands.rs` 只有 wire 类型（HealDriveInfo/Infos/HealResultItem）；无 HealStart/HealStatus/BackgroundHealStatus/ScannerStatus 客户端方法。 -- MinIO：madmin-go 提供完整客户端；mc admin heal/scanner/status/trace 都建立在上面。 -- 影响：mc 等管理工具无法直接对接 RustFS heal/scanner 管理面；自动化运维只能手写 HTTP。 -- 建议：按 madmin-go 接口形状补客户端（服务端已就绪，纯客户端工作）。 -- 验收：用 madmin 客户端完成 start→query→cancel 全流程。 +**HS-05 madmin client methods missing** +- Current state: `crates/madmin/src/heal_commands.rs` has only wire types (HealDriveInfo/Infos/HealResultItem); no HealStart/HealStatus/BackgroundHealStatus/ScannerStatus client methods. +- MinIO: madmin-go provides the full client; mc admin heal/scanner/status/trace are all built on it. +- Impact: admin tools like mc cannot directly drive the RustFS heal/scanner admin surface; automated operations must hand-write HTTP. +- Recommendation: add the client following the madmin-go interface shape (the server side is ready; this is pure client work). +- Acceptance: complete the start→query→cancel flow with the madmin client. -**HS-06 admin heal 序列语义与 MinIO 差异** -- 现状：重复/重叠请求被去重合并（返回 canonical task_id）或驱逐；无 ErrHealAlreadyRunning/ErrHealOverlappingPaths 类型化错误（已亲验：manager.rs:1309 的 already_running 是幂等启动保护，非 admin 语义）；结果为快照式查询（1024 条/8MiB 截断/10min 保留），非 MinIO 的流式增量（clientToken 拉增量 + maxUnconsumedItems=1000 backpressure + 10s 保活 + 24h 未消费 abort）。 -- 影响：mc admin heal 的交互模型（长连接拉增量）对 RustFS 表现为多次快照轮询；自动化脚本难以区分"已合并"与"新启动"。 -- 建议：① 增量语义：channel query 支持自上次 clientToken 起的 items 增量（或 cursor）；② 重叠请求返回类型化错误码（或 receipt 中显式 merged_into 字段——现有 alias 机制已有基础）；③ forceStart 先停旧再启新语义核对。 -- 验收：madmin 兼容客户端按 MinIO 模式轮询能取得全量 items。 +**HS-06 admin heal sequence semantics differ from MinIO** +- Current state: duplicate/overlapping requests are dedup-merged (returning the canonical task_id) or evicted; no ErrHealAlreadyRunning/ErrHealOverlappingPaths typed errors (verified first-hand: manager.rs:1309's already_running is an idempotent-startup guard, not an admin semantic); results are snapshot-style queries (1024 entries/8MiB truncation/10min retention), not MinIO's streaming increments (clientToken pulls increments + maxUnconsumedItems=1000 backpressure + 10s keep-alive + 24h unconsumed abort). +- Impact: mc admin heal's interaction model (long connection pulling increments) behaves against RustFS as multiple snapshot polls; automation scripts cannot easily distinguish "merged" from "newly started". +- Recommendation: ① incremental semantics: channel query supports item increments since the last clientToken (or a cursor); ② overlapping requests return a typed error code (or an explicit merged_into field in the receipt — the existing alias mechanism already provides the base); ③ verify forceStart's stop-old-then-start-new semantics. +- Acceptance: an madmin-compatible client polling in the MinIO style can retrieve the full item set. -**HS-07 healing 进度与盘级 healing 状态对外可见性不足** -- 现状：bytes 恢复进度 `progress.bytes_processed = 0 // set to 0 for now`（erasure_healer.rs:967）；`HealProgress::estimated_completion_time` 恒 None、`HealStatistics::add_healed_objects` 未写入（progress.rs:38,135-139 零调用）；healthinfo 无每盘 HealInfo 等价（MinIO HealingDisk：BytesDone/Failed/Skipped、ObjectsTotal 基线、QueuedBuckets/HealedBuckets、Resume 快照、当前 object）；v3 指标无 drive_health=2(healing) 单一 gauge 等价。 -- 影响：换盘重建（可能数小时~天）期间运维无法回答"进行到哪/还剩多少/预计何时完成"。 -- 建议：① erasure set heal 统计 bytes（heal_object 返回对象大小已可得）；② 从 usage-cache 读对象总量基线（MinIO 同款做法）；③ admin healthinfo/背景状态暴露每盘 healing 快照（DiskInfo.healing 已有，补聚合暴露）；④ ETA 由基线+速率推导。 -- 验收：换盘重建中 admin 可见 bytes 进度与 ETA；mc info 等价输出 Healing 标志。 +**HS-07 healing progress and drive-level healing state insufficiently visible externally** +- Current state: byte-recovery progress `progress.bytes_processed = 0 // set to 0 for now` (erasure_healer.rs:967); `HealProgress::estimated_completion_time` is always None and `HealStatistics::add_healed_objects` is never written (progress.rs:38,135-139 zero calls); healthinfo has no per-drive HealInfo equivalent (MinIO HealingDisk: BytesDone/Failed/Skipped, ObjectsTotal baseline, QueuedBuckets/HealedBuckets, Resume snapshot, current object); v3 metrics lack an equivalent of the single drive_health=2 (healing) gauge. +- Impact: during a drive rebuild (potentially hours to days) operations cannot answer "where are we / how much is left / when will it finish". +- Recommendation: ① accumulate bytes in erasure set heal (heal_object already yields the object size); ② read the object-total baseline from usage-cache (the same approach as MinIO); ③ expose a per-drive healing snapshot in admin healthinfo/background status (DiskInfo.healing already exists; add the aggregated exposure); ④ derive the ETA from baseline + rate. +- Acceptance: during a drive rebuild, admin shows byte progress and ETA; an mc info-equivalent output shows the Healing flag. -**HS-08 prefix 级 usage 未暴露** -- 现状：DataUsageCache 内目录树 entry 存在（hash_path 组织），但 `dui()` 只 flatten 到桶名（data_usage_define.rs:858-915）。 -- MinIO：`loadPrefixUsageFromBackend`（30s cache）从每 set `.usage-cache.bin` 聚合 prefix usage，console 桶前缀统计消费。 -- 影响：console/前端无法展示前缀级用量；大桶定位"哪个前缀占空间"无 API。 -- 建议：实现 flatten 前缀查询 API（数据已在缓存内，纯聚合与暴露工作）。 -- 验收：ListBuckets/PrefixUsage API 返回与前缀过滤匹配的统计。 +**HS-08 prefix-level usage not exposed** +- Current state: the DataUsageCache holds the directory-tree entries (organized by hash_path), but `dui()` flattens only to the bucket name (data_usage_define.rs:858-915). +- MinIO: `loadPrefixUsageFromBackend` (30s cache) aggregates prefix usage from each set's `.usage-cache.bin`, consumed by console bucket-prefix statistics. +- Impact: console/front ends cannot show prefix-level usage; there is no API to locate "which prefix is using the space" in a large bucket. +- Recommendation: implement a prefix-flattening query API (the data is already in the cache; this is pure aggregation and exposure work). +- Acceptance: a ListBuckets/PrefixUsage API returns statistics matching the prefix filter. -### P2（9 项） +### P2 (9 items) -**HS-09 get_disk_status 恒返回 Ok（唯一 TODO）**：`crates/heal/src/heal/storage.rs:930-943`（已亲验）。当前无生产调用方（低风险）。建议：删除该方法或接 ecstore disk 状态真实现（DiskStatus 枚举已定义）。 +**HS-09 get_disk_status always returns Ok (the only TODO)**: `crates/heal/src/heal/storage.rs:930-943` (verified first-hand). Currently no production caller (low risk). Recommendation: delete the method or wire it to the real ecstore disk status (the DiskStatus enum is already defined). -**HS-10 HealStorageAPI 约 1/3 方法为死代码**：get_object_meta/get_object_data/put_object_data/delete_object/verify_object_integrity/ec_decode_rebuild/get_disk_status/format_disk/heal_bucket_metadata/get_object_size/get_object_checksum/list_objects_for_heal（非分页版，自带 memory_heavy 警告）均 0 调用方。建议：随 HS-01 决策一并清理或接线（死接口误导后续维护者以为存在调用路径）。 +**HS-10 About 1/3 of HealStorageAPI methods are dead code**: get_object_meta/get_object_data/put_object_data/delete_object/verify_object_integrity/ec_decode_rebuild/get_disk_status/format_disk/heal_bucket_metadata/get_object_size/get_object_checksum/list_objects_for_heal (the non-paginated version, with its own memory_heavy warning) all have 0 callers. Recommendation: clean up or wire them together with the HS-01 decision (dead interfaces mislead future maintainers into thinking a call path exists). -**HS-11 bitrot 自检缺失**：MinIO 启动时 bitrotSelfTest 对四算法已知向量自检失败即 Fatal（防静默数据损坏）。RustFS 无等价（已亲验 grep）。建议：启动时对 HighwayHash256S 等在用算法做已知向量自检（低成本高价值）。 +**HS-11 bitrot self-test missing**: MinIO at startup runs bitrotSelfTest over known vectors for the four algorithms and exits Fatal on failure (guarding against silent data corruption). RustFS has no equivalent (verified first-hand by grep). Recommendation: at startup, run known-vector self-tests for HighwayHash256S and the other algorithms in use (low cost, high value). -**HS-12 对象级 healing 元数据标记评估**：MinIO heal 期间对象打 `x-minio-healing:true`，RenameData 据此跳过版本清理/legacy purge（漏掉会导致 heal 与并发删除互毁）。RustFS 无对象级标记（已亲验 grep object.rs 无 healing 分支），依赖 NSLock + rename 语义。建议：审计 RustFS rename 提交路径是否存在"heal 提交与并发 delete/version 清理竞争"窗口；若无则文档化差异，若有则补标记等价机制。 +**HS-12 object-level healing metadata marker evaluation**: during heal, MinIO tags objects with `x-minio-healing:true`, and RenameData uses it to skip version cleanup/legacy purge (missing it lets heal and concurrent deletes destroy each other). RustFS has no object-level marker (verified first-hand by grep; object.rs has no healing branch) and relies on NSLock + rename semantics. Recommendation: audit whether the RustFS rename-commit path has a "heal commit racing concurrent delete/version cleanup" window; if not, document the difference, and if so, add a marker-equivalent mechanism. -**HS-13 erasure set heal 无"跳过新写入/ILM 已过期版本"过滤**：MinIO resync 跳过 ModTime>tracker.Started 的版本（避免 heal 追新写入尾巴）与 ILM 已过期版本（避免白做）。RustFS erasure_healer 未实现同款过滤（按版本 dedup 有，时间/ILM 过滤无）。影响：重建尾部长尾（持续写入的桶 heal 完成判定被新版本推迟）与无效 heal 工作量。建议：disk-walk 枚举处加 started_at 时间过滤 + evaluator 预检。 +**HS-13 erasure set heal lacks "skip newly written / ILM-expired versions" filters**: MinIO resync skips versions with ModTime>tracker.Started (so heal does not chase the tail of new writes) and ILM-expired versions (so work is not wasted). RustFS's erasure_healer does not implement such filters (per-version dedup exists; time/ILM filters do not). Impact: a long tail on rebuild completion (the completion decision for a continuously written bucket is pushed out by new versions) and wasted heal work. Recommendation: add a started_at time filter at the disk-walk enumeration point + an evaluator pre-check. -**HS-14 scanner idle 语义方向与 MinIO 相反**：MinIO `scanner:idle_speed=on`（默认）= 集群空闲时才节流、忙时全速；RustFS `RUSTFS_SCANNER_IDLE_MODE=true`（默认）= 限速总闸（false=完全不休眠）。两者默认行为可能相近（都限速）但参数语义不可互换，迁移文档需显式说明；若追求 mc config 兼容需重命名/重语义。建议：先文档化差异，评估是否对齐语义。 +**HS-14 scanner idle semantics point the opposite way from MinIO**: MinIO `scanner:idle_speed=on` (default) means "throttle only when the cluster is idle, full speed when busy"; RustFS `RUSTFS_SCANNER_IDLE_MODE=true` (default) is a master switch for rate limiting (false = never sleep at all). The default behaviors may end up similar (both throttle), but the parameter semantics are not interchangeable; migration docs must state this explicitly; if mc config compatibility is the goal, a rename/re-semantization is needed. Recommendation: document the difference first, then evaluate aligning the semantics. -**HS-15 alert_excess_folders 默认值差异**：RustFS 65538（兼容 PBS/Proxmox 布局，scanner_folder.rs:79）vs MinIO 50000。行为差异默认即触发阈值不同。建议：文档化（保留 65538 有本地理由）。 +**HS-15 alert_excess_folders default differs**: RustFS 65538 (compatible with the PBS/Proxmox layout, scanner_folder.rs:79) vs MinIO 50000. The behavioral difference is that the trigger threshold differs out of the box. Recommendation: document it (keeping 65538 has local rationale). -**HS-16 单机默认周期钩子未启用**：`single_disk_default_cycle_secs(_features) -> None` 恒空（scanner.rs:1428-1430），单机部署无专属默认周期覆盖。建议：决定单机默认周期策略后启用或删除钩子。 +**HS-16 single-node default-cycle hook not enabled**: `single_disk_default_cycle_secs(_features) -> None` is always empty (scanner.rs:1428-1430); single-node deployments get no dedicated default-cycle override. Recommendation: after deciding the single-node default-cycle policy, enable or delete the hook. -**HS-17 DeleteAllVersions 批量优化核对**：MinIO 用 DeletePrefix+DeletePrefixObject 单调用代替逐版本 fan-out。RustFS expiry 队列路径是否同款优化未逐行核实（集成测试覆盖行为正确性）。建议：核对 `apply_expiry_rule` 全版本删除路径，若无前缀单调用优化则评估补齐。 +**HS-17 DeleteAllVersions batch-optimization check**: MinIO uses the single DeletePrefix+DeletePrefixObject call instead of per-version fan-out. Whether RustFS's expiry-queue path has the same optimization was not verified line by line (integration tests cover behavioral correctness). Recommendation: check the `apply_expiry_rule` all-versions delete path; if there is no prefix single-call optimization, evaluate adding it. -### P3（3 项） +### P3 (3 items) -**HS-18 trash/临时目录二段清理细节核对**：MinIO `.minio.sys/tmp/.trash` 清理（delete_cleanup_interval 默认 5m + deleteCleanupSleeper）与 stale uploads rename-into-trash 二段式。RustFS 有 delete_tail_activity.rs 与 stale multipart 任务，二段语义是否完整对齐未逐行核实。建议：对照补齐或文档化。 +**HS-18 trash/temp-directory two-stage cleanup detail check**: MinIO cleans `.minio.sys/tmp/.trash` (delete_cleanup_interval default 5m + deleteCleanupSleeper) and stale uploads are renamed into trash in two stages. RustFS has delete_tail_activity.rs and the stale multipart task; whether the two-stage semantics are fully aligned was not verified line by line. Recommendation: align or document. -**HS-19 root heal 直连死路径清理**：`should_handle_root_heal_directly` 恒 false（admin/handlers/heal.rs:1200-1202，测试锁定），store.heal_format 直连分支不可达。建议：删除死分支或恢复直连路径作为集群协调失败的降级。 +**HS-19 root-heal direct path is dead code**: `should_handle_root_heal_directly` is always false (admin/handlers/heal.rs:1200-1202, locked by a test); the store.heal_format direct branch is unreachable. Recommendation: delete the dead branch or restore the direct path as a fallback for cluster-coordination failure. -**HS-20 兼容旗标与死指标清理**：`RUSTFS_SCANNER_INLINE_HEAL_ENABLE`（开启仅告警）+ `rustfs_scanner_inline_heal_total` 死指标 + `rustfs_common::metrics` 中 scanner 域代码分层迁移（backlog #1843 已登记）。建议：随分层迁移一并清理。 +**HS-20 compat flags and dead metrics cleanup**: `RUSTFS_SCANNER_INLINE_HEAL_ENABLE` (enabling only warns) + the dead `rustfs_scanner_inline_heal_total` metric + the scanner-domain code in `rustfs_common::metrics` awaiting layering migration (backlog #1843 already filed). Recommendation: clean up along with the layering migration. -### 按设计不追平（7 项，记录以防后续误判为缺口） +### Not pursuing parity by design (7 items, recorded to prevent later misreading as gaps) -1. **bloom filter**：MinIO master 已删除；RustFS `.bloomcycle.bin` 复用为 cycle/epoch 围栏与 MinIO 现状一致。 -2. **scanner 集群单 leader**：双方一致；RustFS 额外有 epoch 围栏。 -3. **heal 不发 S3 bucket notification**：双方一致（heal 结果走 admin status）。 -4. **incomplete multipart 不在 scanner/ILM 内执行**：双方一致（独立后台例程）。 -5. **内联 heal 移除**：RustFS 有意为之（scanner 只入队），MinIO 的 applyHealing 内联路径不做对标。 -6. **heal 序列常驻保活（10s 空白回写）**：RustFS 快照式查询模型不同，按 HS-06 处理增量语义即可，不复制流式保活。 -7. **`.trash`/`tmp-old` 路径名兼容**：RustFS 布局常量独立，不逐字对齐 MinIO 路径。 +1. **bloom filter**: removed from MinIO master; RustFS reuses `.bloomcycle.bin` as the cycle/epoch fence, consistent with MinIO's current state. +2. **scanner cluster single leader**: both sides agree; RustFS additionally has the epoch fence. +3. **heal emits no S3 bucket notification**: both sides agree (heal results go through admin status). +4. **incomplete multipart not executed inside scanner/ILM**: both sides agree (independent background routine). +5. **inline heal removal**: a deliberate RustFS choice (the scanner only enqueues); MinIO's applyHealing inline path is not a parity target. +6. **heal-sequence resident keep-alive (10s blank write-back)**: RustFS's snapshot-query model differs; handling incremental semantics per HS-06 is enough — do not copy the streaming keep-alive. +7. **`.trash`/`tmp-old` path-name compatibility**: RustFS's layout constants are independent; no literal alignment with MinIO paths. --- -## 6. 配置默认值总表（RustFS） +## 6. Configuration defaults master table (RustFS) -heal（env 前缀 `RUSTFS_HEAL_`，`crates/config/src/constants/heal.rs`，消费于 `manager.rs:724-800`）： +heal (env prefix `RUSTFS_HEAL_`, `crates/config/src/constants/heal.rs`, consumed at `manager.rs:724-800`): -| 配置 | 默认 | 热更新 | +| Setting | Default | Hot update | |---|---|---| -| AUTO_HEAL_ENABLE | true | 否 | -| QUEUE_SIZE | 10000 | 否 | -| INTERVAL_SECS | 10 | 否（启动时固定） | -| TASK_TIMEOUT_SECS | 300 | 否 | -| MAX_CONCURRENT_HEALS | 4 | 否 | -| MAX_CONCURRENT_PER_SET | 1（≤min(全局,值)） | 否 | -| LOW_PRIORITY_MERGE_ENABLE | true | 否 | -| LOW_PRIORITY_DROP_WHEN_FULL | true | 否 | -| PAGE_OBJECT_CONCURRENCY | 8（Deep/AutoHeal 强制 1） | 否 | -| EVENT_DRIVEN_SCHEDULER_ENABLE | true | 否 | -| SET_BULKHEAD_ENABLE | true | 否 | -| PAGE_PARALLEL_ENABLE | true | 否 | -| MAINLINE_THROTTLE_ENABLE | true | 否 | -| MAINLINE_READ/WRITE_UTILIZATION_HIGH_PERCENT | 80/80 | 否 | -| MAINLINE_MAX_SLEEP_MS | 250 | 否 | -| （总开关）RUSTFS_HEAL_ENABLED | true | 否 | -| admin 子系统 heal.bitrot_cycle | 30d | 是（经 scanner runtime config） | +| AUTO_HEAL_ENABLE | true | no | +| QUEUE_SIZE | 10000 | no | +| INTERVAL_SECS | 10 | no (fixed at startup) | +| TASK_TIMEOUT_SECS | 300 | no | +| MAX_CONCURRENT_HEALS | 4 | no | +| MAX_CONCURRENT_PER_SET | 1 (≤min(global, value)) | no | +| LOW_PRIORITY_MERGE_ENABLE | true | no | +| LOW_PRIORITY_DROP_WHEN_FULL | true | no | +| PAGE_OBJECT_CONCURRENCY | 8 (Deep/AutoHeal forced to 1) | no | +| EVENT_DRIVEN_SCHEDULER_ENABLE | true | no | +| SET_BULKHEAD_ENABLE | true | no | +| PAGE_PARALLEL_ENABLE | true | no | +| MAINLINE_THROTTLE_ENABLE | true | no | +| MAINLINE_READ/WRITE_UTILIZATION_HIGH_PERCENT | 80/80 | no | +| MAINLINE_MAX_SLEEP_MS | 250 | no | +| (master switch) RUSTFS_HEAL_ENABLED | true | no | +| admin subsystem heal.bitrot_cycle | 30d | yes (via scanner runtime config) | -scanner（admin 子系统 `scanner`，`crates/config/src/constants/scanner.rs` + `ecstore/src/config/scanner.rs` + `runtime_config.rs:527-673`）： +scanner (admin subsystem `scanner`, `crates/config/src/constants/scanner.rs` + `ecstore/src/config/scanner.rs` + `runtime_config.rs:527-673`): -| 键 | env | 默认 | +| Key | env | Default | |---|---|---| -| speed | RUSTFS_SCANNER_SPEED | default（2x/1s/60s） | -| delay / max_wait / cycle / start_delay | RUSTFS_SCANNER_* | 派生/空 | -| cycle_max_duration/objects/directories | …_MAX_* | 0（不限） | -| bitrot_cycle | …_BITROT_CYCLE_SECS | 2592000（30d；0/on=每轮，off=禁用） | -| idle_mode | …_IDLE_MODE | true | -| cache_save_timeout | …_CACHE_SAVE_TIMEOUT_SECS | 30s | -| max_concurrent_set_scans / disk_scans | …_MAX_CONCURRENT_* | 4/4 | -| yield_every_n_objects | …_YIELD_EVERY_N_OBJECTS | 128 | -| alert_excess_versions / version_size / folders | …_ALERT_* | 100 / 1TiB / 65538 | - -scanner 内部 env：`RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=16`、`RUSTFS_HEAL_OBJECT_SELECT_PROB=1024`、`RUSTFS_SCANNER_DEEP_VERIFY_COOLDOWN_SECS=60`、`RUSTFS_DATA_USAGE_FAILED_OBJECT_TTL_SECS=86400`/`_MAX=10000`、`RUSTFS_LOCK_ACQUIRE_TIMEOUT=5s`、`RUSTFS_SCANNER_ENABLED=true`、`RUSTFS_SCANNER_INLINE_HEAL_ENABLE=false`（兼容告警）。 - -全部 17 个 scanner 键支持 env > config 双通道 + admin PUT 热更（generation+Notify 即时生效）；heal 运行时参数目前仅 env（无 admin 热更入口，`Arc>` 结构已预留）。 - ---- - -## 7. 相关 backlog / 历史索引 - -- 换盘自动修复系列（已闭环）：backlog #1786（冗余假绿算法）、#1787（目标槽位限定）、#1789（resume 与 healing marker 绑定 replacement 实例）、#1791（黑白盒验收矩阵）。 -- #801 DiskInfo.healing 从未赋值（已修复闭环，现 `set_disk/mod.rs:4988` 有赋值链）。 -- #1651 Scanner 指标节点/source/bucket-drive 维度（OPEN，本分析 §3.8/§4.6 相关）。 -- #1843 crates/common 83% scanner/heal 域代码分层迁移（OPEN，含 HS-20）。 -- 代码注释引用的历史缺陷（现已有防护与回归测试）：#856/#799 B7（离线盘误记 healed）、#855/B6/#1033（skip 不得标记完成）、#920（sub-quorum 并集枚举）、#856 B5（按版本续扫）、#5173（bitrot trailing bytes）、#5029（回归节点 stale 版本合并）。 -- v1 对标文档：`docs/rustfs-heal-scanner-vs-minio-parity-assessment.md`（本文取代）、落地手册 `docs/rustfs-heal-scanner-vs-minio-improvement-playbook.md`（部分条目已被后续实现超越）。 -- 换盘深度分析：`docs/new-disk-replacement-and-healing-deep-analysis-zh.md`、`docs/node-disk-identity-and-healing-analysis-zh.md`。 - -## 8. 审计方法与局限 - -- 四路并行审计（heal crate 逐文件、scanner crate 逐文件、ecstore 集成层 wiring、MinIO master 源码研究）+ 主会话对关键"缺失"结论逐条亲验（get_disk_status TODO、HealEvent 零外部引用、.bloomcycle.bin 无 bloom 实现、check_abandoned_parts 三层 NotImplemented、ETag 兜底已实现、trace 通道零命中、already_running 语义）。 -- 未逐行核实的点（已在文中标注"未确认/未逐行核"）：DeleteAllVersions 前缀单调用优化（HS-17）、trash 二段清理细节（HS-18）、ilm worker 默认值对照、stale multipart 默认值对照、mc CLI flag 逐字拼写（MinIO 侧）。其中 HS-17 与 HS-18 已于 2026-08-19 完成逐行核实，结论见 §9.2/§9.3。 -- MinIO 侧引用以其 master `7aac2a2c5b` 为准；RustFS 侧行号以 2026-08-16 工作区为准，后续演进请以符号名检索为准。 - -## 9. 落地结果（2026-08-19 更新） - -本审计衍生的 14 个子 issue（backlog #1865~#1878）已全部闭环。本节为差距清单 HS-01~HS-20 的最终处置记录，也是下一轮对标重审的增量基线。 - -### 9.1 已落地（PR 均已合并 main） - -- HS-01 MRF 接线 + 持久化修复账本（#1865，PR #6189）：决策选 (a)。common MRF channel（bounded 8192、try_send 永不阻塞）+ heal mrf_queue（100k 条 / 8MiB 双限环形）+ `buckets/.heal/mrf/journal.bin` CRC 持久化回放（torn tail 截断、回放后删除）+ 三投递点（read decode_error→Urgent ECDecode、scanner 元数据损坏→High Metadata、add_partial→Normal）+ `RUSTFS_HEAL_MRF_ENABLE` 一键回退。 -- HS-02 abandoned parts/data-dir 对账（#1866，PR #6179）：接通 abandoned 检查入口，保留 dry-run / reclaim 计数。 -- HS-03 heal/scanner trace 通道（#1867，PR #6179）：进程内 trace bus + `/v3/trace` admin 流式订阅 + heal task / abandoned-parts / scanner folder / ILM / heal-candidate trace producer。 -- HS-04 scanner 超限 S3 事件（#1868，PR #6176）：`s3:Scanner:ManyVersions/LargeVersions/BigPrefix` 三事件 + 24h 边沿冷却；HS-15 阈值差异文档化（`docs/operations/scanner-excess-alerts.md`）。 -- HS-05 madmin 客户端一期（#1869，PR #6166）：SigV4 admin 客户端 heal/scanner 方法；增量消费方法待 follow-up（协议已由 HS-06 并入）。 -- HS-06 admin heal 增量语义与类型化重叠（#1870，PR #6206）：`sinceSeq/nextSeq/minSeq` 增量游标（wire additive、缺省=全量快照）+ `RUSTFS_HEAL_OVERLAP_POLICY`（默认 merge 不变；minio_error 下 AlreadyRunning/OverlappingPaths 类型化拒绝）+ forceStart 先停旧再启新。 -- HS-07 healing 进度可见性（#1871，PR #6179）：data-usage 总量基线 + baseline/current/healed 计数。 -- HS-08 prefix usage（#1872，PR #6171）：`GET /v3/usage/{bucket}`。 -- HS-11 bitrot 启动自检（#1873，PR #6165）。 -- HS-13 heal 跳过过滤（#1875，PR #6179）：过滤命中版本不再计为失败。 -- HS-16 单机周期钩子（#1878，PR #6250）：删恒 None 钩子，决策记录见 `docs/operations/heal-scanner-parity-notes-zh.md`。 -- HS-09/10/19/20 死代码清理批（#1877，PR #6256）：净 −911 行零行为变更；`get_disk_status` TODO（全仓库唯一产品 TODO）清零；HS-01 联动的 `ec_decode_rebuild`/`get_object_meta` 保留并加 Reserved 注释（MRF 当前经 `heal_object` 执行）。 - -### 9.2 核对后确认"已实现 / 非缺口"（审计期误判修正，累计四例） - -- bloom filter（§0 已修正）：MinIO master 已删除，双方现状一致。 -- ETag 兜底仲裁（§0 已修正）：RustFS 已有实现（`set_disk/ops/heal.rs`）。 -- HS-17（#1876，2026-08-19 逐行核实后关闭）：DeleteAllVersions 前缀单调用优化 RustFS 已完整实现——`apply_expiry_on_non_transitioned_objects` 对 `delete_all()` 两 action 设 `delete_prefix + delete_prefix_object` 后单次 `delete_object`（`bucket_lifecycle_ops.rs:5047-5056`），SetDisks 分支一次写锁 + 一次全版本 quorum 读 + 内联逐版本 object-lock 检查（`set_disk/ops/object.rs:5566-5612`），与 MinIO `expire.go` 的 `applyExpiryOnNonTransitionedObjects` 逐行对齐。§8 原列"未逐行核实"的本项已有结论：现状即优化路径，无需实现。 -- HS-14（#1878，PR #6250 附带核对）：MinIO"idle=空闲才节流"是 2024-01 minio/minio#18734 之前的行为（`scannerIdleMode` 现为静态配置，`idle_speed=on` 默认即始终按速度档节流，"idle"命名是历史残留）；RustFS `RUSTFS_SCANNER_IDLE_MODE` 与 MinIO 当前语义方向一致，且另有 MinIO 没有的前台读退避下限。真实迁移陷阱（变量须 `RUSTFS_` 前缀、`on/off` vs `true/false` 词表、`false` 连前台保护一起关）已文档化于 `docs/operations/heal-scanner-parity-notes-zh.md`。 - -### 9.3 审计型结论（无需改代码） - -- HS-12（#1874，PR #6183）：不存在 MinIO 用 `x-minio-healing` 防御的那类竞争——所有同 (bucket, object) 提交面在同一把对象级 ns 写锁互斥，heal 锁 guard 覆盖 rename 提交全程；交付 2 个并发不变量回归测试 + `docs/operations/heal-concurrency-safety-notes-zh.md` 交点矩阵。 -- HS-18（#1878，2026-08-19 逐行核实）：trash/tmp 三段清理全对齐——stale multipart 隔离-清理等价且更安全（`delete_all_with_quorum` 逐盘递归删即 `move_to_trash` rename 进 `.rustfs.sys/tmp/.trash`，另有锁 + fence）、trash 排空基本等价（无逐条 sleeper 节流，5m 周期天然限频）、tmp 非 trash 24h 回收等价（RustFS 5m 比 MinIO 6h 更及时）；周期默认 24h/6h/5m 三项全对齐。§8 原列"未逐行核实"的本项已有结论。 - -### 9.4 移交 follow-up（汇总于 backlog#1862 评论区） - -HS-01 bitrot GET→MRF 全链路 e2e、kill -9 journal 回放 e2e、队列满压测 RSS（≤ 预算+10%）；HS-05/06 madmin 增量消费方法 + wire 单一来源化 + embedded e2e + 多轮轮询 soak；HS-08 多盘 scanner 周期 e2e；HS-04 超限审计条目；HS-18 低于 quorum 的 stale-multipart 崩溃残留窗口（扇出中途崩溃且已清盘数 > parity 时 FileNotFound 不在忽略集导致不自然收敛，修复需专用 quorum 变体）。 - -下一轮重审建议：跟随 heal/scanner 下一个大特性落地后触发，以本节为增量基线。 +| speed | RUSTFS_SCANNER_SPEED | default (2x/1s/60s) | +| delay / max_wait / cycle / start_delay | RUSTFS_SCANNER_* | derived/empty | +| cycle_max_duration/objects/directories | …_MAX_* | 0 (unlimited) | +| bitrot_cycle | …_BITROT_CYCLE_SECS | 2592000 (30d; 0/on=every cycle, off=disabled) | diff --git a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md new file mode 100644 index 000000000..a75971d1a --- /dev/null +++ b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md @@ -0,0 +1,568 @@ +# RustFS heal / scanner 全量功能分析与 MinIO 对标（v2） + +> English version: [rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md](rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md) + +- 日期：2026-08-16（基于 main 分支当日代码，审计时 HEAD ≈ `a118d7e4f`） +- 范围：`crates/heal`（src 19,560 行 + tests 2,274 行）、`crates/scanner`（src 约 26,000 行 + tests）、`crates/data-usage`、`crates/ecstore` 中 heal/heal_walk/bitrot_self_verify 与 config、`crates/common/src/heal_channel.rs`、`crates/madmin`（heal/scanner wire 类型）、`rustfs/src`（startup wiring、admin handlers、集群 RPC） +- 对标基线：minio/minio master（HEAD `7aac2a2c5b`，仓库已进入维护模式，master 冻结，即最终态） +- 方法：四路并行审计（heal crate / scanner crate / ecstore 集成层 / MinIO 源码研究），关键结论逐条人工抽验（文内标注"已亲验"处为一手验证） +- 本文档取代 `docs/rustfs-heal-scanner-vs-minio-parity-assessment.md`（2026-06-15 v1）。v1 之后 heal/scanner 相关提交超过 80 个（换盘自动修复全链路、resume 状态机、usage 收敛权威化、集群级 heal 协调、ILM restore 语义等），v1 的功能清单与差距判断已全面过时；v1 中"bloom filter 缺失"等结论经本次核实为**误判**（详见 §5.4）。 + +--- + +## 0. 结论摘要 + +1. **总体判断：heal 与 scanner 的核心功能链路已经完整**。对象级 heal（quorum 仲裁 + ETag 兜底 + bitrot Deep 校验 + dangling 处理）、erasure set 深扫（per-set disk-walk 并集枚举）、按版本断点续扫（schema 化持久层 + CAS 原子发布 + 崩溃窗口补齐）、换盘自动修复（readiness 校验 + 身份围栏 + durable intent + completion proof）、scanner 周期循环（leader lock + 持久化 leader-epoch 围栏）、data usage 统计（桶级/集群级、主+备+观测快照、epoch/cycle 防回退）、ILM 全动作（expiry/transition/noncurrent/free-version/delete-marker 清理）、admin Start/Query/Cancel 协议（clientToken 语义对齐 madmin）——以上均有实现且带回归测试。两个 crate 内**没有空实现/早退桩**，异常路径全部有日志 + 指标 + 错误语义。 +2. **主要缺口集中在"入口与观测面"，而不是修复算法本身**：MRF/ECDecode/Metadata 三类任务执行体已实现但无生产触发入口（`HealEvent` 完全未接线）；`CheckAbandonedParts` 在 ecstore 三层全部 `NotImplemented`；heal/scanner trace 通道缺失；scanner 超限 S3 事件缺失；madmin 客户端方法缺失（只有 wire 类型）；heal 字节级进度/ETA 未实现。 +3. **与 v1 认知的重要修正**：bloom filter 在 MinIO 当前 master **已删除**（`.bloomcycle.bin` 只存 cycle 计数），RustFS 现状与 MinIO 一致；MinIO scanner 同样是**集群级 leader 单例**，RustFS 的 leader.lock 模型与 MinIO 同型；RustFS 的 ETag 多数派兜底仲裁已实现（`crates/ecstore/src/set_disk/ops/heal.rs:525-567,679`，已亲验），v1 担心的仲裁缺口不存在。 +4. **RustFS 在多处超出 MinIO**：remote_scanner RPC 协议（远端 peer 本地扫描而非 leader 跨网读远盘）、持久化 leader-epoch CAS 围栏、周期预算与 per-set/per-disk 并发闸、pending-heal 账本、durable replacement intent + completion proof 状态机、前台压力门控（mainline throttle）、集群 heal control coordinator + envelope 重放防护。 +5. 差距分级统计：P1（行为/运维对齐缺口）8 项，P2（完善性）9 项，P3（清理/低风险）3 项，"按设计不追平"7 项。完整清单见 §6。 + +--- + +## 1. 架构总览 + +### 1.1 RustFS 三层架构 + +RustFS 把 MinIO 在 `cmd/` 内单体的 heal/scanner 拆成三层 + 两个独立 crate： + +| 层 | 位置 | 职责 | +|---|---|---| +| 原语层 | `crates/ecstore/src/set_disk/ops/heal.rs`（~3,240 行）、`ops/heal_walk.rs`、`ops/bitrot_self_verify.rs`；上层封装 `store/heal.rs`、`store/heal_walk.rs`、`core/sets.rs` | 对象/桶/format/替换盘格式修复、disk-walk 并集枚举、写入路径 bitrot 自校验；由 `SetDisks`/`Sets`/`ECStore` 实现 `rustfs_storage_api::HealOperations` 契约（`crates/storage-api/src/object.rs:503-519`） | +| heal 运行时 | `crates/heal` | 进程级 HealManager（优先级队列/调度器/auto disk scanner/断点续传 resume）、HealChannelProcessor（消费全局 heal channel）、换盘替换恢复状态机 | +| scanner 运行时 | `crates/scanner` | 数据使用扫描、ILM 评估与入队、heal 候选生产、复制用量统计、remote scanner RPC | +| 共享协议 | `crates/common/src/heal_channel.rs`（~776 行） | Start/Query/Cancel 命令通道、`HealOpts`/`HealScanMode`/`HealRequestSource`/`HealAdmission*` 共享类型、`HealResultItem`（madmin） | +| 共享数据 | `crates/data-usage` | `DataUsageEntry/Info`、直方图、`hash_path`；scanner 产生、ecstore/admin 消费 | + +启动链路（已亲验 wiring）： + +1. `rustfs/src/startup_services.rs:93` → `init_background_service_runtime(store)`。 +2. `rustfs/src/startup_background.rs:41-81`：创建全局 heal 服务取消令牌；读 `RUSTFS_SCANNER_ENABLED`（别名 `RUSTFS_ENABLE_SCANNER`，默认 true）与 `RUSTFS_HEAL_ENABLED`（别名 `RUSTFS_ENABLE_HEAL`，默认 true）；**只要 heal 或 scanner 任一开启就初始化 heal manager**（scanner 产生的 heal 候选需要消费端；两者都关时 heal channel 不初始化，`send_heal_request` 报 "Heal channel not initialized"）。 +3. `crates/heal/src/lib.rs:142-216`：owned task 内原子初始化（caller 取消不会遗留半初始化 manager，`lib.rs:123-131`；`GLOBAL_HEAL_RUNTIME_INIT` 互斥单飞）→ `HealManager::start()` → `rustfs_common::heal_channel::init_heal_channels()` → spawn `HealChannelProcessor::start_with_receipts`。 +4. `crates/heal/src/heal/manager.rs:1301-1356` `HealManager::start`：`start_scheduler()`（`manager.rs:2394-2461`，interval 默认 10s + `Notify` 事件驱动唤醒）→ `process_unclean_shutdown()`（`manager.rs:1362-1695`）→ `enable_auto_heal`（默认 true）时 `start_auto_disk_scanner()`（`manager.rs:2464-2999`）。 +5. server ready 后 `rustfs/src/startup_lifecycle.rs:150-152`：`enable_scanner` 时 `init_data_scanner(token, store)`（`crates/scanner/src/scanner.rs:1293-1372`）。 +6. 优雅停机：`rustfs/src/startup_shutdown.rs:308` `shutdown_ahm_services()`（取消令牌）；`:414` `clear_unclean_shutdown_markers()`。 + +### 1.2 MinIO 对应结构（master 最终态） + +| MinIO 文件 | 职责 | +|---|---| +| `cmd/admin-heal-ops.go` | 手动 admin heal 序列（healSequence、clientToken/forceStart/forceStop） | +| `cmd/global-heal.go` | 常驻后台 heal 队列（newBgHealSequence，token 固定 `0000-…`，永不结束）+ `healErasureSet`（逐 set 全量对象 heal） | +| `cmd/background-heal-ops.go` | healRoutine worker 池（`_MINIO_HEAL_WORKERS`，默认 GOMAXPROCS/2）消费 healTask | +| `cmd/mrf.go` | MRF（Most Recent Fail）队列（容量 100,000），进程退出时持久化 `.minio.sys/buckets/.heal/mrf/list.bin` 并启动回放 | +| `cmd/background-newdisks-heal-ops.go` | 新盘/换盘自动 resync（monitorLocalDisksAndHeal 10s 轮询 + healFreshDisk + healingTracker） | +| `cmd/erasure-healing.go` / `erasure-healing-common.go` | 对象级 heal 核心（~800 行）、listAndHeal | +| `cmd/data-scanner.go` | scanner 循环（globalLeaderLock 集群单例）+ folderScanner + applyActions | +| `cmd/erasure.go`（nsScanner）/ `erasure-server-pool.go` | NSScanner 三层结构 | +| `cmd/bucket-lifecycle.go` | ILM 执行器（expiry/transition worker 池） | +| `cmd/xl-storage.go` | DiskInfo.Healing、CheckParts/VerifyFile、CleanAbandonedData、RenameData healing 分支 | +| `cmd/prepare-storage.go` | waitForFormatErasure 新盘启动握手 | + +### 1.3 架构级差异（设计取舍，非缺陷） + +1. **heal 队列模型**：MinIO 所有 heal（scanner 抽样/MRF/admin/新盘 resync）汇入单 channel + 固定 worker 池（新盘 resync 另有 per-drive worker 池）；RustFS 是优先级堆 + 去重合并 + 容量分级丢弃 + per-set bulkhead + 前台压力门控的多策略调度器（`manager.rs:3003-3420`）。RustFS 表达力更强，代价是"重复请求被合并"的可观测性问题（v1 已指出，现有 `HealAdmissionReceipt` canonical task_id + alias 机制回应了它，`manager.rs:1759-1846`）。 +2. **scanner 远端盘访问**：MinIO leader 通过磁盘抽象层透明读写远端节点磁盘；RustFS leader 通过 remote_scanner RPC 把扫描执行下放到远端 peer 本地进行（`crates/scanner/src/remote_scanner.rs`），只回传结果与进度心跳。两者都是集群单 leader。RustFS 方案省 leader↔远端的元数据读放大，代价是需要维护独立 RPC 协议（HMAC 逐帧认证、会话重放缓存、fence 复验，`remote_scanner.rs:52-61,405-496,1024-1065`）。 +3. **heal 状态持久化**：MinIO 用单文件 `.healing.bin`（msgp healingTracker，diskID 不匹配即重置）；RustFS 用 schema 化多文件（resume/checkpoint/intent/seal/proof 各自 CAS 发布，`resume.rs:38-61`），崩溃窗口显式补齐（`erasure_healer.rs:389-402`、`resume.rs:1027-1057`）。 +4. **写路径自保护**：MinIO 写入后靠后台 heal 收敛；RustFS 在 PutObject/CompleteMultipartUpload 提交 rename 后主动检查 `convergence.needs_heal()` 并立即入队对象 heal（`set_disk/ops/object.rs:2291-2306`、`ops/multipart.rs:2574-2589`），另有读修复 read repair（`io_primitives.rs:1040-1160`）。 + +--- + +## 2. Heal 已实现功能全景 + +### 2.1 任务类型（`HealType`，`crates/heal/src/heal/task.rs:85-111`） + +| 类型 | 语义 | 执行体 | 生产触发方 | +|---|---|---|---| +| `Cluster` | 所有 bucket 依次 heal（结构 + 可选递归对象），批内重试 ≤3 | `heal_cluster` task.rs:1420-1490 | channel：bucket 为空即 Cluster（channel.rs:576-577） | +| `Object{bucket,object,version_id}` | 单对象/版本；不存在时按 `recreate_missing` 重建或报错 | `heal_object` task.rs:855-1146 | admin、scanner、read-repair、写路径收敛、add_partial | +| `Bucket{bucket}` | 桶元数据/结构；`recursive` 再遍历全部对象版本 | `heal_bucket` task.rs:1284-1418 + `heal_bucket_objects` task.rs:1508-1698 | admin（POST /v3/heal/{bucket}）、scanner `build_bucket_heal_request` | +| `Prefix{bucket,prefix}` | 按前缀递归 | `heal_prefix` task.rs:1492-1506 | channel：`recursive && prefix` 非空（channel.rs:578-585） | +| `ErasureSet{buckets,set_disk_id}` | format 修复 + healing 标记 + 逐桶预处理 + 可恢复逐版本深扫 | `heal_erasure_set` task.rs:2158-2642 | admin（pool/set 参数）、auto disk scanner、unclean shutdown、renew_disk、durable replacement 恢复 | +| `Metadata{bucket,object}` | 仅元数据（Deep、不重建数据） | `heal_metadata` task.rs:1700-1859 | **无生产触发方**（§6 HS-01） | +| `MRF{meta_path}` | 失败路径驱动的 Deep 修复（recursive+update_parity） | `heal_mrf` task.rs:1861-1992 | **无生产触发方**（仅 `HealEvent` 可生成，未接线） | +| `ECDecode{bucket,object,version_id}` | EC 解码重建（Deep+recreate+update_parity），Urgent 优先级 | `heal_ec_decode` task.rs:1994-2156 | **无生产触发方**（仅 `HealEvent` 可生成，未接线） | + +优先级 `Low/Normal/High/Urgent`（task.rs:168-179）；状态机 `Pending/Running/Retrying/Completed/Failed/Cancelled/Timeout`（task.rs:225-241）。 + +### 2.2 触发路径全景（admin 之外） + +| 通道 | source | 优先级 | 证据 | +|---|---|---|---| +| Scanner 周期抽样（1/1024，`RUSTFS_HEAL_OBJECT_SELECT_PROB`） | Scanner | Low | `scanner_folder.rs:2117-2136`、`:1150`；`remove_corrupted=HEAL_DELETE_DANGLING(true)`、`recreate_missing=false`（`common/heal_channel.rs:24`、`scanner_folder.rs:510-511`） | +| Scanner 元数据损坏（get_size 失败分类 HealMetadata） | Scanner | High | `scanner_folder.rs:2147-2208`、`:1244-1260` | +| Scanner abandoned children（缓存有、盘上无，list_path_raw quorum 核查） | Scanner | High（桶级+对象级） | `scanner_folder.rs:2528-2792` | +| Scanner pending-heal 账本重试（heal 通道满被拒后持久化，每桶每轮 ≤128 条、上限 10k） | Scanner | 原优先级 | `scanner_folder.rs:1721-1763`、`:99-100` | +| auto disk scanner（unformatted 盘经 replacement_readiness 确认 / `runtime_state=="returning"` 盘 / durable intent 重入） | AutoHeal | Low | `manager.rs:2464-2999` | +| unclean shutdown 恢复（启动读 `unclean-shutdown` 标记 → 全部本地 set ErasureSet heal） | AutoHeal | Low | `manager.rs:1362-1695` | +| 写路径收敛（PutObject/CompleteMultipartUpload 后 `convergence.needs_heal()`） | Internal | Normal | `set_disk/ops/object.rs:2291-2306`、`ops/multipart.rs:2574-2589` | +| 部分对象 heal（add_partial） | Internal | Normal | `set_disk/ops/object.rs:5808-5825` | +| 旧数据目录清理残留 enqueue | Internal | Normal | `set_disk/core/io_primitives.rs:3880-3907` | +| 读修复（metadata_read_error / missing_shards / decode_error，TTL 去重缓存） | ReadRepair | Low | `set_disk/read.rs:407,995,1079` → `submit_read_repair_heal`（`io_primitives.rs:1105-1160`），`recreate_missing=true` | +| 盘重连遇 UnformattedDisk → send_heal_disk | AutoHeal | Normal | `set_disk/ops/locking.rs:339-347` | +| Admin API（含集群 coordinator 路由） | Admin | High | `rustfs/src/admin/handlers/heal.rs:174-212`、`:771-930` | +| 集群 RPC heal（peer 调用） | — | — | `rustfs/src/storage/rpc/node_service/heal.rs`、`ecstore/src/cluster/rpc/peer_s3_client.rs:296,1209` | + +注意：MinIO 的 MRF 通道（读路径检出 part 缺失/损坏即时投递 + 队列持久化 + shutdown 回放，`cmd/mrf.go`、`erasure-object.go:395-410,800-812`）在 RustFS 由 read-repair + 写路径收敛**部分替代**；`HealType::MRF`/`ECDecode`/`Metadata` 三个执行体没有生产入口（详见 §6 HS-01）。 + +### 2.3 对象级 heal 语义（ecstore `set_disk/ops/heal.rs`） + +流程（`heal_object_with_explicit_version_regen` :426 起）： + +1. 取对象写锁（除非 `no_lock`）；`object` 以 `/` 结尾走对象目录 heal（`heal_object_dir_locked` :1587-1717：dangling 判定 + `remove` 删除 + 缺 volume 重建）。 +2. `read_all_fileinfo` 全盘读 xl.meta，全部 not-found 视为已删除返回。 +3. **quorum 仲裁 + ETag 兜底**（已亲验）：`list_online_disks` 以 mod-time quorum 为准；quorum 失效时回退 ETag 多数派仲裁（`:525-567` `filter_by_etag`/`quorum_etag`）；`pick_valid_fileinfo` 选 canonical 元数据；"meta 坏盘数 > parity" 的 cannotHeal 判定在 ETag 全盘一致时豁免（`:679`）。与 MinIO `filterDisksByETag` 双仲裁一致。 +4. `disks_with_all_parts`（:562-572）按 `scan_mode` 校验 part：**Normal 仅 stat（CheckParts 语义），Deep 做全量 bitrot 校验（VerifyFile 语义）**；Normal 扫描检出 `FileCorrupt` 自动升级 Deep 重试一次（`:2022-2031`，与 MinIO erasure-healing.go:1101-1106 同型）；无 parity 对象（EC:0）bitrot 失败判不可恢复（`:700-726`）。 +5. `should_heal_object_on_disk`（:606-650）逐盘分类 missing/corrupt/offline/outdated → 重建：per-part bitrot reader/writer（用 per-part checksum + 算法）、写临时卷后 rename 提交（`HEAL_RENAME_INCOMPLETE` 重试语义 :24）；dangling 删除安全检查 `dangling_delete_safety`（:1488）；**孤儿数据目录回收 `reclaim_orphan_data_dirs_best_effort`（:1428）**——这部分覆盖了 MinIO `CleanAbandonedData` 的主场景（但无独立 `CheckAbandonedParts` API，见 §6 HS-02）。 +6. 版本化对象：枚举"每个版本"（`storage.rs:1494-1530`）；delete-marker 路径由 `latest_meta.deleted` 决定（`storage.rs:262-277` 注释）；回归测试 `tests/heal_b5_versioned_regression_test.rs:282,334`。 +7. 显式版本重建 `try_regenerate_explicit_version_meta`（:1318）；transitioned 对象本地残留清理。 +8. 写入路径另有 shard 级 bitrot 自校验 `verify_written_bitrot_shards`（`ops/bitrot_self_verify.rs:45-129`，HighwayHash256S，最终 rename 前校验刚写出的 shard，服务 EC:0 无 parity 场景）——**注意这不是后台 bitrot 巡检**；后台巡检由 scanner bitrot_cycle 驱动 Deep heal 承担。 + +heal crate 侧包装（`task.rs:855-1146`）：存在性检查（瞬时错误转 `TransientSkip` 不误判失败 :551-569）；scanner 合成目录规范化（:1148-1180）；`recreate_missing` 重建（:1183-1282）；data-usage-cache 对象锁超时豁免（:571-653）；not-found → treated_as_deleted 成功（:1012-1029）；结果 `HealResultItem` 保留至多 1024 条 + truncated 标志（:50,845-852）。 + +递归遍历（`heal_bucket_objects` task.rs:1508-1698）：分页枚举全部版本含 delete marker、瞬时错误指数退避重试 ≤3（2^n + 抖动 :620-627）、失败样本日志截断 ≤5 条、聚合 `BatchHealFailure`。 + +### 2.4 erasure set heal 与断点续扫 + +`heal_erasure_set`（task.rs:2158-2642）四阶段（4 步进度跟踪）： + +1. **替换意图与恢复盘选择**（仅 AutoHeal + heal_endpoints 非空）：复用 durable intent 所在盘 / 排除目标端点选幸存盘；已完成代（CleanupPending）幂等收尾。 +2. **格式修复**：`heal_replacement_format(dry_run, pool, set, targets)`（`storage.rs:1372-1384`，trait 默认实现 fail-closed）；逐目标盘结果必须全 ok（`erasure_healer.rs:97-102`）+ 身份围栏复核（task.rs:2410-2420）。 +3. **healing 标记**：对目标盘写 owner CAS 标记 `{set_disk_id}:{task_id}`（`mod.rs:80-229`，CAS + 回滚 + 并发唯一 owner），使 `DiskInfo.healing` 为真（已亲验赋值链 `set_disk/mod.rs:4988`）。 +4. **逐桶预处理 + 可恢复深扫**：`ErasureSetHealer::heal_erasure_set`（`erasure_healer.rs:242-278`）。 + +`ErasureSetHealer` 扫描细节（对标 MinIO `healErasureSet`，`heal_walk.rs:15-23` 模块注释明确引用 MinIO `global-heal.go` 的 listPathRaw + objQuorum=1 + mergeXLV2Versions）： + +- **枚举器选择（backlog#920）**：Deep 或 AutoHeal → per-set **disk-walk 并集枚举** `list_versions_for_heal_page_disk_walk`（"任意盘上存在"即 sub-quorum 可重建；`storage.rs:1559-1644`，页界 1000 对象/10,000 版本，`dw1:` cursor）；普通请求走 read-quorum `list_object_versions`。 +- **续扫游标**：权威 cursor 为 opaque continuation token（`v1:`=marker JSON、`dw1:`=disk-walk key，两命名空间互斥防误读，`storage.rs:81-260`）；每完成一页先持久化 cursor 再清 dedup 集合（`erasure_healer.rs:922-927`）。 +- **页内并发**：FuturesUnordered + Semaphore，默认 `RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY=8`，Deep/AutoHeal 强制 1（`erasure_healer.rs:105-142`）。 +- **per-version dedup**：`compose_key` 长度前缀注入编码（`resume.rs:281-288`）。 +- **错误分类**：真缺席（FileNotFound 等）→ Absent（计成功）；基础设施瞬时（quorum/DiskNotFound/SlowDown 等）→ Transient（计 skipped）；其余 Failed（`erasure_healer.rs:148-182`，注释引 backlog#856/#799 B7：离线盘不得记 healed/absent）。 +- **防死循环**：空页 truncated 或页尾版本身份不前进即中止（:933-949）。 +- **完成判定**：failed/skipped/failed_buckets 任一 >0 不标记完成，`schedule_retry()` 复位 resume+checkpoint 两层（:561-626，backlog#855/B6/#1033：skip 轮不得标记完成）。 +- **替换盘提交证据**：目标端点物理回读 `replacement_targets_have_version`（`ops/heal.rs:340-412`），未确认 → transient skip。 + +### 2.5 换盘自动修复（replacement recovery） + +- **识别**（`replacement_readiness.rs:25-73`）：`replacement_mount_lease_root()` 存在、canonicalize 成功、是挂载点、物理设备 id 非空、与根设备不相交、不与兄弟盘共享物理设备（Linux 用 /proc/self/mountinfo mount-id+dev+ino）。非 root 挂载检查有回归测试（`manager.rs:3549`）。 +- **状态机**（`resume.rs:63-73`）：`Intent → Rebuilding →（写 proof）Verified → CleanupPending → 清理`；`Abandoned` 终态；跨状态迁移先写持久层再变更（`save_state_strict`）。 +- **持久化**（`resume.rs:38-61`，schema ResumeState=5/Checkpoint=5/proof=1）：`{task_id}_ahm_resume_state.json`、`_ahm_checkpoint.json`、`buckets/ahm-replacement/` 命名空间下 intent/seal/completion_proof；torn write + 无 seal 可识别并原子重建（:1316-1338）；CAS 发布、拒绝覆盖并发有效 proof（:1512-1585）。 +- **恢复**：unclean shutdown 与周期扫描都从幸存盘恢复未完成/待清理替换代（`manager.rs:1435-1640,2663-2815`）；多代冲突/校验失败 → 冻结该 set（`replacement_recovery_blocked_sets`，`manager.rs:69-87,2782-2815`）。 +- **对外快照**：`current_replacement_recovery_snapshot`（`lib.rs:262-333`）合并本地幸存盘记录，冲突 → Unknown/非 definitive；admin `GET /v4/heal/replacement-recovery`。 + +### 2.6 调度器（manager.rs） + +- 优先级堆 + 同优先级 FIFO（:148-191,330-347）；dedup key 按类型（:469-506）；入队三态查重 active→queued→retrying（:1759-1785）；重复默认 Merged 并返回 canonical task_id（`HealAdmissionReceipt`，:1821-1846）+ client token alias（:1219-1246）。 +- 容量：队列满时 best-effort 来源（Scanner/AutoHeal/ReadRepair）或低优先级被 Dropped(QueueFull)；Admin/Internal 可驱逐低优先级排队项（`push_displacing_lower_priority` :353-396）；80%/95% 压力分级（:885-909）。 +- 并发：全局 `max_concurrent_heals`（默认 4）+ per-set bulkhead `max_concurrent_per_set`（默认 1）（:3040-3073,3434-3447）。 +- 前台压力门控 mainline throttle：前台读/写 permit 利用率 ≥80% 时延迟 best-effort 任务（:919-1009,2999-3020）。 +- 超时：任务级聚合超时（默认 300s），跨重试保留剩余预算（task.rs:444-451，PR #6101）。 +- 可恢复重试：`is_recoverable_heal()`（error.rs:83-136）≤3 次、2^n 退避封顶 30s；retry 在独立 backoff task 中持有所有权（:3235-3382）。 +- 完成态保留 10 分钟供查询（:42）。 + +### 2.7 Admin API 与集群协调 + +- 路由（`rustfs/src/admin/handlers/heal.rs:174-212`）：`POST /rustfs/admin/v3/heal/`、`/heal/{bucket}`、`/heal/{bucket}/{prefix}`（同一 POST 按 query `clientToken/forceStart/forceStop` 区分 start/query/cancel，与 mc admin heal 语义对齐）；`POST /v3/background-heal/status`；`GET /v4/heal/replacement-recovery`。权限 `HealAdminAction`（route_policy.rs:334-341）。 +- 集群协调（heal.rs:771-930 + `node_service.rs:514-606`）：`heal_topology_fingerprint` + 按拓扑确定性选 coordinator 节点 + coordinator epoch；envelope 校验 + SHA256 digest 重放缓防重放；coordinator 非本机走 peer gRPC `heal_control`；`probe_heal_control` 能力探测（滚动升级场景）。 +- 请求：body 为 `HealOpts`（`recursive/dryRun/remove/recreate/scanMode(0/1/2)/updateParity/nolock/pool/set`，serde camelCase，与 madmin.HealOpts 字段对齐）；根 heal start 需 `recursive=true` 或 `pool+set` 成对；body 上限 1MB。 +- 响应：`HealStartSuccess{clientToken, clientAddress, startTime}`；`HealTaskStatus{summary, detail, startTime, settings, items, truncated, progress}`（summary ∈ running/finished/stopped/notFound）；`BackgroundHealStatus`（bitrot 起始时间/周期/当前模式 + `disabled/uninitialized/idle/active/degraded` 状态——peer 不可达显式 degraded 不冒充 idle，issue #5850 + `healOperations` 按优先级×来源矩阵 + 集群进度）。 +- `HealResultItem`/`HealDriveInfo`/`HealItemType`/DriveState 枚举与 madmin JSON 兼容（`crates/madmin/src/heal_commands.rs:19-65`）。 +- 状态 payload 超 8MiB 对折截断（channel.rs:37,73-104）；path-token 校验（错误 token 拒绝，空 path 仅匹配 Cluster）。 + +### 2.8 heal 指标与日志 + +指标：`rustfs_heal_admission_total{source,result,reason,context}`、`rustfs_heal_task_start_total`、`rustfs_heal_task_running{type,set}`、`rustfs_heal_queue_delay_seconds`、`rustfs_heal_scheduler_skip_total`、`rustfs_heal_mainline_throttle_total`、`rustfs_heal_page_concurrency_current{set}`、`rustfs_heal_candidate_enqueue/merge/drop/priority_reject_total`、`rustfs_heal_read_repair_dedup_total{reason}` 等。日志全部结构化 event style（PR #5720）；per-object 日志降级防风暴（`demote_to_debug_when!`，#5716/#5719/#5727）。 + +--- + +## 3. Scanner 已实现功能全景 + +### 3.1 循环、leader、立即触发 + +- **集群单 leader**：分布式 ns 写锁 `leader.lock`（`scanner.rs:3156-3207`，超时默认 5s）+ **持久化 leader-epoch CAS 围栏**：leader 用 ETag 前置条件向 `.bloomcycle.bin` 写 `RSCYC001` 编码的 (cycle, leader_epoch)（`scanner.rs:118,1850-1861,2177-2334`）；usage 快照再打 epoch fence（:2087-2153）。锁丢失 → 取消当前周期，30s 收敛（:108-111,2623-2642）。 +- 抢锁后立即执行一轮；周期 = `RUSTFS_SCANNER_CYCLE` > config cycle > start_delay > 部署默认 > 速度档位（±10% 抖动、下限 1s）。 +- **clean-idle 指数退避**：连续完整无脏周期间隔 ×2（封顶 24h；bitrot 周期压缩上限；桶有 lifecycle/replication 活动规则禁用，:383-456,1382-1512）。 +- **superseded/deferred 退避**：5s 起指数退避封顶 30min（:105-106,3432-3438）；维护探测失败独立退避（:459-505）。 +- **立即唤醒**：① dirty-usage 快路径——写路径 put/delete/multipart/bucket 操作调用 `record_dirty_usage_bucket`（`scanner_io.rs:222-235`；调用点 `rustfs/src/app/object_usecase.rs:6221` 等），自增 generation 并 Notify 唤醒 leader，脏桶优先排队（`scanner_io.rs:462-488`）；② 维护配置变更（lifecycle/replication 设置时 `record_scanner_maintenance_change`）；③ 运行时配置热更 generation+Notify；④ 集群活动快照变化。 +- **集群协调**：`probe_scanner_activity` 汇集本机+peer 的 `ScannerNodeActivity`（instance_id/namespace_generation/maintenance_generation/protocol_version/topology_digest/data_movement_active/dirty usage），拓扑摘要覆盖 pools/sets/drives URL，协议版本不齐拒绝共享缓存锁（`scanner.rs:970-1068`）；**数据迁移（rebalance/decommission）期间推迟周期**（`scanner_io.rs:2226-2374`）；周期结束逐 peer RPC 确认 dirty-usage ack（`scanner.rs:2925-2952`）。 + +### 3.2 遍历模型 + +- 主遍历是**全量目录 walk**（tokio::fs::read_dir 递归，`scanner_folder.rs:1915-2234`），不走 metacache；metacache/`list_path_raw` 仅用于 abandoned children 跨盘核查（:2528-2792）。 +- 三级并发：leader → per-set（信号量默认 4）→ per-disk 桶扫描（默认 4）→ 单盘递归；每桶每 set 缓存锁 `.scanner-cycle.lock.pool-N.set-M`（锁丢失取消该桶扫描，锁竞争重排队）；每盘单扫描准入（本地盘也走信号量，`scanner_io.rs:3246-3274`）。 +- 桶顺序：shuffle 后按 dirty → 未缓存 → 已缓存重排（`scanner_io.rs:2947-2949,462-488`）；目录内按名字排序 + resume 提示旋转（`scanner_folder.rs:333-359`）。 +- **断点续扫**：`DataUsageScanCheckpoint{version,resume_after,reason}` 持久于缓存 info（`data_usage_define.rs:68,293-307`）；预算耗尽/取消写入，恢复有 Used/Stale/NoHint 指标；续扫单位是目录（无跨周期对象级分页）。 +- erasure 语义：发现 `xl.meta` 即对象边界不下钻；UUID data-dir 候选最多探测 64 entry；有数据无元数据 → 记 failed + 高优 heal；symlink 目录忽略/环跳过。 +- 协作让出：每 N 对象（默认 128）`yield_now`。 + +### 3.3 大桶跳过策略（对标 MinIO compaction） + +1. 缓存当前性复用：桶与扫描计划未变（name/source/snapshot_complete/plan digest/next_cycle/leader_epoch/cache_key_format 全匹配）整桶跳过（`scanner_io.rs:1062-1109`）。 +2. compacted 目录 16 周期轮换窗口：`hash mod (next_cycle, 16)` 命中才重扫，否则从旧缓存拷贝（`scanner_folder.rs:74,2429-2442`）。 +3. compaction 阈值：子项 <500 或纯对象叶子压缩为单 entry；子文件夹 ≥2500（根 10000）预压缩；children ≥10000 归约（:75-78,2314-2340,2846-2887）。 +4. 失败对象 TTL 跳过：86400s/最多 10000 条（:88-91,1354-1381）。 + +与 MinIO master 对比：MinIO 的跳过策略同样是 hash-mod-16 周期 + compaction 阈值树（500/10000/2500），**bloom filter 已从 master 删除**。RustFS 的常量与结构与 MinIO 现状同源（MinIO 未采用跨盘 dirty-generation 优先，RustFS 额外多两层跳过——plan digest 与缓存当前性校验）。 + +### 3.4 data usage 统计 + +- 维度：每目录 entry（size/objects/versions/delete_markers/大小直方图/版本直方图/复制统计/failed_objects/per-tier stats/children/compacted，`data-usage/src/data_usage.rs:661-679`）；每对象 SizeSummary（含 per-ARN 复制目标统计、tier 统计，tier 分类：transitioned 完成记入其 tier 否则按 storage class，free version 不计）；桶级 `BucketUsageInfo`；集群级 `DataUsageInfo`（含 scanner_cycle/scanner_epoch 围栏 + usage_snapshot_complete）。 +- 存储：每桶每 set `{bucket}/.usage-cache.bin`（主 + `.bkp` 备份 + CAS 重试）；权威集群快照 `buckets/data-usage/data-usage.json`（每 10 周期同步 `.bkp`，legacy 路径兼容）；陈旧快照拒绝写入（epoch/cycle/last_update 三重判定）；被竞争 superseded 的观测快照另存 `data-usage-observed.json`。 +- 消费：`replace_bucket_usage_memory_from_info` 刷新桶用量内存 + 两层缓存失效（`scanner.rs:4142-4152`）→ bucket stats/quota/admin account_info/system；写路径内存实时叠加 overlay；启动读快照判断冷缓存跳过启动延迟。 +- 未完成 multipart 不参与统计（与 MinIO 一致，MinIO 也不扫 multipart 桶）。 + +### 3.5 ILM 集成 + +- 每对象 `ScannerItem::apply_actions`（`scanner_folder.rs:747-1032`）：`Evaluator::new(lifecycle).with_lock_retention(...).with_replication_config(...).eval()` 批量评估。 +- 已实现动作（IlmAction 全集，`common/src/metrics.rs:34-45`）：expiry 删除（Delete/DeleteRestored/DeleteRestoredVersion）、全版本删除（DeleteAllVersions/DelMarkerDeleteAllVersions，处理后停止后续版本）、transition（Transition/TransitionVersion，tier 列表运行时读取）、noncurrent 批量（DeleteVersionAction → `enqueue_by_newer_noncurrent`）、free-version 清理（`enqueue_free_version`）、object-lock retention 约束。**与 MinIO 的 9 个 ILM 动作一一对应**。 +- 执行模型：scanner 是"发现与入队"角色（expiry 队列/transition 队列在 ecstore `bucket_lifecycle_ops.rs`），动作由 worker 池消费——与 MinIO globalExpiryState/globalTransitionState 同型。 +- AbortIncompleteMultipartUpload 不在 scanner/ILM 内执行（MinIO 同样不在：`internal/bucket/lifecycle/rule.go` 有 FIXME，实际由 `erasureSets.cleanupStaleUploads` 全局例程承担）；RustFS 由 ecstore 独立后台任务 `init_background_stale_multipart_upload_cleanup`（`bucket_lifecycle_ops.rs:3289-3320`）+ 桶删除时 on-demand。 +- 集成测试覆盖：transition+restore、free-version、noncurrent、delete-marker、0-day、后台扫描过期（`scanner/tests/lifecycle_integration_test.rs:1071-2095`）。 + +### 3.6 heal 候选生产（scanner 侧） + +- 抽样：`hash mod_alt(next_cycle/prob_div, 1024/prob_div)`，进入 compacted 分支重扫时 prob_div=16 等效概率 ×16（与 MinIO 同款补偿，`scanner_folder.rs:125-127,2117-2122`）。 +- deep/normal：周期级 `get_cycle_scan_mode`（bitrot_cycle 默认 30d，`scanner.rs:1626-1657`）→ 对象级带 `HealScanMode::Deep`；新鲜对象（60s 内修改）降级 Normal（:146-155）；状态持久 `.background-heal.json`（`BackgroundHealInfo{bitrot_start_time,bitrot_start_cycle,current_scan_mode}`，与 MinIO 同路径同结构）。 +- scanner 只入队不内联执行（内联 heal 已移除，兼容旗标仅告警，`scanner_folder.rs:411-427`）；`HealScanMode::Deep` 只是标记，bitrot 校验读发生在 heal 消费端（ecstore Deep 路径）。 +- 元数据损坏 → 高优 heal（`classify_get_size_failure` → HealMetadata）；abandoned children → list_path_raw quorum 核查 + 桶级/对象级高优 heal；healing 盘粘性跳过（`should_heal` :1628-1648）。 +- pending-heal 账本：heal 通道满被拒持久化到缓存 info，下轮重试。 +- 复制 heal：`queue_replication_heal` → replication 队列（走 replication 通道而非 heal channel）；per-ARN 复制用量统计。 + +### 3.7 remote_scanner RPC 协议（RustFS 特有） + +请求 ≤16KB msgpack（version/request_id/server_epoch/session_id/session_sequence/bucket/next_cycle/leader_epoch/scan_plan_digest/skip_healing/scan_mode/budget）；帧 ≤2MB、HMAC-SHA256 逐帧认证（域 `rustfs-ns-scanner-frame-v3`）；进度心跳 1s（预算模式 250ms）；阶段播报 Scanning→Persisting；RPC 生命周期上限 24h、断连宽限 2min；防重放 session+sequence 缓存（容量 65536）；服务端校验 leader fence 与持久化 cycle 一致 + 每 5s fence 复验；结果 Complete/Partial/NamespaceNotFound/CycleAhead；不支持 v4 协议的远端盘回退 leader 本地扫描（`remote_scanner.rs` 全文件；`scanner_io.rs:2750-2812`）。 + +### 3.8 限速/预算/热更/观测 + +- DynamicSleeper 比例退避（速度档 fastest/fast/default/slow/slowest，同 MinIO 五档参数）；idle_mode 总闸；前台 S3 读流量每请求 10ms 封顶 250ms 额外退避。 +- 周期预算 ScannerCycleBudget：max_duration/max_objects/max_directories（默认 0=不限），partial 周期仍推进 cycle 计数。 +- runtime_config 三层来源（env > config > default）逐字段来源标记（Env/Config/ScannerCompatConfig/Default），admin `PUT /v3/config` 热更 → generation+Notify 即时生效；`GET /v3/scanner/status` 返回 enabled/freshness(fresh/stale/unknown)/metrics/cycle_schedule/runtime_config；`GET /v3/ilm/expiry/status` 返回 expiry 队列/worker/missed/blocked。 +- 指标：leader lock、周期 complete/partial/deferred/superseded、versions scanned、per-source（Usage/Lifecycle/BucketReplication/SiteReplication/Heal/Bitrot/Alerts）checked/executed/queued/missed、checkpoint set/used/stale、当前路径（per-disk+bucket 实时）、缓存 save 系列、并发系列、告警（excess versions/version size/folders）。 + +--- + +## 4. 与 MinIO 逐项对标 + +### 4.1 heal 触发通道对照 + +| MinIO 通道 | RustFS 对应 | 状态 | +|---|---|---| +| A. 手动 admin heal（healSequence，clientToken/forceStart/forceStop） | heal channel Start/Query/Cancel + 集群 coordinator + envelope 重放防护 | ✅ 等价且增强（集群路由）；序列语义差异见 §6 HS-06 | +| B. 常驻后台 heal 队列（newBgHealSequence + healRoutine worker 池） | HealManager 常驻调度器 + 优先级队列 + bulkhead | ✅ 等价且增强 | +| C. 新盘/换盘自动 resync（monitorLocalDisksAndHeal 10s + healFreshDisk + healingTracker + waitForFormatErasure 握手） | auto disk scanner（10s）+ replacement_readiness + durable intent/proof 状态机 + heal_replacement_format | ✅ 等价且增强（identity fence + completion proof；MinIO 的 tracker 面向对外可见性更强，见 §6 HS-07） | +| D. MRF（队列 100k + 持久化 list.bin + shutdown 回放 + 读路径 corrupt 投递） | read-repair（Low+TTL 去重）+ 写路径 convergence heal 部分承担；`HealType::MRF` 执行体无生产入口 | ⚠️ 部分等价（§6 HS-01） | +| E. Scanner 抽样 heal（1/1024 + compacted ×16 补偿）+ abandoned children | 同款抽样 + ×16 补偿 + abandoned children + pending-heal 账本 | ✅ 等价且增强（账本） | +| F. 读路径内联触发 → MRF（GetObject part 缺失/损坏、元数据重建 missingBlocks>0） | read repair（missing_shards/decode_error/metadata_read_error 三入口） | ✅ 等价（入 heal 队列而非 MRF 队列） | + +### 4.2 对象级 heal 语义对照 + +| 特性 | MinIO | RustFS | 状态 | +|---|---|---|---| +| mod-time quorum 仲裁 | listOnlineDisks | 同 | ✅ | +| ETag 多数派兜底（时钟漂移） | filterDisksByETag | `filter_by_etag`/`quorum_etag`（heal.rs:525-567） | ✅ 已亲验 | +| cannotHeal 的 ETag 豁免 | ETag 全一致豁免重试 | heal.rs:679 | ✅ | +| Normal=CheckParts（stat）/ Deep=VerifyFile（bitrot） | 是 | `disks_with_all_parts` 按 scan_mode（ops/heal.rs:562-572,978-1024） | ✅ | +| Normal 检出 corrupt 自动升 Deep 重试一次 | erasure-healing.go:1101-1106 | ops/heal.rs:2022-2031 | ✅ | +| dangling 判定（not-found > parity）+ 删除审计 | isObjectDangling/deleteIfDangling | `dangling_delete_safety`（:1488）+ scanner HEAL_DELETE_DANGLING | ✅（审计 tags 细节有差异） | +| 孤儿 data-dir/inline 清理（CleanAbandonedData） | CheckAbandonedParts（scanner 抽中 + admin Remove 时显式调用） | heal 路径内 `reclaim_orphan_data_dirs_best_effort`（:1428）；独立 API 三层 NotImplemented | ⚠️ 部分等价（§6 HS-02） | +| 版本化/delete-marker heal | HealObject versionID；nullVersionID 特判 | 逐版本枚举 + delete-marker latest heal（B5 回归） | ✅ | +| 对象级 healing 元数据标记（x-minio-healing，RenameData 跳过版本清理） | 有 | 无对象级标记；依赖盘级 healing.bin + NSLock + rename 语义 | ⚠️ 评估项（§6 HS-12） | +| Distribution/Index 一致性三处防线 | 有（manual modification 拒绝） | 目标盘格式结果全 ok 校验 + 身份围栏 | ✅（粒度不同） | +| 无 parity（EC:0）对象 | bitrot 不可恢复处理 | 判不可恢复（:700-726）+ 写入自校验 | ✅ 增强（写路径自校验） | +| 三层分布不一致拒绝 heal | 有 | heal_walk 归一化 + 页界防御 | ✅（实现方式不同） | +| multipart 孤儿对账 | CheckAbandonedParts 承担 | 显式 NotImplemented（由 lifecycle 清理承担） | ⚠️ §6 HS-02 | +| suspended/decommissioned pool 处理 | IsSuspended 跳过 | deferral 语义（store/heal.rs:192-207，PR #5876） | ✅ | +| heal 与并发删除互斥 | NSLock + healing 标记 | NSLock + 写锁 | ✅ | + +### 4.3 新盘 resync 对照 + +| MinIO | RustFS | 状态 | +|---|---|---| +| waitForFormatErasure 四类可恢复错误无限等待握手 | startup 盘解析 + renew_disk 重连路径 | ✅（模型不同：RustFS 不在启动时阻塞等待 format） | +| HealFormat NSLock + errNoHealRequired + refFormat 不一致拒绝 | `heal_format`/`heal_replacement_format` fail-closed + 目标槽位限定（PR #1787 语义） | ✅ 增强 | +| per (pool,set) 分布式锁防并发 resync | set 级队列去重 + bulkhead（manager.rs:2854-2889） | ✅ | +| 全新集群检测（待 heal 盘数==总盘数不触发） | replacement_readiness（独立挂载点/物理设备校验，非 root） | ✅ 增强 | +| healingTracker（.healing.bin：Bytes/Items 计数、QueuedBuckets/HealedBuckets、Resume 快照、RetryAttempts ≤4、HealID 联动、diskID 变更重置） | resume/checkpoint schema 化持久层 + durable intent/proof（per-task 文件，CAS） | ✅ 等价且增强（崩溃窗口补齐）；但**对外快照可见性**弱于 MinIO（§6 HS-07） | +| 跳过 heal 开始后新写入版本（ModTime > Started） | 无同款过滤 | ⚠️ §6 HS-13 | +| 跳过 ILM 已过期版本（filterLifecycle） | 无同款过滤 | ⚠️ §6 HS-13 | +| worker 数 max(GOMAXPROCS,NR)/4 下限 4，heal:drive_workers 覆盖 | 页内并发 8（Deep/AutoHeal 强制 1）+ per-set bulkhead | ✅（参数模型不同） | +| 每 entry waitForLowHTTPReq 让路 | mainline throttle（前台利用率门控） | ✅ 增强 | +| heal 范围含 `.minio.sys/config`、`.minio.sys/buckets` 两个伪桶；最新桶优先 | ErasureSet 任务逐 bucket 预处理（含 meta bucket 语义由 heal_bucket 承担） | ✅（顺序无"最新优先"） | +| 失败整体重试 ≤4 次（resetHealing + errRetryHealing） | schedule_retry 复位双层 + 可恢复重试 ≤3 | ✅ | + +### 4.4 scanner 对照 + +| MinIO | RustFS | 状态 | +|---|---|---| +| 集群单 leader（globalLeaderLock） | leader.lock + 持久化 leader-epoch CAS 围栏 | ✅ 增强（epoch 围栏防脑裂，MinIO 无持久化 epoch） | +| `.bloomcycle.bin` 只存 cycle（bloom 已删除） | 同路径存 cycle+leader_epoch（RSCYC001） | ✅ 对齐（v1 误判已修正） | +| folderScanner hash-mod-16 + compaction（500/10000/2500） | 同款常量 + plan digest + 缓存当前性校验 + dirty 优先 | ✅ 增强 | +| 每盘扫描并行 ≤GOMAXPROCS；healing 盘排除 | per-set/per-disk 信号量 + healing 盘粘性跳过 | ✅ | +| scannerSleeper（factor 2/max 1s，speed 档热更） | DynamicSleeper 同款 + idle_mode + 前台读退避 | ✅ 增强 | +| idle 语义：`scanner:idle_speed=on`（空闲时段才节流，忙时全速） | `RUSTFS_SCANNER_IDLE_MODE=true`（启用限速总闸） | ⚠️ 语义方向相反，§6 HS-14 | +| applyActions 顺序（heal→ILM→复制→告警） | apply_actions 同序（heal 候选→ILM→复制 heal→告警） | ✅ | +| ILM 9 动作 + 批量评估 + DeletePrefixObject 优化 | 同 9 动作 + 批量评估 + expiry 队列 | ✅（DeleteAllVersions 是否单调用优化未逐行核） | +| abandoned children（listPathRaw minDisks=N/2 发现漏写盘） | list_path_raw + quorum 核查 + 高优 heal | ✅ | +| incomplete multipart 独立例程（6h 间隔/24h 过期，rename 进 .trash） | ecstore 独立后台任务（可配间隔/过期） | ✅（trash 二段清理细节差异，§6 HS-18） | +| usage 维度（size/objects/versions/DM/直方图/复制/tier/bucket 级） | 全覆盖 + 集群快照三重防回退 | ✅ 增强 | +| prefix 级 usage（loadPrefixUsageFromBackend，console 消费） | 缓存内有目录树但仅 flatten 桶级 | ❌ §6 HS-08 | +| 超限事件 s3:ObjectManyVersions/LargeVersions/PrefixManyFolders + 审计 | 仅指标 alert_excess_*（默认 100/1TiB/65538 vs MinIO 100/1TB/50000） | ⚠️ §6 HS-04/HS-17 | +| scanner 指标 v3（bucket_scans/directories/objects/versions/last_activity） | rustfs_scanner_* 全套 + freshness | ✅（命名体系不同） | +| TraceScanner / realtime metrics（mc admin scanner status/trace） | 无 trace 通道；/v3/scanner/status 自有结构 | ⚠️ §6 HS-03 | + +### 4.5 admin/CLI/API 面对照 + +| MinIO | RustFS | 状态 | +|---|---|---| +| `POST /minio/admin/v3/heal/...` start/status/cancel | `POST /rustfs/admin/v3/heal/...` 同三态 | ✅（路径前缀不同属预期） | +| `HealStartSuccess`/`HealTaskStatus`/`HealResultItem`/DriveState | 同名字段 JSON 兼容 | ✅ | +| `POST /v3/background-heal/status`（BgHealState 聚合） | 同路径 + degraded 语义 + operations 矩阵 | ✅ 增强（MRF per-endpoint 子状态无，因无 MRF） | +| `GET /v3/healthinfo` 每 drive `HealInfo *HealingDisk` | 无同款 healthinfo heal 字段（replacement-recovery v4 承担部分） | ⚠️ §6 HS-07 | +| madmin 客户端 HealStart/HealStatus/BackgroundHealStatus/ScannerStatus 方法 | 仅 wire 类型，无客户端方法 | ❌ §6 HS-05 | +| mc admin heal --pool/--set、--scan-mode、--force-start/stop | HealOpts 全字段支持（pool/set/scanMode/forceStart/forceStop） | ✅（服务端就绪；缺 mc 侧入口，HS-05） | +| ErrHealAlreadyRunning / ErrHealOverlappingPaths 类型化错误 | 去重合并 + 驱逐语义；无类型化重叠拒绝 | ⚠️ §6 HS-06 | +| 结果 backpressure（maxUnconsumedItems=1000、10s 保活流式、24h 未消费 abort） | 快照式查询（1024 条 + 8MiB 截断 + 10min 保留） | ⚠️ §6 HS-06 | +| `mc support inspect`/healing-bin 离线 dump | 无（inspect.rs 存在但 healing dump 未确认） | ⚠️ P3 | + +### 4.6 观测面对照 + +| 维度 | MinIO | RustFS | 状态 | +|---|---|---|---| +| heal 指标 | minio_heal_objects_total/heal_total/errors_total/time_last_activity + v3 drive_health 2=healing | rustfs_heal_* 全套（admission/queue delay/running/throttle/page concurrency） | ✅（RustFS 缺 drive_health=healing 单一 gauge 等价物；DiskInfo.healing 已赋值） | +| scanner 指标 | v3 6 个 + realtime 18 项 | rustfs_scanner_* 全套 + per-source 维度 | ✅ | +| ILM 指标 | v3 5 个（expiry/transition pending/active/missed + versions_scanned） | ilm expiry status API + scanner per-source | ✅（指标与 API 形态不同） | +| trace | TraceHealing/TraceScanner 两通道 | 无 | ❌ §6 HS-03 | +| 审计 | HealObject 事件、dangling 删除审计、scanner:manyversions 等 | 结构化日志（event style）+ 指标；无 audit log 事件 | ⚠️ §6 HS-04 | +| 进度 | healingTracker Bytes/Items/QueuedBuckets/当前对象 + usage-cache 总量基线 | HealProgress{scanned/healed/failed/bytes/current_object/percentage}；bytes_processed 注释为 0、estimated_completion_time 恒 None | ⚠️ §6 HS-07 | + +### 4.7 配置面对照（默认值） + +| MinIO | RustFS | 备注 | +|---|---|---| +| `heal:bitrotscan`（默认 off；on=每轮；Nm=N×30×24h） | `heal.bitrot_cycle` / `RUSTFS_SCANNER_BITROT_CYCLE_SECS`（默认 30d=2592000s；0/on=每轮 Deep，off=禁用） | ✅ 同语义（RustFS 默认 30d，MinIO 默认 off——**默认值不同**，RustFS 更激进） | +| `heal:max_io=100`/`max_sleep=250ms`（waitForLowIO） | mainline throttle 阈值 80%/80%、max_sleep 250ms | ✅ 同型（阈值模型不同） | +| `heal:drive_workers`（默认 -1 自动） | 页内并发 8 + per-set 1 | ✅ 同型 | +| `_MINIO_HEAL_WORKERS`（GOMAXPROCS/2） | `RUSTFS_HEAL_MAX_CONCURRENT_HEALS=4` + `_MAX_CONCURRENT_PER_SET=1` | ✅ | +| `_MINIO_AUTO_DRIVE_HEALING`（on） | `RUSTFS_HEAL_AUTO_HEAL_ENABLE=true` | ✅ | +| `_MINIO_SCANNER`（on） | `RUSTFS_SCANNER_ENABLED=true` | ✅ | +| `scanner:speed` 五档（default=2x/1s/1m） | 同五档同名同参数 | ✅ | +| `scanner:idle_speed`（on） | `RUSTFS_SCANNER_IDLE_MODE`（true） | ⚠️ 语义方向（HS-14） | +| `scanner:alert_excess_versions=100` | 100 | ✅ | +| `scanner:alert_excess_folders=50000` | 65538（兼容 PBS 布局） | ⚠️ HS-17 | +| `ilm:expiration_workers=100`/`transition_workers=100` | ecstore expiry/transition worker 池（键见 ilm 子系统） | ✅（默认值未逐项核对） | +| `api:stale_upload_cleanup_interval=6h`/`expiry=24h` | ecstore 后台任务 env 可配 | ✅（默认值未逐项核对） | +| —（无） | `RUSTFS_HEAL_QUEUE_SIZE=10000`、`_TASK_TIMEOUT_SECS=300`、`_INTERVAL_SECS=10`、`_LOW_PRIORITY_MERGE/DROP`、`_PAGE_*`、`_SET_BULKHEAD`、`_MAINLINE_*`、`RUSTFS_SCANNER_CYCLE_MAX_*` 预算、`_MAX_CONCURRENT_SET/DISK_SCANS=4`、`_YIELD_EVERY_N_OBJECTS=128` 等 | RustFS 特有（更细粒度） | + +### 4.8 RustFS 超出 MinIO 的部分 + +1. remote_scanner RPC（扫描执行下放远端 peer 本地，含 HMAC 认证/重放缓存/fence 复验/断连宽限）。 +2. 持久化 leader-epoch CAS 围栏 + usage 快照 epoch/cycle 防回退（MinIO 仅锁，无持久 epoch）。 +3. 周期预算（max_duration/objects/directories）+ partial 周期推进语义。 +4. per-set/per-disk 扫描并发闸 + 每桶每 set 缓存锁。 +5. pending-heal 账本（heal 通道满不丢候选）。 +6. 换盘 durable intent + completion proof 状态机 + 身份围栏（MinIO healingTracker 无 proof）。 +7. mainline throttle 前台压力门控（permit 利用率驱动）。 +8. 集群 heal control coordinator + envelope 重放防护 + degraded 显式降级。 +9. 写路径 shard bitrot 自校验（EC:0 场景）。 +10. dirty-usage 快路径唤醒（写路径即时通知 + 脏桶优先）。 +11. heal 运行时可观测矩阵（优先级×来源 operations snapshot）。 +12. workload admission 联动（heal 调度器读前台压力快照）。 + +--- + +## 5. 差距与改进清单 + +分级定义：P1=行为/运维对齐缺口（影响生产运维或工具链兼容）；P2=完善性（功能在但缺一角）；P3=清理/低风险。每项含现状证据、MinIO 行为、影响、建议、验收方式。 + +### P1（8 项） + +**HS-01 MRF/ECDecode/Metadata 三类 heal 任务无生产触发入口，HealEvent 未接线** +- 现状：`HealType::MRF/ECDecode/Metadata` 执行体完整（task.rs:1700-2156）但全仓库无生产触发方；`HealEvent`/`HealEventHandler`（event.rs:50-367）crate 外零引用（已亲验 grep）；channel 转换只产生 Cluster/Object/Bucket/Prefix/ErasureSet（channel.rs:566-601）。 +- MinIO：mrf.go 独立 MRF 队列（容量 100k，满丢弃计数）、进程退出 msgp 持久化 `.heal/mrf/list.bin` + 启动回放、入队 <1s 延迟 1s（等网络恢复）、healSleeper 限速；读路径 GetObject part 缺失/损坏、元数据重建 missingBlocks>0、Put 部分成功、DeleteObject、multipart、peer client 共 7+ 投递点。 +- 影响：RustFS 的 read-repair + 写路径收敛覆盖了主场景，但缺少：① 事件驱动的 Urgent ECDecode 重建入口（ecstore 解码失败时目前仅 Low read-repair）；② Metadata-only heal 入口（scanner HealMetadata 分类存在但走普通对象 heal）；③ MRF 队列持久化（重启丢未消费修复意图——scanner pending-heal 账本部分缓解）。 +- 建议：三选一决策——(a) 接线 HealEvent（在 ecstore 解码失败/metadata 损坏点发事件）+ 实现持久化重试账本；(b) 删除 MRF/ECDecode/Metadata 死代码只保留文档说明；(c) 保留执行体、把 HealEvent 降级为内部 API。推荐 (a) 但需先量化 read-repair 是否已覆盖解码失败场景的响应时间要求。 +- 验收：解码失败 → Urgent heal 请求链路 e2e；重启后 pending 修复意图回放；HealEvent 环形缓冲指标。 + +**HS-02 CheckAbandonedParts 三层 NotImplemented（abandoned data 独立对账入口缺失）** +- 现状：`set_disk/ops/heal.rs:2052-2056`、`core/sets.rs:1144-1148`、`store/heal.rs:258-266` 三层显式 `Err(NotImplemented)`（已亲验），注释"intentionally retained above the set layer until there is a concrete caller"。 +- MinIO：`CheckAbandonedParts` → 每盘 `CleanAbandonedData`：读 xl.meta → 列 UUID data-dir + inline entries → 与 getDataDirs 差集 → 删多余 data-dir/inline 并重写 xl.meta；由 scanner 抽中 heal 与 admin heal Remove 时显式调用。 +- 影响：RustFS heal 路径内 `reclaim_orphan_data_dirs_best_effort`（:1428）覆盖"heal 时回收孤儿目录"，但 ① 无独立触发点（MinIO 在对象未到 heal 阈值时也能清 abandoned data）；② inline data 孤儿条目清理未确认；③ multipart 孤儿对账明确不做（设计决定，由 lifecycle 承担）。 +- 建议：评估把 `reclaim_orphan_data_dirs_best_effort` 提升为 heal_object 固定步骤（若尚非）+ 实现 HealOperations::check_abandoned_parts 真实现（调用同一回收逻辑），或明确文档化"由 lifecycle 承担"并关闭 API 面。 +- 验收：构造 data-dir/inline 孤儿 → scanner 抽样/admin heal 后被清理；三层 API 返回成功或显式 NotSupported 文档化。 + +**HS-03 heal/scanner trace 通道缺失** +- 现状：TraceHealing/TraceScanner 零命中（已亲验 grep 全仓库）。 +- MinIO：`madmin.TraceHealing`（mc admin trace --healing，FuncName=heal.Bucket/heal.Object/heal.CheckAbandonedParts，带 dry/remove/mode/version-id/disks/bytes）、`TraceScanner`（mc admin scanner trace，支持 --filter-size/--response-duration）。 +- 影响：无法实时观测单个 heal/scanner 动作的耗时与参数；排障只能靠指标聚合与日志。 +- 建议：在 heal channel 执行与 scanner folder/item 处理埋点，接入现有 admin trace 订阅面（若 rustfs 已有 trace 基建则复用，无则按 madmin TraceType 扩展）。 +- 验收：mc 等价工具能订阅 heal/scanner trace 流。 + +**HS-04 scanner 超限 S3 事件与审计缺失** +- 现状：仅 `rustfs_scanner_excess_*_total` 指标（versions 100/version size 1TiB/folders 65538）。 +- MinIO：发 `s3:ObjectManyVersions`（>100 版本）、`s3:ObjectLargeVersions`（累计 >1TB）、`s3:PrefixManyFolders`（>50000 子目录）事件（UserAgent: Scanner）+ scanner:manyversions/largeversions/manyprefixes 审计。 +- 影响：依赖事件订阅做容量治理的用户（console/外部审计）收不到告警。 +- 建议：scanner_folder 告警点接入 notify 事件发布（复用 lifecycle 事件通道语义）。 +- 验收：配置桶通知后超限对象触发事件。 + +**HS-05 madmin 客户端方法缺失** +- 现状：`crates/madmin/src/heal_commands.rs` 只有 wire 类型（HealDriveInfo/Infos/HealResultItem）；无 HealStart/HealStatus/BackgroundHealStatus/ScannerStatus 客户端方法。 +- MinIO：madmin-go 提供完整客户端；mc admin heal/scanner/status/trace 都建立在上面。 +- 影响：mc 等管理工具无法直接对接 RustFS heal/scanner 管理面；自动化运维只能手写 HTTP。 +- 建议：按 madmin-go 接口形状补客户端（服务端已就绪，纯客户端工作）。 +- 验收：用 madmin 客户端完成 start→query→cancel 全流程。 + +**HS-06 admin heal 序列语义与 MinIO 差异** +- 现状：重复/重叠请求被去重合并（返回 canonical task_id）或驱逐；无 ErrHealAlreadyRunning/ErrHealOverlappingPaths 类型化错误（已亲验：manager.rs:1309 的 already_running 是幂等启动保护，非 admin 语义）；结果为快照式查询（1024 条/8MiB 截断/10min 保留），非 MinIO 的流式增量（clientToken 拉增量 + maxUnconsumedItems=1000 backpressure + 10s 保活 + 24h 未消费 abort）。 +- 影响：mc admin heal 的交互模型（长连接拉增量）对 RustFS 表现为多次快照轮询；自动化脚本难以区分"已合并"与"新启动"。 +- 建议：① 增量语义：channel query 支持自上次 clientToken 起的 items 增量（或 cursor）；② 重叠请求返回类型化错误码（或 receipt 中显式 merged_into 字段——现有 alias 机制已有基础）；③ forceStart 先停旧再启新语义核对。 +- 验收：madmin 兼容客户端按 MinIO 模式轮询能取得全量 items。 + +**HS-07 healing 进度与盘级 healing 状态对外可见性不足** +- 现状：bytes 恢复进度 `progress.bytes_processed = 0 // set to 0 for now`（erasure_healer.rs:967）；`HealProgress::estimated_completion_time` 恒 None、`HealStatistics::add_healed_objects` 未写入（progress.rs:38,135-139 零调用）；healthinfo 无每盘 HealInfo 等价（MinIO HealingDisk：BytesDone/Failed/Skipped、ObjectsTotal 基线、QueuedBuckets/HealedBuckets、Resume 快照、当前 object）；v3 指标无 drive_health=2(healing) 单一 gauge 等价。 +- 影响：换盘重建（可能数小时~天）期间运维无法回答"进行到哪/还剩多少/预计何时完成"。 +- 建议：① erasure set heal 统计 bytes（heal_object 返回对象大小已可得）；② 从 usage-cache 读对象总量基线（MinIO 同款做法）；③ admin healthinfo/背景状态暴露每盘 healing 快照（DiskInfo.healing 已有，补聚合暴露）；④ ETA 由基线+速率推导。 +- 验收：换盘重建中 admin 可见 bytes 进度与 ETA；mc info 等价输出 Healing 标志。 + +**HS-08 prefix 级 usage 未暴露** +- 现状：DataUsageCache 内目录树 entry 存在（hash_path 组织），但 `dui()` 只 flatten 到桶名（data_usage_define.rs:858-915）。 +- MinIO：`loadPrefixUsageFromBackend`（30s cache）从每 set `.usage-cache.bin` 聚合 prefix usage，console 桶前缀统计消费。 +- 影响：console/前端无法展示前缀级用量；大桶定位"哪个前缀占空间"无 API。 +- 建议：实现 flatten 前缀查询 API（数据已在缓存内，纯聚合与暴露工作）。 +- 验收：ListBuckets/PrefixUsage API 返回与前缀过滤匹配的统计。 + +### P2（9 项） + +**HS-09 get_disk_status 恒返回 Ok（唯一 TODO）**：`crates/heal/src/heal/storage.rs:930-943`（已亲验）。当前无生产调用方（低风险）。建议：删除该方法或接 ecstore disk 状态真实现（DiskStatus 枚举已定义）。 + +**HS-10 HealStorageAPI 约 1/3 方法为死代码**：get_object_meta/get_object_data/put_object_data/delete_object/verify_object_integrity/ec_decode_rebuild/get_disk_status/format_disk/heal_bucket_metadata/get_object_size/get_object_checksum/list_objects_for_heal（非分页版，自带 memory_heavy 警告）均 0 调用方。建议：随 HS-01 决策一并清理或接线（死接口误导后续维护者以为存在调用路径）。 + +**HS-11 bitrot 自检缺失**：MinIO 启动时 bitrotSelfTest 对四算法已知向量自检失败即 Fatal（防静默数据损坏）。RustFS 无等价（已亲验 grep）。建议：启动时对 HighwayHash256S 等在用算法做已知向量自检（低成本高价值）。 + +**HS-12 对象级 healing 元数据标记评估**：MinIO heal 期间对象打 `x-minio-healing:true`，RenameData 据此跳过版本清理/legacy purge（漏掉会导致 heal 与并发删除互毁）。RustFS 无对象级标记（已亲验 grep object.rs 无 healing 分支），依赖 NSLock + rename 语义。建议：审计 RustFS rename 提交路径是否存在"heal 提交与并发 delete/version 清理竞争"窗口；若无则文档化差异，若有则补标记等价机制。 + +**HS-13 erasure set heal 无"跳过新写入/ILM 已过期版本"过滤**：MinIO resync 跳过 ModTime>tracker.Started 的版本（避免 heal 追新写入尾巴）与 ILM 已过期版本（避免白做）。RustFS erasure_healer 未实现同款过滤（按版本 dedup 有，时间/ILM 过滤无）。影响：重建尾部长尾（持续写入的桶 heal 完成判定被新版本推迟）与无效 heal 工作量。建议：disk-walk 枚举处加 started_at 时间过滤 + evaluator 预检。 + +**HS-14 scanner idle 语义方向与 MinIO 相反**：MinIO `scanner:idle_speed=on`（默认）= 集群空闲时才节流、忙时全速；RustFS `RUSTFS_SCANNER_IDLE_MODE=true`（默认）= 限速总闸（false=完全不休眠）。两者默认行为可能相近（都限速）但参数语义不可互换，迁移文档需显式说明；若追求 mc config 兼容需重命名/重语义。建议：先文档化差异，评估是否对齐语义。 + +**HS-15 alert_excess_folders 默认值差异**：RustFS 65538（兼容 PBS/Proxmox 布局，scanner_folder.rs:79）vs MinIO 50000。行为差异默认即触发阈值不同。建议：文档化（保留 65538 有本地理由）。 + +**HS-16 单机默认周期钩子未启用**：`single_disk_default_cycle_secs(_features) -> None` 恒空（scanner.rs:1428-1430），单机部署无专属默认周期覆盖。建议：决定单机默认周期策略后启用或删除钩子。 + +**HS-17 DeleteAllVersions 批量优化核对**：MinIO 用 DeletePrefix+DeletePrefixObject 单调用代替逐版本 fan-out。RustFS expiry 队列路径是否同款优化未逐行核实（集成测试覆盖行为正确性）。建议：核对 `apply_expiry_rule` 全版本删除路径，若无前缀单调用优化则评估补齐。 + +### P3（3 项） + +**HS-18 trash/临时目录二段清理细节核对**：MinIO `.minio.sys/tmp/.trash` 清理（delete_cleanup_interval 默认 5m + deleteCleanupSleeper）与 stale uploads rename-into-trash 二段式。RustFS 有 delete_tail_activity.rs 与 stale multipart 任务，二段语义是否完整对齐未逐行核实。建议：对照补齐或文档化。 + +**HS-19 root heal 直连死路径清理**：`should_handle_root_heal_directly` 恒 false（admin/handlers/heal.rs:1200-1202，测试锁定），store.heal_format 直连分支不可达。建议：删除死分支或恢复直连路径作为集群协调失败的降级。 + +**HS-20 兼容旗标与死指标清理**：`RUSTFS_SCANNER_INLINE_HEAL_ENABLE`（开启仅告警）+ `rustfs_scanner_inline_heal_total` 死指标 + `rustfs_common::metrics` 中 scanner 域代码分层迁移（backlog #1843 已登记）。建议：随分层迁移一并清理。 + +### 按设计不追平（7 项，记录以防后续误判为缺口） + +1. **bloom filter**：MinIO master 已删除；RustFS `.bloomcycle.bin` 复用为 cycle/epoch 围栏与 MinIO 现状一致。 +2. **scanner 集群单 leader**：双方一致；RustFS 额外有 epoch 围栏。 +3. **heal 不发 S3 bucket notification**：双方一致（heal 结果走 admin status）。 +4. **incomplete multipart 不在 scanner/ILM 内执行**：双方一致（独立后台例程）。 +5. **内联 heal 移除**：RustFS 有意为之（scanner 只入队），MinIO 的 applyHealing 内联路径不做对标。 +6. **heal 序列常驻保活（10s 空白回写）**：RustFS 快照式查询模型不同，按 HS-06 处理增量语义即可，不复制流式保活。 +7. **`.trash`/`tmp-old` 路径名兼容**：RustFS 布局常量独立，不逐字对齐 MinIO 路径。 + +--- + +## 6. 配置默认值总表（RustFS） + +heal（env 前缀 `RUSTFS_HEAL_`，`crates/config/src/constants/heal.rs`，消费于 `manager.rs:724-800`）： + +| 配置 | 默认 | 热更新 | +|---|---|---| +| AUTO_HEAL_ENABLE | true | 否 | +| QUEUE_SIZE | 10000 | 否 | +| INTERVAL_SECS | 10 | 否（启动时固定） | +| TASK_TIMEOUT_SECS | 300 | 否 | +| MAX_CONCURRENT_HEALS | 4 | 否 | +| MAX_CONCURRENT_PER_SET | 1（≤min(全局,值)） | 否 | +| LOW_PRIORITY_MERGE_ENABLE | true | 否 | +| LOW_PRIORITY_DROP_WHEN_FULL | true | 否 | +| PAGE_OBJECT_CONCURRENCY | 8（Deep/AutoHeal 强制 1） | 否 | +| EVENT_DRIVEN_SCHEDULER_ENABLE | true | 否 | +| SET_BULKHEAD_ENABLE | true | 否 | +| PAGE_PARALLEL_ENABLE | true | 否 | +| MAINLINE_THROTTLE_ENABLE | true | 否 | +| MAINLINE_READ/WRITE_UTILIZATION_HIGH_PERCENT | 80/80 | 否 | +| MAINLINE_MAX_SLEEP_MS | 250 | 否 | +| （总开关）RUSTFS_HEAL_ENABLED | true | 否 | +| admin 子系统 heal.bitrot_cycle | 30d | 是（经 scanner runtime config） | + +scanner（admin 子系统 `scanner`，`crates/config/src/constants/scanner.rs` + `ecstore/src/config/scanner.rs` + `runtime_config.rs:527-673`）： + +| 键 | env | 默认 | +|---|---|---| +| speed | RUSTFS_SCANNER_SPEED | default（2x/1s/60s） | +| delay / max_wait / cycle / start_delay | RUSTFS_SCANNER_* | 派生/空 | +| cycle_max_duration/objects/directories | …_MAX_* | 0（不限） | +| bitrot_cycle | …_BITROT_CYCLE_SECS | 2592000（30d；0/on=每轮，off=禁用） | +| idle_mode | …_IDLE_MODE | true | +| cache_save_timeout | …_CACHE_SAVE_TIMEOUT_SECS | 30s | +| max_concurrent_set_scans / disk_scans | …_MAX_CONCURRENT_* | 4/4 | +| yield_every_n_objects | …_YIELD_EVERY_N_OBJECTS | 128 | +| alert_excess_versions / version_size / folders | …_ALERT_* | 100 / 1TiB / 65538 | + +scanner 内部 env：`RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=16`、`RUSTFS_HEAL_OBJECT_SELECT_PROB=1024`、`RUSTFS_SCANNER_DEEP_VERIFY_COOLDOWN_SECS=60`、`RUSTFS_DATA_USAGE_FAILED_OBJECT_TTL_SECS=86400`/`_MAX=10000`、`RUSTFS_LOCK_ACQUIRE_TIMEOUT=5s`、`RUSTFS_SCANNER_ENABLED=true`、`RUSTFS_SCANNER_INLINE_HEAL_ENABLE=false`（兼容告警）。 + +全部 17 个 scanner 键支持 env > config 双通道 + admin PUT 热更（generation+Notify 即时生效）；heal 运行时参数目前仅 env（无 admin 热更入口，`Arc>` 结构已预留）。 + +--- + +## 7. 相关 backlog / 历史索引 + +- 换盘自动修复系列（已闭环）：backlog #1786（冗余假绿算法）、#1787（目标槽位限定）、#1789（resume 与 healing marker 绑定 replacement 实例）、#1791（黑白盒验收矩阵）。 +- #801 DiskInfo.healing 从未赋值（已修复闭环，现 `set_disk/mod.rs:4988` 有赋值链）。 +- #1651 Scanner 指标节点/source/bucket-drive 维度（OPEN，本分析 §3.8/§4.6 相关）。 +- #1843 crates/common 83% scanner/heal 域代码分层迁移（OPEN，含 HS-20）。 +- 代码注释引用的历史缺陷（现已有防护与回归测试）：#856/#799 B7（离线盘误记 healed）、#855/B6/#1033（skip 不得标记完成）、#920（sub-quorum 并集枚举）、#856 B5（按版本续扫）、#5173（bitrot trailing bytes）、#5029（回归节点 stale 版本合并）。 +- v1 对标文档：`docs/rustfs-heal-scanner-vs-minio-parity-assessment.md`（本文取代）、落地手册 `docs/rustfs-heal-scanner-vs-minio-improvement-playbook.md`（部分条目已被后续实现超越）。 +- 换盘深度分析：`docs/new-disk-replacement-and-healing-deep-analysis-zh.md`、`docs/node-disk-identity-and-healing-analysis-zh.md`。 + +## 8. 审计方法与局限 + +- 四路并行审计（heal crate 逐文件、scanner crate 逐文件、ecstore 集成层 wiring、MinIO master 源码研究）+ 主会话对关键"缺失"结论逐条亲验（get_disk_status TODO、HealEvent 零外部引用、.bloomcycle.bin 无 bloom 实现、check_abandoned_parts 三层 NotImplemented、ETag 兜底已实现、trace 通道零命中、already_running 语义）。 +- 未逐行核实的点（已在文中标注"未确认/未逐行核"）：DeleteAllVersions 前缀单调用优化（HS-17）、trash 二段清理细节（HS-18）、ilm worker 默认值对照、stale multipart 默认值对照、mc CLI flag 逐字拼写（MinIO 侧）。其中 HS-17 与 HS-18 已于 2026-08-19 完成逐行核实，结论见 §9.2/§9.3。 +- MinIO 侧引用以其 master `7aac2a2c5b` 为准；RustFS 侧行号以 2026-08-16 工作区为准，后续演进请以符号名检索为准。 + +## 9. 落地结果（2026-08-19 更新） + +本审计衍生的 14 个子 issue（backlog #1865~#1878）已全部闭环。本节为差距清单 HS-01~HS-20 的最终处置记录，也是下一轮对标重审的增量基线。 + +### 9.1 已落地（PR 均已合并 main） + +- HS-01 MRF 接线 + 持久化修复账本（#1865，PR #6189）：决策选 (a)。common MRF channel（bounded 8192、try_send 永不阻塞）+ heal mrf_queue（100k 条 / 8MiB 双限环形）+ `buckets/.heal/mrf/journal.bin` CRC 持久化回放（torn tail 截断、回放后删除）+ 三投递点（read decode_error→Urgent ECDecode、scanner 元数据损坏→High Metadata、add_partial→Normal）+ `RUSTFS_HEAL_MRF_ENABLE` 一键回退。 +- HS-02 abandoned parts/data-dir 对账（#1866，PR #6179）：接通 abandoned 检查入口，保留 dry-run / reclaim 计数。 +- HS-03 heal/scanner trace 通道（#1867，PR #6179）：进程内 trace bus + `/v3/trace` admin 流式订阅 + heal task / abandoned-parts / scanner folder / ILM / heal-candidate trace producer。 +- HS-04 scanner 超限 S3 事件（#1868，PR #6176）：`s3:Scanner:ManyVersions/LargeVersions/BigPrefix` 三事件 + 24h 边沿冷却；HS-15 阈值差异文档化（`docs/operations/scanner-excess-alerts.md`）。 +- HS-05 madmin 客户端一期（#1869，PR #6166）：SigV4 admin 客户端 heal/scanner 方法；增量消费方法待 follow-up（协议已由 HS-06 并入）。 +- HS-06 admin heal 增量语义与类型化重叠（#1870，PR #6206）：`sinceSeq/nextSeq/minSeq` 增量游标（wire additive、缺省=全量快照）+ `RUSTFS_HEAL_OVERLAP_POLICY`（默认 merge 不变；minio_error 下 AlreadyRunning/OverlappingPaths 类型化拒绝）+ forceStart 先停旧再启新。 +- HS-07 healing 进度可见性（#1871，PR #6179）：data-usage 总量基线 + baseline/current/healed 计数。 +- HS-08 prefix usage（#1872，PR #6171）：`GET /v3/usage/{bucket}`。 +- HS-11 bitrot 启动自检（#1873，PR #6165）。 +- HS-13 heal 跳过过滤（#1875，PR #6179）：过滤命中版本不再计为失败。 +- HS-16 单机周期钩子（#1878，PR #6250）：删恒 None 钩子，决策记录见 `docs/operations/heal-scanner-parity-notes-zh.md`。 +- HS-09/10/19/20 死代码清理批（#1877，PR #6256）：净 −911 行零行为变更；`get_disk_status` TODO（全仓库唯一产品 TODO）清零；HS-01 联动的 `ec_decode_rebuild`/`get_object_meta` 保留并加 Reserved 注释（MRF 当前经 `heal_object` 执行）。 + +### 9.2 核对后确认"已实现 / 非缺口"（审计期误判修正，累计四例） + +- bloom filter（§0 已修正）：MinIO master 已删除，双方现状一致。 +- ETag 兜底仲裁（§0 已修正）：RustFS 已有实现（`set_disk/ops/heal.rs`）。 +- HS-17（#1876，2026-08-19 逐行核实后关闭）：DeleteAllVersions 前缀单调用优化 RustFS 已完整实现——`apply_expiry_on_non_transitioned_objects` 对 `delete_all()` 两 action 设 `delete_prefix + delete_prefix_object` 后单次 `delete_object`（`bucket_lifecycle_ops.rs:5047-5056`），SetDisks 分支一次写锁 + 一次全版本 quorum 读 + 内联逐版本 object-lock 检查（`set_disk/ops/object.rs:5566-5612`），与 MinIO `expire.go` 的 `applyExpiryOnNonTransitionedObjects` 逐行对齐。§8 原列"未逐行核实"的本项已有结论：现状即优化路径，无需实现。 +- HS-14（#1878，PR #6250 附带核对）：MinIO"idle=空闲才节流"是 2024-01 minio/minio#18734 之前的行为（`scannerIdleMode` 现为静态配置，`idle_speed=on` 默认即始终按速度档节流，"idle"命名是历史残留）；RustFS `RUSTFS_SCANNER_IDLE_MODE` 与 MinIO 当前语义方向一致，且另有 MinIO 没有的前台读退避下限。真实迁移陷阱（变量须 `RUSTFS_` 前缀、`on/off` vs `true/false` 词表、`false` 连前台保护一起关）已文档化于 `docs/operations/heal-scanner-parity-notes-zh.md`。 + +### 9.3 审计型结论（无需改代码） + +- HS-12（#1874，PR #6183）：不存在 MinIO 用 `x-minio-healing` 防御的那类竞争——所有同 (bucket, object) 提交面在同一把对象级 ns 写锁互斥，heal 锁 guard 覆盖 rename 提交全程；交付 2 个并发不变量回归测试 + `docs/operations/heal-concurrency-safety-notes-zh.md` 交点矩阵。 +- HS-18（#1878，2026-08-19 逐行核实）：trash/tmp 三段清理全对齐——stale multipart 隔离-清理等价且更安全（`delete_all_with_quorum` 逐盘递归删即 `move_to_trash` rename 进 `.rustfs.sys/tmp/.trash`，另有锁 + fence）、trash 排空基本等价（无逐条 sleeper 节流，5m 周期天然限频）、tmp 非 trash 24h 回收等价（RustFS 5m 比 MinIO 6h 更及时）；周期默认 24h/6h/5m 三项全对齐。§8 原列"未逐行核实"的本项已有结论。 + +### 9.4 移交 follow-up（汇总于 backlog#1862 评论区） + +HS-01 bitrot GET→MRF 全链路 e2e、kill -9 journal 回放 e2e、队列满压测 RSS（≤ 预算+10%）；HS-05/06 madmin 增量消费方法 + wire 单一来源化 + embedded e2e + 多轮轮询 soak；HS-08 多盘 scanner 周期 e2e；HS-04 超限审计条目；HS-18 低于 quorum 的 stale-multipart 崩溃残留窗口（扇出中途崩溃且已清盘数 > parity 时 FileNotFound 不在忽略集导致不自然收敛，修复需专用 quorum 变体）。 + +下一轮重审建议：跟随 heal/scanner 下一个大特性落地后触发，以本节为增量基线。 From 07cef6789bd3ca08a9b530c3e1db792e0d02ec61 Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 19 Aug 2026 14:26:43 +0800 Subject: [PATCH 17/27] feat(ecstore): expose rename sync tail metrics (#6257) Add default-off PUT stage helpers for fdatasync batch shape and rename quorum fanout shape so #925 follow-up probes can distinguish shard sync batching opportunities from fanout convergence. Co-authored-by: heihutu --- crates/ecstore/src/disk/os.rs | 8 ++ .../src/set_disk/core/io_primitives.rs | 19 +++ crates/io-metrics/src/lib.rs | 110 ++++++++++++++++++ 3 files changed, 137 insertions(+) diff --git a/crates/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs index 94392fa81..acc3a5e5e 100644 --- a/crates/ecstore/src/disk/os.rs +++ b/crates/ecstore/src/disk/os.rs @@ -1098,6 +1098,10 @@ pub(crate) async fn sync_dir_files_with_limiter(dir: impl AsRef, disk_perm let files = run_file_sync_blocking(disk_permits.clone(), move || { let files = regular_files(&scan_dir)?; if files.len() < PARALLEL_FILE_SYNC_THRESHOLD { + rustfs_io_metrics::record_put_rename_fdatasync_batch( + rustfs_io_metrics::PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL, + files.len(), + ); sync_files(&files)?; let fsync_started = rustfs_io_metrics::put_stage_timer(); let result = fsync_dir_std(scan_dir); @@ -1115,6 +1119,10 @@ pub(crate) async fn sync_dir_files_with_limiter(dir: impl AsRef, disk_perm let Some(files) = files else { return Ok(()); }; + rustfs_io_metrics::record_put_rename_fdatasync_batch( + rustfs_io_metrics::PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL, + files.len(), + ); futures::stream::iter(files.into_iter().map(Ok::<_, io::Error>)) .try_for_each_concurrent(MAX_PARALLEL_FILE_SYNCS, |path| { let disk_permits = disk_permits.clone(); diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index dcf5d11e3..784969051 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -3417,6 +3417,25 @@ impl SetDisks { quorum_wait_started, ); let (results, mut file_infos) = fanout_result.map_err(|_| DiskError::Unexpected)?; + if rustfs_io_metrics::put_stage_metrics_enabled() { + let mut fanout_success = 0; + let mut fanout_error = 0; + let mut fanout_panic = 0; + for result in &results { + match result { + Ok(Ok(_)) => fanout_success += 1, + Ok(Err(_)) => fanout_error += 1, + Err(_) => fanout_panic += 1, + } + } + rustfs_io_metrics::record_put_rename_quorum_wait_fanout( + results.len(), + write_quorum, + fanout_success, + fanout_error, + fanout_panic, + ); + } for (idx, result) in results.iter().enumerate() { match result { diff --git a/crates/io-metrics/src/lib.rs b/crates/io-metrics/src/lib.rs index 785cea246..50a660383 100644 --- a/crates/io-metrics/src/lib.rs +++ b/crates/io-metrics/src/lib.rs @@ -120,6 +120,14 @@ pub const PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC: &str = "set_disk_rename_ba pub const PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC: &str = "set_disk_rename_ancestor_dir_fsync"; pub const PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL: &str = "set_disk_rename_rename_syscall"; +pub const PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL: &str = "serial"; +pub const PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL: &str = "parallel"; +pub const PUT_RENAME_QUORUM_FANOUT_STATE_SCHEDULED: &str = "scheduled"; +pub const PUT_RENAME_QUORUM_FANOUT_STATE_WRITE_QUORUM: &str = "write_quorum"; +pub const PUT_RENAME_QUORUM_FANOUT_STATE_SUCCESS: &str = "success"; +pub const PUT_RENAME_QUORUM_FANOUT_STATE_ERROR: &str = "error"; +pub const PUT_RENAME_QUORUM_FANOUT_STATE_PANIC: &str = "panic"; + #[inline(always)] pub fn get_stage_metrics_enabled() -> bool { GET_STAGE_METRICS_ENABLED.load(Ordering::Relaxed) @@ -2042,6 +2050,44 @@ pub fn record_put_object_stage_duration_from(stage: &'static str, started_at: Op } } +#[inline(always)] +fn put_stage_count_value(value: usize) -> f64 { + match u32::try_from(value) { + Ok(value) => f64::from(value), + Err(_) => f64::from(u32::MAX), + } +} + +#[inline(always)] +pub fn record_put_rename_fdatasync_batch(mode: &'static str, files: usize) { + if !put_stage_metrics_enabled() { + return; + } + histogram!("rustfs_s3_put_object_rename_fdatasync_batch_files", "mode" => mode).record(put_stage_count_value(files)); +} + +#[inline(always)] +pub fn record_put_rename_quorum_wait_fanout( + scheduled: usize, + write_quorum: usize, + success: usize, + error: usize, + panicked: usize, +) { + if !put_stage_metrics_enabled() { + return; + } + for (state, count) in [ + (PUT_RENAME_QUORUM_FANOUT_STATE_SCHEDULED, scheduled), + (PUT_RENAME_QUORUM_FANOUT_STATE_WRITE_QUORUM, write_quorum), + (PUT_RENAME_QUORUM_FANOUT_STATE_SUCCESS, success), + (PUT_RENAME_QUORUM_FANOUT_STATE_ERROR, error), + (PUT_RENAME_QUORUM_FANOUT_STATE_PANIC, panicked), + ] { + histogram!("rustfs_s3_put_object_rename_quorum_wait_fanout_disks", "state" => state).record(put_stage_count_value(count)); + } +} + /// Record generic internal operation stage duration (non-PUT paths). /// Use this for metacache walks, listing, lifecycle, and other background /// operations that are NOT part of the PUT object hot path. @@ -3122,6 +3168,70 @@ mod tests { assert!(stages.iter().all(|stage| recorded.contains(*stage))); } + #[test] + fn put_rename_code_level_metrics_are_static_and_gated() { + let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::with_local_recorder(&recorder, || { + set_put_stage_metrics_enabled(false); + record_put_rename_fdatasync_batch(PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL, 2); + record_put_rename_quorum_wait_fanout(4, 3, 3, 1, 0); + + set_put_stage_metrics_enabled(true); + record_put_rename_fdatasync_batch(PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL, 9); + record_put_rename_quorum_wait_fanout(4, 3, 3, 1, 0); + set_put_stage_metrics_enabled(false); + }); + + let rows = snapshotter.snapshot().into_vec(); + assert_eq!(histogram_samples(&rows, "rustfs_s3_put_object_rename_fdatasync_batch_files"), vec![9.0]); + let batch_modes = rows + .iter() + .filter(|(composite, _, _, _)| { + composite.kind() == MetricKind::Histogram + && composite.key().name() == "rustfs_s3_put_object_rename_fdatasync_batch_files" + }) + .flat_map(|(composite, _, _, _)| { + composite + .key() + .labels() + .filter(|label| label.key() == "mode") + .map(|label| label.value().to_string()) + .collect::>() + }) + .collect::>(); + assert_eq!(batch_modes, HashSet::from([PUT_RENAME_FDATASYNC_BATCH_MODE_PARALLEL.to_string()])); + + let quorum_samples = histogram_samples(&rows, "rustfs_s3_put_object_rename_quorum_wait_fanout_disks"); + assert_eq!(quorum_samples, vec![0.0, 1.0, 3.0, 3.0, 4.0]); + let quorum_states = rows + .iter() + .filter(|(composite, _, _, _)| { + composite.kind() == MetricKind::Histogram + && composite.key().name() == "rustfs_s3_put_object_rename_quorum_wait_fanout_disks" + }) + .flat_map(|(composite, _, _, _)| { + composite + .key() + .labels() + .filter(|label| label.key() == "state") + .map(|label| label.value().to_string()) + .collect::>() + }) + .collect::>(); + assert_eq!( + quorum_states, + HashSet::from([ + PUT_RENAME_QUORUM_FANOUT_STATE_SCHEDULED.to_string(), + PUT_RENAME_QUORUM_FANOUT_STATE_WRITE_QUORUM.to_string(), + PUT_RENAME_QUORUM_FANOUT_STATE_SUCCESS.to_string(), + PUT_RENAME_QUORUM_FANOUT_STATE_ERROR.to_string(), + PUT_RENAME_QUORUM_FANOUT_STATE_PANIC.to_string(), + ]) + ); + } + #[test] fn test_put_object_diagnostic_buckets() { assert_eq!(put_object_size_bucket(0), "unknown"); From bce0c05f3c6c84ac2c5e20b24fb7da9e232dc60a Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 19 Aug 2026 14:28:57 +0800 Subject: [PATCH 18/27] chore(rustfs): adjudicate the remaining 36 bare dead_code allows (#6259) --- rustfs/src/admin/handlers/account_info.rs | 1 - rustfs/src/admin/handlers/bucket_meta.rs | 1 - rustfs/src/admin/handlers/replication.rs | 6 --- rustfs/src/admin/handlers/trace.rs | 2 - rustfs/src/admin/mod.rs | 1 - rustfs/src/admin/route_policy.rs | 4 ++ rustfs/src/admin/router.rs | 1 - rustfs/src/app/context/global.rs | 2 - rustfs/src/app/context/handles.rs | 2 - rustfs/src/app/context/interfaces.rs | 2 - rustfs/src/app/object_usecase.rs | 2 +- rustfs/src/capacity/capacity_integration.rs | 1 - rustfs/src/init.rs | 40 ------------------- rustfs/src/license.rs | 1 - rustfs/src/startup_iam.rs | 11 ----- rustfs/src/storage/access.rs | 2 +- rustfs/src/storage/concurrency/manager.rs | 3 +- .../src/storage/concurrency/request_guard.rs | 1 - rustfs/src/storage/ecfs_extend.rs | 21 ++-------- rustfs/src/storage/head_prefix.rs | 9 ----- rustfs/src/storage/options.rs | 20 ---------- rustfs/src/storage/sse.rs | 2 - rustfs/src/storage/tonic_service.rs | 1 - rustfs/src/update.rs | 3 -- 24 files changed, 11 insertions(+), 128 deletions(-) diff --git a/rustfs/src/admin/handlers/account_info.rs b/rustfs/src/admin/handlers/account_info.rs index ca2bda6ea..4a3b45952 100644 --- a/rustfs/src/admin/handlers/account_info.rs +++ b/rustfs/src/admin/handlers/account_info.rs @@ -42,7 +42,6 @@ fn map_data_usage_result(result: Result) -> S3Result) -> std::io:: async fn validate_replication_admin_request(req: &S3Request, action: AdminAction) -> S3Result { authorize_admin_request(req, vec![Action::AdminAction(action)]).await } - -#[allow(dead_code)] -fn is_local_host(_host: String) -> bool { - false -} - pub(crate) async fn cluster_replication_stats(bucket: &str, context: Option>) -> BucketStats { let Some(stats) = current_replication_stats_handle_for_context(context.clone()) else { return BucketStats::default(); diff --git a/rustfs/src/admin/handlers/trace.rs b/rustfs/src/admin/handlers/trace.rs index 17daa67bf..6c22eeec7 100644 --- a/rustfs/src/admin/handlers/trace.rs +++ b/rustfs/src/admin/handlers/trace.rs @@ -21,7 +21,6 @@ use matchit::Params; use rustfs_madmin::service_commands::ServiceTraceOpts; use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; -#[allow(dead_code)] fn extract_trace_options(uri: &Uri) -> S3Result { let mut st_opts = ServiceTraceOpts::default(); st_opts @@ -31,7 +30,6 @@ fn extract_trace_options(uri: &Uri) -> S3Result { Ok(st_opts) } -#[allow(dead_code)] pub struct Trace {} #[async_trait::async_trait] diff --git a/rustfs/src/admin/mod.rs b/rustfs/src/admin/mod.rs index fe07d4056..37c4b34c9 100644 --- a/rustfs/src/admin/mod.rs +++ b/rustfs/src/admin/mod.rs @@ -19,7 +19,6 @@ pub mod handlers; mod plugin_contract; pub(crate) mod replication_metrics_wire; // Contract inventory is validated by tests before later runtime integration. -#[allow(dead_code)] pub(crate) mod route_policy; pub mod router; pub(crate) mod runtime_sources; diff --git a/rustfs/src/admin/route_policy.rs b/rustfs/src/admin/route_policy.rs index 2c1ad5e1b..ccb013211 100644 --- a/rustfs/src/admin/route_policy.rs +++ b/rustfs/src/admin/route_policy.rs @@ -1598,6 +1598,10 @@ pub const DEFERRED_ADMIN_ROUTE_POLICIES: &[DeferredAdminRoutePolicy] = &[ ), ]; +#[allow( + dead_code, + reason = "asserted by this file's tests; the lib target cannot see test-only consumers (backlog#1823)" +)] pub fn validate_admin_route_policy_specs() -> Result<(), AdminRouteMatrixError> { validate_admin_route_specs(ADMIN_ROUTE_POLICY_SPECS) } diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index 14ad1fb5e..498069eb4 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -5800,7 +5800,6 @@ mod tests { } } -#[allow(dead_code)] #[derive(Debug, Clone)] pub struct Extra { pub credentials: Option, diff --git a/rustfs/src/app/context/global.rs b/rustfs/src/app/context/global.rs index 50c79498a..b26151223 100644 --- a/rustfs/src/app/context/global.rs +++ b/rustfs/src/app/context/global.rs @@ -45,7 +45,6 @@ pub struct AppContext { object_store: Arc, iam: Arc, federated_identity: Arc, - #[allow(dead_code)] kms: Arc, kms_runtime: Arc, outbound_tls_runtime: Arc, @@ -162,7 +161,6 @@ impl AppContext { self.federated_identity.publish_handle(service) } - #[allow(dead_code)] pub fn kms(&self) -> Arc { self.kms.clone() } diff --git a/rustfs/src/app/context/handles.rs b/rustfs/src/app/context/handles.rs index 9e376e1a5..8a2bdc29c 100644 --- a/rustfs/src/app/context/handles.rs +++ b/rustfs/src/app/context/handles.rs @@ -49,7 +49,6 @@ use tokio::sync::RwLock; /// Default IAM interface adapter. pub struct IamHandle { - #[allow(dead_code)] iam: Arc>, } @@ -110,7 +109,6 @@ impl FederatedIdentityInterface for FederatedIdentityHandle { } /// Default KMS interface adapter. -#[allow(dead_code)] pub struct KmsHandle { kms: Arc, } diff --git a/rustfs/src/app/context/interfaces.rs b/rustfs/src/app/context/interfaces.rs index 17e029b76..b36a560dd 100644 --- a/rustfs/src/app/context/interfaces.rs +++ b/rustfs/src/app/context/interfaces.rs @@ -36,7 +36,6 @@ use tokio::sync::RwLock; /// IAM interface for application-layer use-cases. pub trait IamInterface: Send + Sync { - #[allow(dead_code)] fn handle(&self) -> Arc>; fn is_ready(&self) -> bool; fn token_signing_key(&self) -> Option { @@ -53,7 +52,6 @@ pub trait FederatedIdentityInterface: Send + Sync { } /// KMS interface for application-layer use-cases. -#[allow(dead_code)] pub trait KmsInterface: Send + Sync { fn handle(&self) -> Arc; } diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 55d863588..e541a5b3e 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -738,7 +738,7 @@ struct GetObjectPreparedRead { } struct GetObjectStrategyContext { - #[allow(dead_code)] + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] io_strategy: concurrency::IoStrategy, optimal_buffer_size: usize, enable_readahead: bool, diff --git a/rustfs/src/capacity/capacity_integration.rs b/rustfs/src/capacity/capacity_integration.rs index 9389049bf..927838f3d 100644 --- a/rustfs/src/capacity/capacity_integration.rs +++ b/rustfs/src/capacity/capacity_integration.rs @@ -31,7 +31,6 @@ pub async fn init_capacity_management_managed() -> Option Option<(u64, String)> { get_cached_capacity_with_metrics() .await diff --git a/rustfs/src/init.rs b/rustfs/src/init.rs index 34a1d6ca0..e76dca19b 100644 --- a/rustfs/src/init.rs +++ b/rustfs/src/init.rs @@ -765,7 +765,6 @@ fn resolve_buffer_profile_config( /// Parse and normalize server address for FTP/FTPS /// Forces IPv4 binding to avoid libunftp IPv6 compatibility issues -#[allow(dead_code)] async fn parse_and_normalize_server_address( address_str: &str, ) -> Result> { @@ -781,45 +780,6 @@ async fn parse_and_normalize_server_address( Ok(normalized_addr) } - -/// Start FTP/FTPS server in background with shutdown support -/// # Arguments -/// * `server` - The FTP/FTPS server instance -/// * `protocol_name` - Name of the protocol (e.g., "FTP", "FTPS") -#[allow(dead_code)] -fn spawn_server(server: S, protocol_name: &'static str) -> tokio::sync::broadcast::Sender<()> -where - S: std::future::Future>> + Send + 'static, -{ - let (shutdown_tx, _) = tokio::sync::broadcast::channel(1); - - tokio::spawn(async move { - if let Err(e) = server.await { - error!( - target: "rustfs::init", - event = "protocol_server_state", - component = LOG_COMPONENT_INIT, - subsystem = LOG_SUBSYSTEM_PROTOCOL, - protocol = protocol_name, - state = "runtime_failed", - error = %e, - "Protocol server failed" - ); - } - info!( - target: "rustfs::init", - event = "protocol_server_state", - component = LOG_COMPONENT_INIT, - subsystem = LOG_SUBSYSTEM_PROTOCOL, - protocol = protocol_name, - state = "stopped", - "Protocol server stopped" - ); - }); - - shutdown_tx -} - /// Starts the auto-tuner for performance optimization if enabled via environment variable. /// /// The auto-tuner reads `RUSTFS_AUTOTUNER_ENABLED` to decide whether to run. diff --git a/rustfs/src/license.rs b/rustfs/src/license.rs index 5163e58bf..75638eaeb 100644 --- a/rustfs/src/license.rs +++ b/rustfs/src/license.rs @@ -211,7 +211,6 @@ fn apply_valid_status(state: &mut LicenseState, token: Token) { /// /// This is the extension point for OEM/build-time overlays. /// Returns `false` if the verifier was already initialized. -#[allow(dead_code)] pub fn set_license_verifier(verifier: SharedLicenseVerifier) -> bool { LICENSE_VERIFIER.set(verifier).is_ok() } diff --git a/rustfs/src/startup_iam.rs b/rustfs/src/startup_iam.rs index 6220e1c12..fe28108b8 100644 --- a/rustfs/src/startup_iam.rs +++ b/rustfs/src/startup_iam.rs @@ -352,17 +352,6 @@ fn should_fail_test_init_attempt() -> bool { false } } - -/// Reset the test failure counter so the next `should_fail_test_init_attempt` -/// call re-reads the environment variable by restoring the sentinel value. -/// Intended for use in integration tests that share a process. -#[doc(hidden)] -#[allow(dead_code)] -pub(crate) fn reset_test_failure_counter() { - use std::sync::atomic::Ordering; - TEST_REMAINING_FAILURES.store(u64::MAX, Ordering::SeqCst); -} - async fn attempt_init_iam_sys( store: Arc, ) -> std::result::Result>, std::io::Error> { diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index d69bde85f..690c2abdd 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -61,7 +61,7 @@ pub(crate) struct ReqInfo { pub object: Option, pub version_id: Option, pub replication_request_authorized: bool, - #[allow(dead_code)] + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] pub region: Option, pub request_context: Option, /// Set by probe-style callers that treat AccessDenied as an expected filter diff --git a/rustfs/src/storage/concurrency/manager.rs b/rustfs/src/storage/concurrency/manager.rs index 5bc456148..bf51b55bf 100644 --- a/rustfs/src/storage/concurrency/manager.rs +++ b/rustfs/src/storage/concurrency/manager.rs @@ -50,7 +50,7 @@ pub struct ConcurrencyManager { /// I/O load metrics for adaptive strategy calculation io_metrics: Arc>, /// I/O priority queue for request scheduling - #[allow(dead_code)] + #[allow(dead_code, reason = "written but never read back (backlog#1823)")] priority_queue: Arc>, /// Bytes pool for buffer allocation and reuse bytes_pool: Arc, @@ -131,7 +131,6 @@ pub enum PutObjectAdmission { Rejected, } -#[allow(dead_code)] impl ConcurrencyManager { /// Create a new concurrency manager with default settings /// diff --git a/rustfs/src/storage/concurrency/request_guard.rs b/rustfs/src/storage/concurrency/request_guard.rs index d435ef2bd..f1847a5c0 100644 --- a/rustfs/src/storage/concurrency/request_guard.rs +++ b/rustfs/src/storage/concurrency/request_guard.rs @@ -64,7 +64,6 @@ impl GetObjectGuard { } /// Get the elapsed time since this guard was created. - #[allow(dead_code)] // This helper is primarily used by unit tests to assert timing. // It's intentionally kept public for callers that may want to inspect // a guard's duration without dropping it. diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index 5d6fba452..9ecd207a9 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -254,7 +254,10 @@ pub(crate) fn apply_bucket_default_lock_retention( /// ); /// ``` /// -#[allow(dead_code)] +#[allow( + dead_code, + reason = "exercised by ecfs_test; the lib target cannot see test-only consumers (backlog#1823)" +)] pub(crate) fn get_adaptive_buffer_size_with_profile(file_size: i64, profile: Option) -> usize { let config = match profile { Some(p) => RustFSBufferConfig::new(p), @@ -798,26 +801,10 @@ fn cache_remove(bucket: &str) { map.remove(bucket); } } - -/// Clear all entries in the cache. -#[allow(dead_code)] -fn cache_clear() { - if let Ok(mut map) = small_cache().write() { - map.clear(); - } -} - /// Invalidate the validation cache for a specific bucket. pub fn invalidate_bucket_validation_cache(bucket: &str) { cache_remove(bucket); } - -/// Invalidate all bucket validation cache entries. -#[allow(dead_code)] -pub fn invalidate_all_bucket_validation_cache() { - cache_clear(); -} - /// Helper function to get store and validate bucket exists. /// /// Uses adaptive cache with 5s TTL to avoid repeated stat_volume() calls. diff --git a/rustfs/src/storage/head_prefix.rs b/rustfs/src/storage/head_prefix.rs index 49f53f452..89ae57a1f 100644 --- a/rustfs/src/storage/head_prefix.rs +++ b/rustfs/src/storage/head_prefix.rs @@ -15,15 +15,6 @@ use super::ECStore; use crate::storage::storage_api::head_prefix_consumer::contract::list::ListOperations as _; use std::sync::Arc; - -/// Determines if the key "looks like a prefix" (ends with `/`). -/// Note: No special handling for empty strings here; the caller must ensure the key has passed `validate_object_key`. -#[allow(dead_code)] -#[inline] -pub(crate) fn is_prefix_key(key: &str) -> bool { - key.ends_with('/') -} - /// Constructs a more explicit error message when `HEAD` is performed on a `prefix`-style key but the directory marker object is missing. /// /// `has_children`: diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index dc5a6ca19..701dbd38b 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -1018,12 +1018,6 @@ pub fn parse_copy_source_range(range_str: &str) -> S3Result { Err(s3_error!(InvalidArgument, "Invalid range format")) } } - -#[allow(dead_code)] -pub(crate) fn get_content_sha256(headers: &HeaderMap) -> Option { - get_content_sha256_with_query(headers, None) -} - pub(crate) fn get_content_sha256_with_query(headers: &HeaderMap, query: Option<&str>) -> Option { match get_request_auth_type_with_query(headers, query) { AuthType::Presigned | AuthType::Signed => { @@ -1036,14 +1030,6 @@ pub(crate) fn get_content_sha256_with_query(headers: &HeaderMap, qu _ => None, } } - -/// skip_content_sha256_cksum returns true if caller needs to skip -/// payload checksum, false if not. -#[allow(dead_code)] -fn skip_content_sha256_cksum(headers: &HeaderMap) -> bool { - skip_content_sha256_cksum_with_query(headers, None) -} - fn skip_content_sha256_cksum_with_query(headers: &HeaderMap, query: Option<&str>) -> bool { let include_query_values = matches!(get_request_auth_type_with_query(headers, query), AuthType::Presigned); let content_sha256 = get_content_sha256_value(headers, query, include_query_values); @@ -1138,12 +1124,6 @@ fn get_content_sha256_value( .and_then(|v| v.to_str().ok()) .map(str::to_owned) } - -#[allow(dead_code)] -fn get_content_sha256_cksum(headers: &HeaderMap, service_type: ServiceType) -> String { - get_content_sha256_cksum_with_query(headers, None, service_type) -} - #[cfg(test)] #[allow(unused_imports)] mod tests { diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index eda06a7dd..33b03f8c7 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -3354,7 +3354,6 @@ async fn get_local_sse_dek_provider() -> Result, ApiErro /// Clears GLOBAL_SSE_DEK_PROVIDER (local/test providers) and /// GLOBAL_KMS_DEK_PROVIDER (test-injected KMS providers). #[cfg(test)] -#[allow(dead_code)] pub fn reset_sse_dek_provider() { if let Ok(mut slot) = GLOBAL_SSE_DEK_PROVIDER.write() { *slot = None; @@ -3365,7 +3364,6 @@ pub fn reset_sse_dek_provider() { } #[cfg(test)] -#[allow(dead_code)] pub fn set_sse_dek_provider_for_test(provider: Arc) { if let Ok(mut slot) = GLOBAL_KMS_DEK_PROVIDER.write() { *slot = Some(provider.clone()); diff --git a/rustfs/src/storage/tonic_service.rs b/rustfs/src/storage/tonic_service.rs index e56a8f235..539361507 100644 --- a/rustfs/src/storage/tonic_service.rs +++ b/rustfs/src/storage/tonic_service.rs @@ -16,5 +16,4 @@ pub(crate) use crate::storage::rpc::node_service::make_heal_control_server_with_ #[cfg(test)] pub(crate) use crate::storage::rpc::node_service::{heal::heal_topology_fingerprint, make_heal_control_server_for_source}; pub use crate::storage::rpc::{make_heal_control_server, make_server, make_tier_mutation_control_server}; -#[allow(dead_code)] pub type NodeService = crate::storage::rpc::NodeService; diff --git a/rustfs/src/update.rs b/rustfs/src/update.rs index 4ff089408..3b7a267cb 100644 --- a/rustfs/src/update.rs +++ b/rustfs/src/update.rs @@ -45,7 +45,6 @@ pub struct VersionInfo { } /// Update check result -#[allow(dead_code)] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct UpdateCheckResult { /// Whether update is available @@ -91,7 +90,6 @@ impl VersionChecker { } /// Create version checker with custom configuration - #[allow(dead_code)] pub fn with_config(url: String, timeout: Duration) -> Self { let client = reqwest::Client::builder() .timeout(timeout) @@ -175,7 +173,6 @@ pub async fn check_updates() -> Result { } /// Update check with custom URL -#[allow(dead_code)] pub async fn check_updates_with_url(url: String) -> Result { let checker = VersionChecker::with_config(url, Duration::from_secs(10)); checker.check_for_updates().await From 5cb12300bc96fc5eca3d5a1e10f05ca2c7432033 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 19 Aug 2026 14:30:04 +0800 Subject: [PATCH 19/27] refactor(sse): keep one bucket-default algorithm mapping for every writer (#6251) --- rustfs/src/app/object_usecase.rs | 29 +++++------------------------ rustfs/src/app/storage_api.rs | 5 +++-- rustfs/src/storage/sse.rs | 26 ++++++++++++++++++++------ rustfs/src/storage/storage_api.rs | 6 +++--- 4 files changed, 31 insertions(+), 35 deletions(-) diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index e541a5b3e..2c7405c62 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -97,7 +97,7 @@ use super::storage_api::object_usecase::set_disk::{ }; use super::storage_api::object_usecase::sse::{ DecryptionRequest, EncryptionRequest, SSEType, SseKmsPrincipal, apply_bucket_default_lock_retention, - authorize_sse_kms_object_read, build_ssec_read_headers, encryption_material_to_metadata, + authorize_sse_kms_object_read, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata, extract_server_side_encryption_from_headers, extract_ssec_params_from_headers, extract_ssekms_context_from_headers, get_buffer_size_opt_in, load_bucket_object_lock_config_state, map_get_object_reader_error, sse_decryption, sse_encryption, validate_bucket_object_lock_enabled_state, @@ -169,8 +169,8 @@ use s3s::dto::{ ObjectLockLegalHoldStatus, ObjectLockMode, ObjectLockRetention, ObjectLockRetentionMode, ObjectPart, PutObjectInput, PutObjectOutput, Range, RequestCharged, RestoreObjectInput, RestoreObjectOutput, RestoreStatus, SSECustomerAlgorithm, SSECustomerKeyMD5, SSEKMSKeyId, SelectObjectContentInput, SelectObjectContentOutput, ServerSideEncryption, - ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, - TaggingHeader, Timestamp, TimestampFormat, WebsiteRedirectLocation, + ServerSideEncryptionConfiguration, StorageClass, StreamingBlob, TaggingDirective, TaggingHeader, Timestamp, TimestampFormat, + WebsiteRedirectLocation, }; use s3s::header::{X_AMZ_RESTORE, X_AMZ_RESTORE_OUTPUT_PATH}; use s3s::stream::{ByteStream, DynByteStream, RemainingLength}; @@ -2676,25 +2676,6 @@ fn has_put_sse_request_headers(headers: &HeaderMap) -> bool { || headers.get(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID).is_some() } -/// Managed SSE resolved from a bucket default encryption rule on the copy path. -/// -/// Unknown algorithms fall back to AES256, the same total mapping as the PUT and -/// extract paths and the storage-layer resolver (`prepare_sse_configuration`), which -/// `sse_encryption` re-runs when it mints the destination DEK. Resolving `None` here -/// instead lets a same-name copy under a malformed bucket default pass the -/// `copy_changes_encryption` guard and take the metadata-only shortcut while the -/// storage layer still encrypts: fresh DEK metadata is committed beside the untouched -/// plaintext blocks and the object becomes unreadable. Reachable only via corrupt or -/// hand-edited bucket metadata — PutBucketEncryption rejects unknown algorithms -/// (backlog#1826). -fn bucket_default_write_sse(sse: &ServerSideEncryptionByDefault) -> ServerSideEncryption { - match sse.sse_algorithm.as_str() { - "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), - _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - } -} - /// Resolve the effective server-side encryption for a write against the bucket's /// default encryption configuration. /// @@ -10115,8 +10096,8 @@ mod tests { DefaultRetention, Delete, DeleteMarkerReplication, DeleteMarkerReplicationStatus, DeleteReplication, DeleteReplicationStatus, Destination, ExistingObjectReplication, ExistingObjectReplicationStatus, ObjectIdentifier, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockRule, ReplicaModifications, ReplicaModificationsStatus, - ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, RestoreRequest, ServerSideEncryptionConfiguration, - ServerSideEncryptionRule, SourceSelectionCriteria, + ReplicationConfiguration, ReplicationRule, ReplicationRuleStatus, RestoreRequest, ServerSideEncryptionByDefault, + ServerSideEncryptionConfiguration, ServerSideEncryptionRule, SourceSelectionCriteria, }; use std::pin::Pin; use std::sync::Arc; diff --git a/rustfs/src/app/storage_api.rs b/rustfs/src/app/storage_api.rs index 0c2b3d4d4..d90cfb6a8 100644 --- a/rustfs/src/app/storage_api.rs +++ b/rustfs/src/app/storage_api.rs @@ -1032,8 +1032,9 @@ pub(crate) mod sse { validate_bucket_object_lock_enabled_state, }; pub(crate) use crate::storage::storage_api::sse_consumer::{ - EncryptionKeyKind, SSEType, build_ssec_read_headers, encryption_material_to_metadata, extract_ssec_params_from_headers, - extract_ssekms_context_from_headers, map_get_object_reader_error, mark_encrypted_multipart_metadata, + EncryptionKeyKind, SSEType, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata, + extract_ssec_params_from_headers, extract_ssekms_context_from_headers, map_get_object_reader_error, + mark_encrypted_multipart_metadata, }; } diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index 33b03f8c7..b3e4f839f 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -164,7 +164,7 @@ use rustfs_utils::http::headers::{ AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, AMZ_SERVER_SIDE_ENCRYPTION_KMS_CONTEXT, }; use rustfs_utils::path::path_join_buf; -use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId}; +use s3s::dto::{SSECustomerAlgorithm, SSECustomerKey, SSECustomerKeyMD5, SSEKMSKeyId, ServerSideEncryptionByDefault}; use std::borrow::Cow; // ============================================================================ @@ -203,6 +203,24 @@ pub struct SseConfiguration { /// Effective KMS key ID (after considering bucket defaults) pub effective_kms_key_id: Option, } +/// Managed SSE resolved from a bucket default encryption rule on a write path. +/// +/// The single mapping shared by every writer: this resolver, and the PUT, COPY +/// and extract paths in `app::object_usecase`, which reach it through +/// `resolve_bucket_default_sse`. Unknown algorithms fall back to AES256 rather +/// than to `None`. Resolving `None` instead lets a same-name copy under a +/// malformed bucket default pass the `copy_changes_encryption` guard and take +/// the metadata-only shortcut while this layer still encrypts: fresh DEK +/// metadata is committed beside the untouched plaintext blocks and the object +/// becomes unreadable. Reachable only via corrupt or hand-edited bucket +/// metadata — PutBucketEncryption rejects unknown algorithms (backlog#1826). +pub(crate) fn bucket_default_write_sse(sse: &ServerSideEncryptionByDefault) -> ServerSideEncryption { + match sse.sse_algorithm.as_str() { + "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), + "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), + _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), + } +} /// Prepare SSE configuration by resolving request parameters with bucket defaults /// @@ -266,11 +284,7 @@ async fn prepare_sse_configuration( has_kms_key_id = sse.kms_master_key_id.is_some(), "Bucket SSE default resolved" ); - match sse.sse_algorithm.as_str() { - "AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256), - "aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS), - _ => ServerSideEncryption::from_static(ServerSideEncryption::AES256), // fallback - } + bucket_default_write_sse(sse) }) }) }); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index ef9a92414..31668783a 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -344,9 +344,9 @@ pub(crate) mod s3_api_consumer { pub(crate) mod sse_consumer { pub(crate) use super::super::sse::{ - EncryptionKeyKind, SSEType, build_ssec_read_headers, encryption_material_to_metadata, extract_ssec_params_from_headers, - extract_ssekms_context_from_headers, log_sse_kms_key_policy_mode, map_get_object_reader_error, - mark_encrypted_multipart_metadata, + EncryptionKeyKind, SSEType, bucket_default_write_sse, build_ssec_read_headers, encryption_material_to_metadata, + extract_ssec_params_from_headers, extract_ssekms_context_from_headers, log_sse_kms_key_policy_mode, + map_get_object_reader_error, mark_encrypted_multipart_metadata, }; pub(crate) use super::{ DecryptionRequest, EncryptionRequest, PrepareEncryptionRequest, SseKmsPrincipal, apply_bucket_default_lock_retention, From 395878132029ec5842aa13dc73eb972623294b6f Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 19 Aug 2026 14:47:17 +0800 Subject: [PATCH 20/27] feat(io-metrics): attribute ReadVersion RPC stages (#6262) --- crates/ecstore/src/cluster/rpc/remote_disk.rs | 87 ++++++++- .../src/cluster/rpc/runtime_sources.rs | 60 +++++- crates/io-metrics/src/internode_metrics.rs | 147 +++++++++++--- rustfs/src/storage/rpc/node_service/disk.rs | 183 +++++++++++++++--- 4 files changed, 414 insertions(+), 63 deletions(-) diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index 26e9f899a..ed8772f53 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -41,6 +41,10 @@ use bytes::Bytes; use futures::lock::Mutex; use metrics::counter; use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo}; +use rustfs_io_metrics::internode_metrics::{ + INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, + INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, +}; use rustfs_protos::ChannelClass; use rustfs_protos::evict_failed_connection; use rustfs_protos::proto_gen::node_service::RenamePartRequest; @@ -64,7 +68,7 @@ use std::{ atomic::{AtomicBool, AtomicU32, Ordering}, }, task::{Context, Poll}, - time::Duration, + time::{Duration, Instant}, }; use tokio::time; use tokio::{ @@ -1790,6 +1794,16 @@ fn decode_msgpack_or_json(binary: &[u8], json: &str, value_ } } +fn read_version_stage_timer(attribution_enabled: bool) -> Option { + attribution_enabled.then(Instant::now) +} + +fn record_read_version_stage(stage: &'static str, started_at: Option) { + if let Some(started_at) = started_at { + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_stage(stage, started_at.elapsed()); + } +} + /// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads /// and falling back to the JSON compatibility strings. Used to size the RPC for the payload /// histogram / large-payload alerting (grpc-optimization P0 instrumentation). @@ -2705,8 +2719,11 @@ impl DiskAPI for RemoteDisk { state = "started", "Remote disk RPC started" ); - let opts_str = compat_json(opts)?; - let opts_bin = encode_msgpack(opts)?; + let read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + let encode_started = read_version_stage_timer(read_version_attribution_enabled); + let encoded_opts = compat_json(opts).and_then(|opts_str| encode_msgpack(opts).map(|opts_bin| (opts_str, opts_bin))); + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, encode_started); + let (opts_str, opts_bin) = encoded_opts?; // Idempotent version read: eligible for the bounded transient-network retry so a single // reset-by-peer during the read-after-write window does not erode the metadata read @@ -2722,6 +2739,14 @@ impl DiskAPI for RemoteDisk { .get_client() .await .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; + let request_payload_bytes = read_version_attribution_enabled.then(|| { + disk.len() + .saturating_add(volume.len()) + .saturating_add(path.len()) + .saturating_add(version_id.len()) + .saturating_add(opts_str.len()) + .saturating_add(opts_bin.len()) + }); let request = Request::new(ReadVersionRequest { disk, volume: volume.to_string(), @@ -2731,14 +2756,47 @@ impl DiskAPI for RemoteDisk { opts_bin: opts_bin.into(), }); - let response = client.read_version(request).await?.into_inner(); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_request(); + if let Some(request_payload_bytes) = request_payload_bytes { + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_sent_bytes(request_payload_bytes); + } + let rpc_started = read_version_stage_timer(read_version_attribution_enabled); + let response = match client.read_version(request).await { + Ok(response) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, rpc_started); + response.into_inner() + } + Err(err) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, rpc_started); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error(); + return Err(err.into()); + } + }; if !response.success { + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error(); return Err(response.error.unwrap_or_default().into()); } - let file_info = decode_msgpack_or_json::(&response.file_info_bin, &response.file_info, "FileInfo")?; - validate_decoded_file_info(&file_info)?; + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_recv_bytes( + response.file_info.len().saturating_add(response.file_info_bin.len()), + ); + let decode_started = read_version_stage_timer(read_version_attribution_enabled); + let file_info = match decode_msgpack_or_json::(&response.file_info_bin, &response.file_info, "FileInfo") + .and_then(|file_info| { + validate_decoded_file_info(&file_info)?; + Ok(file_info) + }) { + Ok(file_info) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, decode_started); + file_info + } + Err(err) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, decode_started); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error(); + return Err(err); + } + }; Ok(file_info) }, @@ -7931,12 +7989,17 @@ mod tests { } #[tokio::test] + #[serial] async fn read_version_uses_the_metadata_timeout_on_a_stalled_peer() { runtime_sources::ensure_test_rpc_secret(); let Some((base_addr, accept_task)) = spawn_stalled_grpc_peer().await else { return; }; let remote_disk = remote_disk_for_addr(&base_addr).await; + let metrics = rustfs_io_metrics::internode_metrics::global_internode_metrics(); + let previous_stage_metrics = rustfs_io_metrics::get_stage_metrics_enabled(); + metrics.reset_for_test(); + rustfs_io_metrics::set_get_stage_metrics_enabled(true); temp_env::async_with_vars( [ @@ -7960,6 +8023,18 @@ mod tests { ) .await; + rustfs_io_metrics::set_get_stage_metrics_enabled(previous_stage_metrics); + let snapshot = metrics.snapshot(); + assert!( + snapshot.outgoing_requests_total >= 1, + "ReadVersion call site should record outgoing attempts when attribution is enabled" + ); + assert!( + snapshot.sent_bytes_total > 0, + "ReadVersion call site should record request payload bytes when attribution is enabled" + ); + metrics.reset_for_test(); + remote_disk.cancel_token.cancel(); accept_task.abort(); } diff --git a/crates/ecstore/src/cluster/rpc/runtime_sources.rs b/crates/ecstore/src/cluster/rpc/runtime_sources.rs index 03d19f60f..0c8393a2e 100644 --- a/crates/ecstore/src/cluster/rpc/runtime_sources.rs +++ b/crates/ecstore/src/cluster/rpc/runtime_sources.rs @@ -14,10 +14,11 @@ use rustfs_io_metrics::internode_metrics::{ INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE, - INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_WRITE_ALL, - INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC, - INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics, + INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, + INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics, }; +use std::time::Duration; #[cfg(test)] use rustfs_io_metrics::internode_metrics::InternodeMetricsSnapshot; @@ -82,6 +83,59 @@ pub(crate) fn record_remote_disk_grpc_read_all_request() { .record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC); } +pub(crate) fn record_remote_disk_grpc_read_version_request() { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_outgoing_request_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); +} + +pub(crate) fn record_remote_disk_grpc_read_version_error() { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics() + .record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC); +} + +pub(crate) fn record_remote_disk_grpc_read_version_sent_bytes(bytes: usize) { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_sent_bytes_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + bytes, + ); +} + +pub(crate) fn record_remote_disk_grpc_read_version_recv_bytes(bytes: usize) { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_recv_bytes_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + bytes, + ); + record_grpc_payload_size(INTERNODE_OPERATION_GRPC_READ_VERSION, bytes); +} + +pub(crate) fn record_remote_disk_grpc_read_version_stage(stage: &'static str, duration: Duration) { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_stage_duration_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + stage, + duration, + ); +} + pub(crate) fn record_remote_disk_grpc_read_all_recv_bytes(bytes: usize) { global_internode_metrics().record_recv_bytes_for_operation_and_backend( INTERNODE_OPERATION_GRPC_READ_ALL, diff --git a/crates/io-metrics/src/internode_metrics.rs b/crates/io-metrics/src/internode_metrics.rs index a12a8edfe..fb12a9911 100644 --- a/crates/io-metrics/src/internode_metrics.rs +++ b/crates/io-metrics/src/internode_metrics.rs @@ -47,6 +47,13 @@ pub const INTERNODE_MSGPACK_DIRECTION_REQUEST: &str = "request"; pub const INTERNODE_MSGPACK_DIRECTION_RESPONSE: &str = "response"; pub const INTERNODE_MSGPACK_CODEC_MSGPACK: &str = "msgpack"; pub const INTERNODE_MSGPACK_CODEC_JSON: &str = "json"; +pub const INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE: &str = "read_version_request_encode"; +pub const INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE: &str = "read_version_request_decode"; +pub const INTERNODE_STAGE_READ_VERSION_DISK_READ: &str = "read_version_disk_read"; +pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "read_version_response_json_encode"; +pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "read_version_response_msgpack_encode"; +pub const INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP: &str = "read_version_rpc_roundtrip"; +pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE: &str = "read_version_response_decode"; const OPERATION_LABEL: &str = "operation"; const BACKEND_LABEL: &str = "backend"; @@ -67,6 +74,7 @@ const INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network const INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_incoming_total"; const INTERNODE_OPERATION_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_errors_total"; const INTERNODE_OPERATION_DURATION_MS: &str = "rustfs_system_network_internode_operation_duration_ms"; +const INTERNODE_OPERATION_STAGE_DURATION_MS: &str = "rustfs_system_network_internode_operation_stage_duration_ms"; const INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_classified_errors_total"; const INTERNODE_OPERATION_RETRIES_TOTAL: &str = "rustfs_system_network_internode_operation_retries_total"; const INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL: &str = "rustfs_system_network_internode_operation_retry_successes_total"; @@ -105,6 +113,7 @@ const SERVER_OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[SERVER_LABEL, OP const SERVER_OPERATION_BACKEND_FAILURE_REASON_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, FAILURE_REASON_LABEL]; const SERVER_OPERATION_BACKEND_RPC_PATH_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL]; +const SERVER_OPERATION_BACKEND_STAGE_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, STAGE_LABEL]; const SERVER_LABELS: &[&str] = &[SERVER_LABEL]; const SERVER_REASON_LABELS: &[&str] = &[SERVER_LABEL, REASON_LABEL]; const SERVER_QUORUM_FAILURE_LABELS: &[&str] = &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]; @@ -134,6 +143,10 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = & name: INTERNODE_OPERATION_DURATION_MS, labels: SERVER_OPERATION_BACKEND_LABELS, }, + InternodeOperationMetricDescriptor { + name: INTERNODE_OPERATION_STAGE_DURATION_MS, + labels: SERVER_OPERATION_BACKEND_STAGE_LABELS, + }, InternodeOperationMetricDescriptor { name: INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL, labels: SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS, @@ -394,6 +407,24 @@ impl InternodeMetrics { .record(duration_ms); } + pub fn record_stage_duration_for_operation_and_backend( + &self, + operation: &'static str, + backend: &'static str, + stage: &'static str, + duration: Duration, + ) { + let duration_ms = duration.as_secs_f64() * 1000.0; + metrics::histogram!( + INTERNODE_OPERATION_STAGE_DURATION_MS, + SERVER_LABEL => current_server_label(), + OPERATION_LABEL => operation, + BACKEND_LABEL => backend, + STAGE_LABEL => stage + ) + .record(duration_ms); + } + pub fn record_classified_error_for_operation_and_backend( &self, operation: &'static str, @@ -988,42 +1019,90 @@ mod tests { assert_eq!(snapshot.replay_cache_evictions_total, 3); } + #[test] + fn operation_stage_duration_records_low_cardinality_stage_labels() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let metrics = InternodeMetrics::default(); + + with_local_recorder(&recorder, || { + metrics.record_stage_duration_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, + Duration::from_micros(125), + ); + }); + + let entries: Vec<_> = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter(|(composite, _, _, _)| composite.key().name() == INTERNODE_OPERATION_STAGE_DURATION_MS) + .collect(); + assert_eq!(entries.len(), 1); + let labels: HashMap<_, _> = entries[0] + .0 + .key() + .labels() + .map(|label| (label.key().to_string(), label.value().to_string())) + .collect(); + assert_eq!( + labels.get(OPERATION_LABEL).map(String::as_str), + Some(INTERNODE_OPERATION_GRPC_READ_VERSION) + ); + assert_eq!(labels.get(BACKEND_LABEL).map(String::as_str), Some(INTERNODE_TRANSPORT_BACKEND_GRPC)); + assert_eq!( + labels.get(STAGE_LABEL).map(String::as_str), + Some(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP) + ); + assert!(labels.get(SERVER_LABEL).is_some_and(|value| !value.is_empty())); + match &entries[0].3 { + DebugValue::Histogram(samples) => assert_eq!(samples.iter().map(|sample| sample.0).collect::>(), vec![0.125]), + other => panic!("{INTERNODE_OPERATION_STAGE_DURATION_MS} must be a histogram, got {other:?}"), + } + } + #[test] fn operation_metric_descriptors_include_backend_and_operation_labels() { - assert_eq!(INTERNODE_OPERATION_METRICS.len(), 21); + assert_eq!(INTERNODE_OPERATION_METRICS.len(), 22); for metric in &INTERNODE_OPERATION_METRICS[..6] { assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]); } - for metric in &INTERNODE_OPERATION_METRICS[6..9] { + assert_eq!( + INTERNODE_OPERATION_METRICS[6].labels, + &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, STAGE_LABEL] + ); + for metric in &INTERNODE_OPERATION_METRICS[7..10] { assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL]); } assert_eq!( - INTERNODE_OPERATION_METRICS[9].labels, + INTERNODE_OPERATION_METRICS[10].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL] ); - for metric in &INTERNODE_OPERATION_METRICS[10..12] { + for metric in &INTERNODE_OPERATION_METRICS[11..13] { assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]); } - assert_eq!( - INTERNODE_OPERATION_METRICS[12].labels, - &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, FAILURE_REASON_LABEL] - ); assert_eq!( INTERNODE_OPERATION_METRICS[13].labels, - &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL] + &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, FAILURE_REASON_LABEL] ); assert_eq!( INTERNODE_OPERATION_METRICS[14].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL] ); - for metric in &INTERNODE_OPERATION_METRICS[15..17] { + assert_eq!( + INTERNODE_OPERATION_METRICS[15].labels, + &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, RPC_PATH_LABEL] + ); + for metric in &INTERNODE_OPERATION_METRICS[16..18] { assert_eq!(metric.labels, &[SERVER_LABEL]); } - assert_eq!(INTERNODE_OPERATION_METRICS[17].labels, &[SERVER_LABEL, REASON_LABEL]); - assert_eq!(INTERNODE_OPERATION_METRICS[18].labels, &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]); + assert_eq!(INTERNODE_OPERATION_METRICS[18].labels, &[SERVER_LABEL, REASON_LABEL]); + assert_eq!(INTERNODE_OPERATION_METRICS[19].labels, &[SERVER_LABEL, STAGE_LABEL, DOMINANT_ERROR_LABEL]); // Payload histogram + large-payload counter carry operation+backend labels. - assert_eq!(INTERNODE_OPERATION_METRICS[19].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]); assert_eq!(INTERNODE_OPERATION_METRICS[20].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]); + assert_eq!(INTERNODE_OPERATION_METRICS[21].labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]); } #[test] @@ -1054,62 +1133,66 @@ mod tests { ); assert_eq!( INTERNODE_OPERATION_METRICS[6].name, - "rustfs_system_network_internode_operation_classified_errors_total" + "rustfs_system_network_internode_operation_stage_duration_ms" ); assert_eq!( INTERNODE_OPERATION_METRICS[7].name, - "rustfs_system_network_internode_operation_retries_total" + "rustfs_system_network_internode_operation_classified_errors_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[8].name, - "rustfs_system_network_internode_operation_retry_successes_total" + "rustfs_system_network_internode_operation_retries_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[9].name, - "rustfs_system_network_internode_operation_http_versions_total" + "rustfs_system_network_internode_operation_retry_successes_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[10].name, - "rustfs_system_network_internode_operation_stall_timeouts_total" + "rustfs_system_network_internode_operation_http_versions_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[11].name, - "rustfs_system_network_internode_operation_write_shutdown_errors_total" + "rustfs_system_network_internode_operation_stall_timeouts_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[12].name, - "rustfs_system_network_internode_rpc_auth_failures_total" + "rustfs_system_network_internode_operation_write_shutdown_errors_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[13].name, - "rustfs_system_network_internode_replay_cache_overflow_by_operation_total" + "rustfs_system_network_internode_rpc_auth_failures_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[14].name, - "rustfs_system_network_internode_replay_cache_records_total" + "rustfs_system_network_internode_replay_cache_overflow_by_operation_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[15].name, - "rustfs_system_network_internode_replay_cache_entries" + "rustfs_system_network_internode_replay_cache_records_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[16].name, - "rustfs_system_network_internode_replay_cache_capacity" + "rustfs_system_network_internode_replay_cache_entries" ); assert_eq!( INTERNODE_OPERATION_METRICS[17].name, - "rustfs_system_network_internode_replay_cache_evictions_total" + "rustfs_system_network_internode_replay_cache_capacity" ); assert_eq!( INTERNODE_OPERATION_METRICS[18].name, - "rustfs_system_storage_erasure_write_quorum_failures_total" + "rustfs_system_network_internode_replay_cache_evictions_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[19].name, - "rustfs_system_network_internode_operation_payload_bytes" + "rustfs_system_storage_erasure_write_quorum_failures_total" ); assert_eq!( INTERNODE_OPERATION_METRICS[20].name, + "rustfs_system_network_internode_operation_payload_bytes" + ); + assert_eq!( + INTERNODE_OPERATION_METRICS[21].name, "rustfs_system_network_internode_operation_large_payloads_total" ); assert_eq!(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, "grpc_read_multiple"); @@ -1129,6 +1212,16 @@ mod tests { assert_eq!(INTERNODE_MSGPACK_DIRECTION_RESPONSE, "response"); assert_eq!(INTERNODE_MSGPACK_CODEC_MSGPACK, "msgpack"); assert_eq!(INTERNODE_MSGPACK_CODEC_JSON, "json"); + assert_eq!(INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, "read_version_request_encode"); + assert_eq!(INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE, "read_version_request_decode"); + assert_eq!(INTERNODE_STAGE_READ_VERSION_DISK_READ, "read_version_disk_read"); + assert_eq!(INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE, "read_version_response_json_encode"); + assert_eq!( + INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE, + "read_version_response_msgpack_encode" + ); + assert_eq!(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, "read_version_rpc_roundtrip"); + assert_eq!(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, "read_version_response_decode"); assert_eq!( INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL, "rustfs_system_network_internode_signature_v1_fallback_total" diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index ce0ba42e5..5d9fbc969 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -23,12 +23,15 @@ use bytes::Bytes; use rustfs_filemeta::FileInfo; use rustfs_io_metrics::internode_metrics::{ INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_REQUEST, - INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC, - global_internode_metrics, + INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, + INTERNODE_STAGE_READ_VERSION_DISK_READ, INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE, + INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE, + INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics, }; use rustfs_protos::proto_gen::node_service::*; use serde::de::DeserializeOwned; use std::io::Cursor; +use std::time::Instant; use tonic::{Request, Response, Status}; use tracing::debug; @@ -201,6 +204,21 @@ fn encode_read_multiple_response_payloads( Ok((read_multiple_resps_json, read_multiple_resps_bin)) } +fn internode_stage_timer(attribution_enabled: bool) -> Option { + attribution_enabled.then(Instant::now) +} + +fn record_read_version_stage(stage: &'static str, started_at: Option) { + if let Some(started_at) = started_at { + global_internode_metrics().record_stage_duration_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + stage, + started_at.elapsed(), + ); + } +} + fn encode_batch_read_version_response_payloads( batch_read_version_resps: &[BatchReadVersionResp], request_decoded_from_msgpack: bool, @@ -685,11 +703,42 @@ impl NodeService { request: Request, ) -> Result, Status> { let request = request.into_inner(); + let metrics = global_internode_metrics(); + let read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + if read_version_attribution_enabled { + metrics.record_incoming_request_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); + metrics.record_recv_bytes_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + request + .disk + .len() + .saturating_add(request.volume.len()) + .saturating_add(request.path.len()) + .saturating_add(request.version_id.len()) + .saturating_add(request.opts.len()) + .saturating_add(request.opts_bin.len()), + ); + } if let Some(disk) = self.find_disk(&request.disk).await { let request_had_msgpack_payload = !request.opts_bin.is_empty(); + let decode_started = internode_stage_timer(read_version_attribution_enabled); let opts = match decode_msgpack_or_json::(&request.opts_bin, &request.opts, "ReadOptions") { - Ok(options) => options, + Ok(options) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE, decode_started); + options + } Err(err) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE, decode_started); + if read_version_attribution_enabled { + metrics.record_error_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); + } return Ok(Response::new(ReadVersionResponse { success: false, file_info: String::new(), @@ -698,42 +747,88 @@ impl NodeService { })); } }; + let disk_read_started = internode_stage_timer(read_version_attribution_enabled); match disk .read_version("", &request.volume, &request.path, &request.version_id, &opts) .await { Ok(file_info) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_DISK_READ, disk_read_started); + let json_encode_started = internode_stage_timer(read_version_attribution_enabled); let file_info_json = compat_response_json(&file_info, request_had_msgpack_payload); + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE, json_encode_started); + let msgpack_encode_started = internode_stage_timer(read_version_attribution_enabled); let file_info_bin = encode_file_info_msgpack(&file_info); + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE, msgpack_encode_started); match (file_info_json, file_info_bin) { - (Ok(file_info), Ok(file_info_bin)) => Ok(Response::new(ReadVersionResponse { - success: true, - file_info, - file_info_bin: file_info_bin.into(), - error: None, - })), - (Err(err), _) => Ok(Response::new(ReadVersionResponse { - success: false, - file_info: String::new(), - file_info_bin: Vec::new().into(), - error: Some(DiskError::other(format!("encode data failed: {err}")).into()), - })), - (_, Err(err)) => Ok(Response::new(ReadVersionResponse { - success: false, - file_info: String::new(), - file_info_bin: Vec::new().into(), - error: Some(DiskError::other(format!("encode data failed: {err}")).into()), - })), + (Ok(file_info), Ok(file_info_bin)) => { + if read_version_attribution_enabled { + metrics.record_sent_bytes_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + file_info.len().saturating_add(file_info_bin.len()), + ); + } + Ok(Response::new(ReadVersionResponse { + success: true, + file_info, + file_info_bin: file_info_bin.into(), + error: None, + })) + } + (Err(err), _) => { + if read_version_attribution_enabled { + metrics.record_error_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); + } + Ok(Response::new(ReadVersionResponse { + success: false, + file_info: String::new(), + file_info_bin: Vec::new().into(), + error: Some(DiskError::other(format!("encode data failed: {err}")).into()), + })) + } + (_, Err(err)) => { + if read_version_attribution_enabled { + metrics.record_error_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); + } + Ok(Response::new(ReadVersionResponse { + success: false, + file_info: String::new(), + file_info_bin: Vec::new().into(), + error: Some(DiskError::other(format!("encode data failed: {err}")).into()), + })) + } } } - Err(err) => Ok(Response::new(ReadVersionResponse { - success: false, - file_info: String::new(), - file_info_bin: Vec::new().into(), - error: Some(err.into()), - })), + Err(err) => { + record_read_version_stage(INTERNODE_STAGE_READ_VERSION_DISK_READ, disk_read_started); + if read_version_attribution_enabled { + metrics.record_error_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); + } + Ok(Response::new(ReadVersionResponse { + success: false, + file_info: String::new(), + file_info_bin: Vec::new().into(), + error: Some(err.into()), + })) + } } } else { + if read_version_attribution_enabled { + metrics.record_error_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); + } Ok(Response::new(ReadVersionResponse { success: false, file_info: String::new(), @@ -1520,12 +1615,16 @@ mod tests { encode_batch_read_version_response_payloads, encode_file_info_msgpack, encode_msgpack, encode_msgpack_named, encode_read_multiple_response_payloads, encode_rename_data_response_payloads, }; + use crate::storage::rpc::node_service::make_server; use crate::storage::storage_api::ReadMultipleResp; use crate::storage::storage_api::RenameDataResp; use crate::storage::storage_api::rpc_consumer::node_service::BatchReadVersionResp; use rustfs_filemeta::FileInfo; use rustfs_io_metrics::internode_metrics::global_internode_metrics; + use rustfs_protos::proto_gen::node_service::ReadVersionRequest; use serde::{Deserialize, Serialize}; + use serial_test::serial; + use tonic::Request; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct SamplePayload { @@ -1533,6 +1632,36 @@ mod tests { count: u32, } + #[tokio::test] + #[serial] + async fn handle_read_version_records_attribution_for_missing_disk() { + let metrics = global_internode_metrics(); + let previous_stage_metrics = rustfs_io_metrics::get_stage_metrics_enabled(); + metrics.reset_for_test(); + rustfs_io_metrics::set_get_stage_metrics_enabled(true); + + let response = make_server() + .handle_read_version(Request::new(ReadVersionRequest { + disk: "missing-disk".to_string(), + volume: "bucket".to_string(), + path: "object".to_string(), + version_id: String::new(), + opts: String::new(), + opts_bin: Vec::new().into(), + })) + .await + .expect("ReadVersion handler should return a response") + .into_inner(); + + rustfs_io_metrics::set_get_stage_metrics_enabled(previous_stage_metrics); + let snapshot = metrics.snapshot(); + assert!(!response.success); + assert_eq!(snapshot.incoming_requests_total, 1); + assert_eq!(snapshot.errors_total, 1); + assert!(snapshot.recv_bytes_total > 0); + metrics.reset_for_test(); + } + #[test] fn decode_msgpack_or_json_prefers_binary_payload() { let payload = SamplePayload { From d7609b68a6a2382e68640e840fb87afb06f873fd Mon Sep 17 00:00:00 2001 From: cxymds Date: Wed, 19 Aug 2026 14:47:33 +0800 Subject: [PATCH 21/27] fix(lock): reject stale lease snapshots (#6249) --- crates/lock/src/client/local.rs | 21 +++- crates/lock/src/fast_lock/manager.rs | 45 ++++++- crates/lock/src/fast_lock/shard.rs | 80 +++++++++++- crates/lock/src/fast_lock/types.rs | 2 +- crates/lock/src/types.rs | 2 + rustfs/src/admin/handlers/diagnostics.rs | 154 +++++++++++++++++++++-- 6 files changed, 277 insertions(+), 27 deletions(-) diff --git a/crates/lock/src/client/local.rs b/crates/lock/src/client/local.rs index 95adaffc3..84ef32a88 100644 --- a/crates/lock/src/client/local.rs +++ b/crates/lock/src/client/local.rs @@ -48,6 +48,7 @@ pub struct LocalClient { struct LocalGuardEntry { guard: FastLockGuard, acquired_at: SystemTime, + last_refreshed: SystemTime, expires_at: SystemTime, deadline: Instant, ttl: Duration, @@ -60,6 +61,7 @@ impl LocalGuardEntry { Self { guard, acquired_at, + last_refreshed: acquired_at, expires_at: acquired_at.checked_add(ttl).unwrap_or(acquired_at), deadline: monotonic_now.checked_add(ttl).unwrap_or(monotonic_now), ttl, @@ -74,6 +76,7 @@ impl LocalGuardEntry { let now = SystemTime::now(); let monotonic_now = Instant::now(); self.expires_at = now.checked_add(self.ttl).unwrap_or(now); + self.last_refreshed = now; self.deadline = monotonic_now.checked_add(self.ttl).unwrap_or(monotonic_now); } } @@ -347,7 +350,7 @@ impl LockClient for LocalClient { owner: entry.guard.owner().to_string(), acquired_at: entry.acquired_at, expires_at: entry.expires_at, - last_refreshed: SystemTime::now(), + last_refreshed: entry.last_refreshed, metadata: LockMetadata::default(), priority: LockPriority::Normal, wait_start_time: None, @@ -371,6 +374,7 @@ impl LockClient for LocalClient { owner: entry.guard.owner().to_string(), acquired_at: entry.acquired_at, remaining_ttl: entry.deadline.saturating_duration_since(Instant::now()), + guard_id: (!entry.guard.is_disabled()).then(|| entry.guard.guard_id()), })); } leases @@ -484,6 +488,12 @@ mod tests { .success ); let initial = client.list_lock_leases().await.pop().expect("acquired lock should be listed"); + let initial_status = client + .check_status(&lock_id) + .await + .expect("initial lock status should be readable") + .expect("newly acquired lock should remain held"); + assert_eq!(initial_status.last_refreshed, initial_status.acquired_at); tokio::time::advance(Duration::from_secs(20)).await; let aging = client @@ -492,6 +502,13 @@ mod tests { .pop() .expect("held lock should remain listed before refresh"); assert_eq!(aging.remaining_ttl, Duration::from_secs(10)); + let aging_status = client + .check_status(&lock_id) + .await + .expect("aging lock status should be readable") + .expect("aging lock should remain held"); + assert_eq!(aging_status.last_refreshed, initial_status.last_refreshed); + assert!(client.refresh(&lock_id).await.expect("refresh should return a result")); let refreshed = client @@ -506,7 +523,9 @@ mod tests { .expect("refreshed lock should remain held"); assert_eq!(refreshed.acquired_at, initial.acquired_at); + assert_eq!(refreshed.guard_id, initial.guard_id); assert_eq!(status.acquired_at, initial.acquired_at); + assert!(status.last_refreshed > initial_status.last_refreshed); assert_eq!(refreshed.remaining_ttl, Duration::from_secs(30)); tokio::time::advance(Duration::from_secs(30)).await; diff --git a/crates/lock/src/fast_lock/manager.rs b/crates/lock/src/fast_lock/manager.rs index 30a1c4d18..fecb2d92f 100644 --- a/crates/lock/src/fast_lock/manager.rs +++ b/crates/lock/src/fast_lock/manager.rs @@ -100,7 +100,7 @@ impl FastObjectLockManager { Ok(()) => { let guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone()); // Register guard to prevent premature cleanup - shard.register_guard(guard.guard_id()); + shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner()); Ok(guard) } Err(err) => Err(err), @@ -223,7 +223,7 @@ impl FastObjectLockManager { if acquired { let guard = FastLockGuard::new(key.clone(), mode, owner.clone(), shard.clone()); - shard.register_guard(guard.guard_id()); + shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner()); all_successful.push(key); guards.push(guard); } @@ -252,7 +252,7 @@ impl FastObjectLockManager { match shard.acquire_lock(request).await { Ok(()) => { let guard = FastLockGuard::new(request.key.clone(), request.mode, request.owner.clone(), shard.clone()); - shard.register_guard(guard.guard_id()); + shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner()); acquired_guards.push(guard); } Err(err) => { @@ -310,6 +310,15 @@ impl FastObjectLockManager { infos } + /// Enumerate held locks with holder counts and stable holder identities. + pub fn list_locks_with_holder_generations(&self) -> Vec<(crate::fast_lock::types::ObjectLockInfo, u32, Option>)> { + let mut infos = Vec::new(); + for shard in &self.shards { + infos.extend(shard.list_locks_with_holder_generations()); + } + infos + } + /// Force-release every holder of the lock on `key`. /// /// Returns the number of owners released (0 if the resource was not locked). @@ -556,15 +565,15 @@ mod tests { let write_key = ObjectKey::new("bucket", "write-object"); let read_key = ObjectKey::new("bucket", "read-object"); - let _write_guard = manager + let write_guard = manager .acquire_write_lock(write_key.clone(), "writer") .await .expect("write lock should acquire"); - let _read_guard = manager + let read_guard = manager .acquire_read_lock(read_key.clone(), "reader") .await .expect("read lock should acquire"); - let _second_read_guard = manager + let second_read_guard = manager .acquire_read_lock(read_key.clone(), "reader") .await .expect("second read lock should acquire"); @@ -593,6 +602,30 @@ mod tests { .expect("write holder count listed"); assert_eq!(*write_holder_count, 1); + let generations = manager.list_locks_with_holder_generations(); + let (_, _, read_generations) = generations + .iter() + .find(|(info, _, _)| info.key == read_key) + .expect("read holder generations listed"); + let mut expected_read_generations = vec![read_guard.guard_id(), second_read_guard.guard_id()]; + expected_read_generations.sort_unstable(); + assert_eq!(read_generations.as_ref(), Some(&expected_read_generations)); + + let (_, _, write_generations) = generations + .iter() + .find(|(info, _, _)| info.key == write_key) + .expect("write holder generation listed"); + assert_eq!(write_generations.as_ref(), Some(&vec![write_guard.guard_id()])); + + drop(read_guard); + let remaining = manager.list_locks_with_holder_generations(); + let (_, remaining_count, remaining_generations) = remaining + .iter() + .find(|(info, _, _)| info.key == read_key) + .expect("remaining read holder generation listed"); + assert_eq!(*remaining_count, 1); + assert_eq!(remaining_generations.as_ref(), Some(&vec![second_read_guard.guard_id()])); + manager.shutdown().await; } diff --git a/crates/lock/src/fast_lock/shard.rs b/crates/lock/src/fast_lock/shard.rs index 8b55b968b..5da4d355b 100644 --- a/crates/lock/src/fast_lock/shard.rs +++ b/crates/lock/src/fast_lock/shard.rs @@ -24,7 +24,20 @@ use crate::fast_lock::{ state::ObjectLockState, types::{LockMode, LockResult, ObjectKey, ObjectLockRequest}, }; -use std::collections::HashSet; + +#[derive(Debug)] +struct ActiveGuardInfo { + key: ObjectKey, + mode: LockMode, + owner: Arc, +} + +#[derive(Debug, PartialEq, Eq, Hash)] +struct GuardHolderKey { + key: ObjectKey, + mode: LockMode, + owner: Arc, +} /// Lock shard to reduce global contention #[derive(Debug)] @@ -38,7 +51,7 @@ pub struct LockShard { /// Shard ID for debugging _shard_id: usize, /// Active guard IDs to prevent cleanup of locks with live guards - active_guards: parking_lot::Mutex>, + active_guards: parking_lot::Mutex>>, } /// Cancellation-safe waiter counter ticket. @@ -84,7 +97,7 @@ impl LockShard { object_pool: ObjectStatePool::new(), metrics: ShardMetrics::new(), _shard_id: shard_id, - active_guards: parking_lot::Mutex::new(HashSet::new()), + active_guards: parking_lot::Mutex::new(HashMap::new()), } } @@ -327,7 +340,7 @@ impl LockShard { // First, try to remove the guard from active set let guard_was_active = { let mut guards = self.active_guards.lock(); - guards.remove(&guard_id) + guards.remove(&guard_id).is_some() }; // If guard was not active, this is a double-release attempt @@ -375,8 +388,19 @@ impl LockShard { /// Register a guard to prevent premature cleanup pub fn register_guard(&self, guard_id: u64) { + self.active_guards.lock().insert(guard_id, None); + } + + pub(crate) fn register_guard_with_info(&self, guard_id: u64, key: &ObjectKey, mode: LockMode, owner: &Arc) { let mut guards = self.active_guards.lock(); - guards.insert(guard_id); + guards.insert( + guard_id, + Some(ActiveGuardInfo { + key: key.clone(), + mode, + owner: owner.clone(), + }), + ); } /// Unregister a guard (called when guard is dropped) @@ -396,7 +420,7 @@ impl LockShard { #[cfg(test)] pub fn is_guard_active(&self, guard_id: u64) -> bool { let guards = self.active_guards.lock(); - guards.contains(&guard_id) + guards.contains_key(&guard_id) } /// Calculate adaptive timeout based on current system load and request priority @@ -602,6 +626,50 @@ impl LockShard { infos } + pub(crate) fn list_locks_with_holder_generations( + &self, + ) -> Vec<(crate::fast_lock::types::ObjectLockInfo, u32, Option>)> { + // Snapshot lock state before guard registrations. Acquires register after + // mutating state, while releases unregister before mutating state, so a + // concurrent transition can only make the cohort mismatch and fall back. + let infos = self.list_locks_with_holder_counts(); + let guards = self.active_guards.lock(); + let mut guard_ids_by_holder: HashMap> = HashMap::with_capacity(guards.len()); + for (&guard_id, guard) in guards + .iter() + .filter_map(|(guard_id, guard)| guard.as_ref().map(|guard| (guard_id, guard))) + { + let key = GuardHolderKey { + key: guard.key.clone(), + mode: guard.mode, + owner: guard.owner.clone(), + }; + guard_ids_by_holder + .entry(key) + .and_modify(|guard_ids| guard_ids.push(guard_id)) + .or_insert_with(|| vec![guard_id]); + } + drop(guards); + for guard_ids in guard_ids_by_holder.values_mut() { + guard_ids.sort_unstable(); + } + + infos + .into_iter() + .map(|(info, holder_count)| { + let key = GuardHolderKey { + key: info.key.clone(), + mode: info.mode, + owner: info.owner.clone(), + }; + let generation = guard_ids_by_holder + .remove(&key) + .filter(|guard_ids| u32::try_from(guard_ids.len()).ok() == Some(holder_count)); + (info, holder_count, generation) + }) + .collect() + } + /// Force-release every holder of a lock on `key`, regardless of owner. /// /// Returns the number of owners that were released. Used by the admin diff --git a/crates/lock/src/fast_lock/types.rs b/crates/lock/src/fast_lock/types.rs index 8337971fe..fbf51f9ae 100644 --- a/crates/lock/src/fast_lock/types.rs +++ b/crates/lock/src/fast_lock/types.rs @@ -257,7 +257,7 @@ impl std::fmt::Display for ObjectKey { } /// Lock type for object operations -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum LockMode { /// Shared lock for read operations Shared, diff --git a/crates/lock/src/types.rs b/crates/lock/src/types.rs index 5336fb93e..77a344dea 100644 --- a/crates/lock/src/types.rs +++ b/crates/lock/src/types.rs @@ -90,6 +90,8 @@ pub struct LockLeaseInfo { pub owner: String, /// Original acquisition time. Refreshes do not change this value. pub acquired_at: SystemTime, + /// Opaque guard identity used to reject stale diagnostic snapshots. + pub guard_id: Option, /// Remaining lease duration derived from the monotonic lease deadline. pub remaining_ttl: Duration, } diff --git a/rustfs/src/admin/handlers/diagnostics.rs b/rustfs/src/admin/handlers/diagnostics.rs index 35410c159..83c3625fe 100644 --- a/rustfs/src/admin/handlers/diagnostics.rs +++ b/rustfs/src/admin/handlers/diagnostics.rs @@ -42,6 +42,7 @@ use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, StdError, use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::pin::Pin; +use std::sync::Arc; use std::task::{Context, Poll}; use std::time::{Duration, SystemTime}; use tokio::sync::{Semaphore, SemaphorePermit, mpsc}; @@ -248,13 +249,14 @@ struct LeaseHolderState { acquired_at: SystemTime, ttl_secs: u64, holder_count: u32, + guard_ids: Option>, } fn build_top_locks_response( limit: usize, now: SystemTime, lease_infos: Vec, - fast_infos: Vec<(rustfs_lock::ObjectLockInfo, u32)>, + fast_infos: Vec<(rustfs_lock::ObjectLockInfo, u32, Option>)>, ) -> TopLocksResponse { let mut lease_holders = HashMap::with_capacity(lease_infos.len()); @@ -277,17 +279,27 @@ fn build_top_locks_response( } state.ttl_secs = state.ttl_secs.max(ttl_secs); state.holder_count = state.holder_count.saturating_add(1); + match (state.guard_ids.as_mut(), info.guard_id) { + (Some(guard_ids), Some(guard_id)) => guard_ids.push(guard_id), + _ => state.guard_ids = None, + } }) .or_insert(LeaseHolderState { acquired_at: info.acquired_at, ttl_secs, holder_count: 1, + guard_ids: info.guard_id.map(|guard_id| vec![guard_id]), }); } + for state in lease_holders.values_mut() { + if let Some(guard_ids) = &mut state.guard_ids { + guard_ids.sort_unstable(); + } + } let mut infos: Vec<_> = fast_infos .into_iter() - .map(|(info, holder_count)| { + .map(|(info, holder_count, guard_ids)| { let mode = match info.mode { LockMode::Shared => TopLockMode::Read, LockMode::Exclusive => TopLockMode::Write, @@ -298,11 +310,10 @@ fn build_top_locks_response( owner: info.owner.to_string(), }; let priority = lock_priority_label(info.priority); - // Shared-owner timestamps do not roll back when a newer sibling releases, so only their count is stable. + // Match the complete holder cohort so replacements cannot reuse stale lease data. let state = match lease_holders.remove(&key) { Some(lease) - if lease.holder_count == holder_count - && (mode == TopLockMode::Read || info.acquired_at <= lease.acquired_at) => + if lease.holder_count == holder_count && lease.guard_ids.is_some() && lease.guard_ids == guard_ids => { TopLockState { acquired_at: lease.acquired_at, @@ -349,8 +360,11 @@ fn build_top_locks_response( } } -async fn collect_top_locks(limit: usize) -> TopLocksResponse { - let manager = get_global_lock_manager(); +async fn collect_top_locks_with_clients( + limit: usize, + manager: Arc, + clients: Vec>, +) -> TopLocksResponse { let Some(fast) = manager.as_fast_lock_manager() else { return TopLocksResponse { total: 0, @@ -362,21 +376,29 @@ async fn collect_top_locks(limit: usize) -> TopLocksResponse { }; }; - let lease_infos = if let Some(clients) = get_global_lock_clients() { - join_all(clients.values().map(|client| client.list_lock_leases())) + let lease_infos = if clients.is_empty() { + Vec::new() + } else { + join_all(clients.iter().map(|client| client.list_lock_leases())) .await .into_iter() .flatten() .collect() - } else { - Vec::new() }; // Capture holders last so released or replaced lease guards fail the merge checks. - let fast_infos = fast.list_locks_with_holder_counts(); + let fast_infos = fast.list_locks_with_holder_generations(); build_top_locks_response(limit, SystemTime::now(), lease_infos, fast_infos) } +async fn collect_top_locks(limit: usize) -> TopLocksResponse { + let manager = get_global_lock_manager(); + let clients = get_global_lock_clients() + .map(|clients| clients.values().cloned().collect()) + .unwrap_or_default(); + collect_top_locks_with_clients(limit, manager, clients).await +} + fn parse_top_locks_limit(uri: &Uri) -> usize { query_value(uri, "count") .and_then(|v| v.parse::().ok()) @@ -1229,6 +1251,7 @@ mod tests { let mixed_resource = ObjectKey::new("bucket", "mixed-object"); let replaced_resource = ObjectKey::new("bucket", "replaced-object"); let remaining_shared_resource = ObjectKey::new("bucket", "remaining-shared-object"); + let opaque_resource = ObjectKey::new("bucket", "opaque-object"); let response = build_top_locks_response( TOP_LOCKS_DEFAULT_LIMIT, @@ -1239,6 +1262,7 @@ mod tests { lock_type: LockType::Shared, owner: "owner-a".to_string(), acquired_at: now - Duration::from_secs(50), + guard_id: Some(11), remaining_ttl: Duration::from_secs(5), }, LockLeaseInfo { @@ -1246,6 +1270,7 @@ mod tests { lock_type: LockType::Shared, owner: "owner-a".to_string(), acquired_at: now - Duration::from_secs(40), + guard_id: Some(18), remaining_ttl: Duration::from_secs(20), }, LockLeaseInfo { @@ -1253,6 +1278,7 @@ mod tests { lock_type: LockType::Shared, owner: "owner-c".to_string(), acquired_at: now - Duration::from_secs(30), + guard_id: Some(13), remaining_ttl: Duration::from_secs(25), }, LockLeaseInfo { @@ -1260,6 +1286,7 @@ mod tests { lock_type: LockType::Exclusive, owner: "owner-d".to_string(), acquired_at: now - Duration::from_secs(15), + guard_id: Some(12), remaining_ttl: Duration::from_secs(18), }, LockLeaseInfo { @@ -1267,6 +1294,7 @@ mod tests { lock_type: LockType::Exclusive, owner: "owner-e".to_string(), acquired_at: now - Duration::from_secs(30), + guard_id: Some(14), remaining_ttl: Duration::from_secs(25), }, LockLeaseInfo { @@ -1274,8 +1302,17 @@ mod tests { lock_type: LockType::Shared, owner: "owner-f".to_string(), acquired_at: now - Duration::from_secs(30), + guard_id: Some(16), remaining_ttl: Duration::from_secs(22), }, + LockLeaseInfo { + resource: opaque_resource.clone(), + lock_type: LockType::Exclusive, + owner: "owner-g".to_string(), + acquired_at: now - Duration::from_secs(30), + guard_id: None, + remaining_ttl: Duration::from_secs(30), + }, ], vec![ ( @@ -1288,6 +1325,7 @@ mod tests { priority: rustfs_lock::fast_lock::LockPriority::Normal, }, 1, + Some(vec![15]), ), ( rustfs_lock::ObjectLockInfo { @@ -1299,6 +1337,7 @@ mod tests { priority: rustfs_lock::fast_lock::LockPriority::Normal, }, 1, + Some(vec![16]), ), ( rustfs_lock::ObjectLockInfo { @@ -1310,6 +1349,7 @@ mod tests { priority: rustfs_lock::fast_lock::LockPriority::Normal, }, 1, + Some(vec![12]), ), ( rustfs_lock::ObjectLockInfo { @@ -1321,6 +1361,7 @@ mod tests { priority: rustfs_lock::fast_lock::LockPriority::Normal, }, 2, + Some(vec![11, 18]), ), ( rustfs_lock::ObjectLockInfo { @@ -1332,6 +1373,7 @@ mod tests { priority: rustfs_lock::fast_lock::LockPriority::Normal, }, 1, + Some(vec![17]), ), ( rustfs_lock::ObjectLockInfo { @@ -1343,11 +1385,24 @@ mod tests { priority: rustfs_lock::fast_lock::LockPriority::Normal, }, 2, + None, + ), + ( + rustfs_lock::ObjectLockInfo { + key: opaque_resource, + mode: LockMode::Exclusive, + owner: "owner-g".into(), + acquired_at: now - Duration::from_secs(4), + expires_at: now + Duration::from_secs(6), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 1, + None, ), ], ); - assert_eq!(response.total, 6); + assert_eq!(response.total, 7); let leased = response .locks .iter() @@ -1392,6 +1447,47 @@ mod tests { .find(|entry| entry.object == "remaining-shared-object") .expect("an older surviving shared lease should remain lease-backed"); assert_eq!(remaining_shared.ttl_secs, 22); + + let opaque = response + .locks + .iter() + .find(|entry| entry.object == "opaque-object") + .expect("generation-less holder should remain visible"); + assert_eq!(opaque.ttl_secs, 6); + } + + #[test] + fn top_locks_rejects_replaced_shared_generation() { + let now = SystemTime::UNIX_EPOCH + Duration::from_secs(100); + let resource = ObjectKey::new("bucket", "replaced-shared-object"); + let response = build_top_locks_response( + TOP_LOCKS_DEFAULT_LIMIT, + now, + vec![LockLeaseInfo { + resource: resource.clone(), + lock_type: LockType::Shared, + owner: "owner-a".to_string(), + acquired_at: now - Duration::from_secs(30), + guard_id: Some(1), + remaining_ttl: Duration::from_secs(20), + }], + vec![( + rustfs_lock::ObjectLockInfo { + key: resource, + mode: LockMode::Shared, + owner: "owner-a".into(), + acquired_at: now - Duration::from_secs(2), + expires_at: now + Duration::from_secs(4), + priority: rustfs_lock::fast_lock::LockPriority::Normal, + }, + 1, + Some(vec![2]), + )], + ); + + let entry = response.locks.first().expect("replacement remains visible"); + assert_eq!(entry.ttl_secs, 4); + assert_eq!(entry.elapsed_secs, 2); } #[tokio::test] @@ -1431,4 +1527,36 @@ mod tests { drop(guard); } + + #[tokio::test(start_paused = true)] + async fn collect_top_locks_uses_refreshed_local_lease() { + use rustfs_lock::{FastObjectLockManager, GlobalLockManager, LocalClient, LockClient, LockRequest}; + + let manager = Arc::new(GlobalLockManager::Enabled(Arc::new(FastObjectLockManager::new()))); + let client = Arc::new(LocalClient::with_manager(manager.clone())); + let request = LockRequest::new(ObjectKey::new("diag-bucket", "renewed-object"), LockType::Exclusive, "diag-owner") + .with_ttl(Duration::from_secs(30)); + let lock_id = request.lock_id.clone(); + assert!( + client + .acquire_lock(&request) + .await + .expect("local lock acquisition should succeed") + .success + ); + + tokio::time::advance(Duration::from_secs(20)).await; + assert!(client.refresh(&lock_id).await.expect("local lease refresh should succeed")); + + let clients: Vec> = vec![client.clone()]; + let response = collect_top_locks_with_clients(TOP_LOCKS_DEFAULT_LIMIT, manager, clients).await; + let entry = response + .locks + .iter() + .find(|entry| entry.bucket == "diag-bucket" && entry.object == "renewed-object") + .expect("refreshed local lock should be listed"); + assert!(entry.ttl_secs >= 29, "collector must use the refreshed lease deadline"); + + assert!(client.release(&lock_id).await.expect("local lock release should succeed")); + } } From 1f8359537b97b18ddb2ced03acd99fe4747a4040 Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 19 Aug 2026 18:15:03 +0800 Subject: [PATCH 22/27] docs(operations): restore the truncated tail of the English audit baseline (#6268) The merge of rustfs#6261 lost the last 64 lines of the English translation: merging main (to pick up rustfs#6258) resolved the conflict on the renamed file by cutting it mid-table in section 6, which dropped section 7 (backlog/history index), section 8 (audit method and limitations) and section 9 (landing results) that the Chinese counterpart still carries. Restore them verbatim from the translation commit (0e051602f) so both language versions are complete 568-line mirrors of the full 0-9 baseline, as the PR body promised. Co-authored-by: heihutu --- ...minio-comprehensive-analysis-2026-08-16.md | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md index d696b6871..931d020e8 100644 --- a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md +++ b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md @@ -502,3 +502,67 @@ scanner (admin subsystem `scanner`, `crates/config/src/constants/scanner.rs` + ` | delay / max_wait / cycle / start_delay | RUSTFS_SCANNER_* | derived/empty | | cycle_max_duration/objects/directories | …_MAX_* | 0 (unlimited) | | bitrot_cycle | …_BITROT_CYCLE_SECS | 2592000 (30d; 0/on=every cycle, off=disabled) | +| idle_mode | …_IDLE_MODE | true | +| cache_save_timeout | …_CACHE_SAVE_TIMEOUT_SECS | 30s | +| max_concurrent_set_scans / disk_scans | …_MAX_CONCURRENT_* | 4/4 | +| yield_every_n_objects | …_YIELD_EVERY_N_OBJECTS | 128 | +| alert_excess_versions / version_size / folders | …_ALERT_* | 100 / 1TiB / 65538 | + +scanner-internal env: `RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=16`, `RUSTFS_HEAL_OBJECT_SELECT_PROB=1024`, `RUSTFS_SCANNER_DEEP_VERIFY_COOLDOWN_SECS=60`, `RUSTFS_DATA_USAGE_FAILED_OBJECT_TTL_SECS=86400`/`_MAX=10000`, `RUSTFS_LOCK_ACQUIRE_TIMEOUT=5s`, `RUSTFS_SCANNER_ENABLED=true`, `RUSTFS_SCANNER_INLINE_HEAL_ENABLE=false` (compat warning). + +All 17 scanner keys support the env > config dual channel + admin PUT hot update (generation+Notify takes effect immediately); heal runtime parameters are currently env-only (no admin hot-update entry; the `Arc>` structure is already reserved). + +--- + +## 7. Related backlog / history index + +- Automatic drive-replacement healing series (closed loop): backlog #1786 (redundant false-green algorithm), #1787 (target-slot restriction), #1789 (binding resume and the healing marker to the replacement instance), #1791 (black-box/white-box acceptance matrix). +- #801 DiskInfo.healing never assigned (fixed and closed; the assignment chain now lives at `set_disk/mod.rs:4988`). +- #1651 Scanner metrics node/source/bucket-drive dimensions (OPEN; related to §3.8/§4.6 of this analysis). +- #1843 crates/common 83% scanner/heal domain code layering migration (OPEN; includes HS-20). +- Historical defects cited in code comments (now guarded with regression tests): #856/#799 B7 (offline drive falsely recorded healed), #855/B6/#1033 (a skip round must not be marked complete), #920 (sub-quorum union enumeration), #856 B5 (per-version resume), #5173 (bitrot trailing bytes), #5029 (stale-version merge at regression nodes). +- v1 parity document: `docs/rustfs-heal-scanner-vs-minio-parity-assessment.md` (superseded by this document); the landing playbook `docs/rustfs-heal-scanner-vs-minio-improvement-playbook.md` (some entries have since been overtaken by implementation). +- Drive-replacement deep analyses: `docs/new-disk-replacement-and-healing-deep-analysis-zh.md`, `docs/node-disk-identity-and-healing-analysis-zh.md`. + +## 8. Audit method and limitations + +- Four parallel audit tracks (heal crate file by file, scanner crate file by file, ecstore integration-layer wiring, MinIO master source study) + the main session verifying each key "missing" conclusion first-hand (the get_disk_status TODO, HealEvent's zero external references, .bloomcycle.bin having no bloom implementation, check_abandoned_parts NotImplemented at all three layers, the ETag fallback being implemented, zero trace-channel hits, the already_running semantics). +- Points not verified line by line (marked "unconfirmed / not checked line by line" in the text): the DeleteAllVersions prefix single-call optimization (HS-17), trash two-stage cleanup details (HS-18), ilm worker default comparisons, stale multipart default comparisons, mc CLI flag spellings (MinIO side). Of these, HS-17 and HS-18 completed line-by-line verification on 2026-08-19; conclusions in §9.2/§9.3. +- MinIO-side references follow its master `7aac2a2c5b`; RustFS-side line numbers follow the 2026-08-16 workspace — for later evolution, search by symbol name instead. + +## 9. Landing results (updated 2026-08-19) + +All 14 sub-issues derived from this audit (backlog #1865~#1878) are closed. This section is the final disposition record for the gap list HS-01~HS-20, and also the incremental baseline for the next parity re-audit. + +### 9.1 Landed (all PRs merged to main) + +- HS-01 MRF wiring + persistent repair ledger (#1865, PR #6189): decision (a) chosen. common MRF channel (bounded 8192, try_send never blocks) + heal mrf_queue (100k entries / 8MiB dual-capacity ring) + `buckets/.heal/mrf/journal.bin` CRC-persisted replay (torn tail truncated, deleted after replay) + three delivery points (read decode_error→Urgent ECDecode, scanner metadata corruption→High Metadata, add_partial→Normal) + `RUSTFS_HEAL_MRF_ENABLE` one-switch rollback. +- HS-02 abandoned parts/data-dir reconciliation (#1866, PR #6179): wired up the abandoned-check entry, retaining dry-run / reclaim counters. +- HS-03 heal/scanner trace channels (#1867, PR #6179): in-process trace bus + `/v3/trace` admin streaming subscription + heal task / abandoned-parts / scanner folder / ILM / heal-candidate trace producers. +- HS-04 scanner excess S3 events (#1868, PR #6176): the three events `s3:Scanner:ManyVersions/LargeVersions/BigPrefix` + 24h edge cooldown; the HS-15 threshold delta documented (`docs/operations/scanner-excess-alerts.md`). +- HS-05 madmin client phase 1 (#1869, PR #6166): SigV4 admin client heal/scanner methods; incremental-consumption methods await a follow-up (the protocol was already folded in by HS-06). +- HS-06 admin heal incremental semantics and typed overlap (#1870, PR #6206): `sinceSeq/nextSeq/minSeq` incremental cursor (wire additive; absent = full snapshot) + `RUSTFS_HEAL_OVERLAP_POLICY` (default merge unchanged; under minio_error, typed AlreadyRunning/OverlappingPaths rejections) + forceStart stops the old sequence before starting the new one. +- HS-07 healing progress visibility (#1871, PR #6179): data-usage total baseline + baseline/current/healed counters. +- HS-08 prefix usage (#1872, PR #6171): `GET /v3/usage/{bucket}`. +- HS-11 bitrot startup self-test (#1873, PR #6165). +- HS-13 heal skip filters (#1875, PR #6179): filter-hit versions are no longer counted as failures. +- HS-16 single-node cycle hook (#1878, PR #6250): removed the always-None hook; the decision record is in `docs/operations/heal-scanner-parity-notes-zh.md`. +- HS-09/10/19/20 dead-code cleanup batch (#1877, PR #6256): net −911 lines, zero behavior change; the `get_disk_status` TODO (the repo's only product TODO) cleared to zero; `ec_decode_rebuild`/`get_object_meta`, kept due to the HS-01 linkage, are retained with Reserved annotations (MRF currently executes via `heal_object`). + +### 9.2 Confirmed "already implemented / not a gap" after verification (audit-period misjudgment corrections, four in total) + +- bloom filter (corrected in §0): removed from MinIO master; both sides now agree. +- ETag fallback arbitration (corrected in §0): RustFS already has the implementation (`set_disk/ops/heal.rs`). +- HS-17 (#1876, closed after line-by-line verification on 2026-08-19): the DeleteAllVersions prefix single-call optimization is fully implemented in RustFS — `apply_expiry_on_non_transitioned_objects` sets `delete_prefix + delete_prefix_object` for the two `delete_all()` actions and then performs a single `delete_object` call (`bucket_lifecycle_ops.rs:5047-5056`); the SetDisks branch takes one write lock + one all-version quorum read + inline per-version object-lock checks (`set_disk/ops/object.rs:5566-5612`), aligned line by line with MinIO `expire.go`'s `applyExpiryOnNonTransitionedObjects`. The item §8 listed as "not verified line by line" now has a conclusion: the current state is already the optimized path; nothing to implement. +- HS-14 (#1878, checked alongside PR #6250): MinIO's "idle = throttle only when idle" was the behavior before 2024-01 minio/minio#18734 (`scannerIdleMode` is now a static config; `idle_speed=on` by default means always throttling per the speed tier — the "idle" naming is a historical leftover); RustFS's `RUSTFS_SCANNER_IDLE_MODE` points the same way as MinIO's current semantics, and additionally has a foreground-read backoff floor that MinIO lacks. The real migration traps (the variable must carry the `RUSTFS_` prefix, the `on/off` vs `true/false` vocabulary, `false` also turning off foreground protection) are documented in `docs/operations/heal-scanner-parity-notes-zh.md`. + +### 9.3 Audit-style conclusions (no code change needed) + +- HS-12 (#1874, PR #6183): the class of race MinIO defends against with `x-minio-healing` does not exist — every commit surface for the same (bucket, object) is mutually exclusive under the same object-level ns write lock, and the heal lock guard covers the whole rename commit; delivered 2 concurrency-invariant regression tests + the intersection matrix in `docs/operations/heal-concurrency-safety-notes-zh.md`. +- HS-18 (#1878, line-by-line verification on 2026-08-19): trash/tmp three-stage cleanup fully aligned — stale multipart isolation-cleanup is equivalent and safer (`delete_all_with_quorum` recursively deletes per drive, i.e. the `move_to_trash` rename into `.rustfs.sys/tmp/.trash`, plus lock + fence); trash draining is essentially equivalent (no per-entry sleeper throttling; the 5m cycle naturally rate-limits); tmp non-trash 24h reclamation is equivalent (RustFS's 5m is more timely than MinIO's 6h); the three cycle defaults 24h/6h/5m all align. The item §8 listed as "not verified line by line" now has a conclusion. + +### 9.4 Handed over to follow-ups (summarized in the backlog#1862 comment thread) + +HS-01 bitrot GET→MRF full-chain e2e, kill -9 journal replay e2e, queue-full RSS stress test (≤ budget+10%); HS-05/06 madmin incremental-consumption methods + single-source wire + embedded e2e + multi-round polling soak; HS-08 multi-drive scanner cycle e2e; HS-04 excess audit entries; HS-18 the stale-multipart crash-residue window below quorum (crashing mid-fan-out with already-cleaned drives > parity means FileNotFound is not in the ignore set, so convergence is unnatural; the fix needs a dedicated quorum variant). + +Recommendation for the next re-audit: trigger it after the next big heal/scanner feature lands, using this section as the incremental baseline. From 9fed675185ae7f0e9d67d54c82e90c0171b0b982 Mon Sep 17 00:00:00 2001 From: hector <42570491+majinghe@users.noreply.github.com> Date: Wed, 19 Aug 2026 21:01:53 +0800 Subject: [PATCH 23/27] feat(helm): add istio gateway class support (#6264) --- helm/README.md | 4 ++-- .../gateway-api/istio-destinationrule.yaml | 16 ++++++++++++++++ helm/rustfs/values.yaml | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 helm/rustfs/templates/gateway-api/istio-destinationrule.yaml diff --git a/helm/README.md b/helm/README.md index 79b7c6fd6..c246a3bac 100644 --- a/helm/README.md +++ b/helm/README.md @@ -267,7 +267,7 @@ uer. `ClusterIssuer` or `Issuer`. | | topologySpreadConstraints.enabled | bool | `false` | Enable custom topology spread constraints on distributed-mode StatefulSet pods. | | topologySpreadConstraints.constraints | list | `[]` | Raw `spec.template.spec.topologySpreadConstraints` entries applied to the distributed StatefulSet when enabled. | | gatewayApi.enabled | bool | `false` | To enable/disable gateway api support. | -| gatewayApi.gatewayClass | string | `traefik` | Gateway class implementation. | +| gatewayApi.gatewayClass | string | `traefik` | Gateway class implementation (traefik, contour, istio). | | gatewayApi.httpToHttpsRedirect | bool | `true` | To enable/disable the redirect httproute. | | gatewayApi.listeners.http.name | string | `web` | Gateway API http listener name. | | gatewayApi.listeners.http.port| int | `8000` | Gateway API http listener port. | @@ -437,7 +437,7 @@ helm install rustfs rustfs/rustfs -n rustfs --set tls.enabled=true,--set-file tl # Gateway API support (alpha) -Due to [ingress nginx retirement](https://kubernetes.io/blog/2025/11/11/ingress-nginx-retirement/) in March 2026, so RustFS adds support for [gateway api](https://gateway-api.sigs.k8s.io/). Currently, RustFS only supports traefik as gateway class, more and more gateway class support will be added in the future after those classes are tested. If you want to enable gateway api, specify `gatewayApi.enabled` to `true` while specify `ingress.enabled` to `false`. After installation, you can find the `Gateway` and `HttpRoute` resources, +Due to [ingress nginx retirement](https://kubernetes.io/blog/2025/11/11/ingress-nginx-retirement/) in March 2026, so RustFS adds support for [gateway api](https://gateway-api.sigs.k8s.io/). Currently, RustFS supports traefik, contour, and istio as gateway classes. If you want to enable gateway api, specify `gatewayApi.enabled` to `true` while specify `ingress.enabled` to `false`. After installation, you can find the `Gateway` and `HttpRoute` resources, ``` $ kubectl -n rustfs get gateway diff --git a/helm/rustfs/templates/gateway-api/istio-destinationrule.yaml b/helm/rustfs/templates/gateway-api/istio-destinationrule.yaml new file mode 100644 index 000000000..3c4b5cd43 --- /dev/null +++ b/helm/rustfs/templates/gateway-api/istio-destinationrule.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.gatewayApi.enabled (eq .Values.gatewayApi.gatewayClass "istio") }} +apiVersion: networking.istio.io/v1 +kind: DestinationRule +metadata: + name: {{ include "rustfs.fullname" . }}-sticky + namespace: {{ .Release.Namespace }} +spec: + host: {{ include "rustfs.fullname" . }}-svc.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }} + trafficPolicy: + loadBalancer: + consistentHash: + httpCookie: + name: rustfs + path: / + ttl: 0s +{{- end }} diff --git a/helm/rustfs/values.yaml b/helm/rustfs/values.yaml index d57317243..365a1a57c 100644 --- a/helm/rustfs/values.yaml +++ b/helm/rustfs/values.yaml @@ -361,7 +361,7 @@ ingress: gatewayApi: enabled: false - gatewayClass: traefik # Only support for traefik and contour gatewayClass at the moment. + gatewayClass: traefik # Supported gatewayClass: traefik, contour, istio. listeners: # Specify which listeners to create on the Gateway. http: name: web From 24cfce12edd296226efcce72b465c551247f44ca Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Wed, 19 Aug 2026 21:58:22 +0800 Subject: [PATCH 24/27] fix(metrics): remove duplicate scanner counter producers (#6245) * fix(metrics): remove duplicate scanner counter producers * chore(deps): centralize metrics test dependencies * ci: avoid apt mirror for ripgrep setup * test(protos): track read-version encoder refactor --------- Co-authored-by: Henry Guo Co-authored-by: houseme --- .../architecture-migration-rules.yml | 6 +-- .github/workflows/ci-docs-only.yml | 4 +- .github/workflows/ci.yml | 4 +- Cargo.lock | 1 + Cargo.toml | 2 + crates/common/src/metrics.rs | 20 -------- crates/io-metrics/Cargo.toml | 2 +- crates/kms/Cargo.toml | 2 +- crates/lifecycle/Cargo.toml | 2 +- crates/object-data-cache/Cargo.toml | 2 +- crates/obs/Cargo.toml | 3 +- crates/obs/src/metrics/collectors/scanner.rs | 51 +++++++++++++++++++ crates/protos/src/lib.rs | 2 +- rustfs/Cargo.toml | 2 +- 14 files changed, 71 insertions(+), 32 deletions(-) diff --git a/.github/workflows/architecture-migration-rules.yml b/.github/workflows/architecture-migration-rules.yml index 365168dc7..c9eabcea7 100644 --- a/.github/workflows/architecture-migration-rules.yml +++ b/.github/workflows/architecture-migration-rules.yml @@ -53,9 +53,9 @@ jobs: persist-credentials: false - name: Install ripgrep - run: | - sudo apt-get update - sudo apt-get install -y ripgrep + uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2 + with: + tool: ripgrep@15.2.0 - name: Check architecture migration rules run: ./scripts/check_architecture_migration_rules.sh diff --git a/.github/workflows/ci-docs-only.yml b/.github/workflows/ci-docs-only.yml index 5cd8ab733..89ae4d18f 100644 --- a/.github/workflows/ci-docs-only.yml +++ b/.github/workflows/ci-docs-only.yml @@ -83,7 +83,9 @@ jobs: persist-credentials: false - name: Install ripgrep - run: sudo apt-get update && sudo apt-get install -y ripgrep + uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2 + with: + tool: ripgrep@15.2.0 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23a37d82b..4812b118f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,9 @@ jobs: persist-credentials: false - name: Install ripgrep - run: sudo apt-get update && sudo apt-get install -y ripgrep + uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2 + with: + tool: ripgrep@15.2.0 - name: Install Rust toolchain uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable diff --git a/Cargo.lock b/Cargo.lock index c73ac8d64..e42b1732b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9933,6 +9933,7 @@ dependencies = [ "libc", "log", "metrics", + "metrics-util", "num_cpus", "nvml-wrapper", "opentelemetry", diff --git a/Cargo.toml b/Cargo.toml index b6a37b028..438472db2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -264,6 +264,7 @@ lazy_static = "1.5.0" libc = "0.2.189" libsystemd = "0.7.2" local-ip-address = "0.6.13" +log = "0.4" memmap2 = "0.9.11" lz4 = "1.28.1" matchit = "0.9.2" @@ -326,6 +327,7 @@ zstd = "0.13.3" # Observability and Metrics metrics = "0.24.6" +metrics-util = "0.20" dial9-tokio-telemetry = "0.3" opentelemetry = { version = "0.32.0" } opentelemetry-appender-tracing = { version = "0.32.0" } diff --git a/crates/common/src/metrics.rs b/crates/common/src/metrics.rs index 4c409279f..00521b55b 100644 --- a/crates/common/src/metrics.rs +++ b/crates/common/src/metrics.rs @@ -1401,25 +1401,11 @@ impl CurrentCycle { } /// OTEL metric name constants for scanner metrics -const OTEL_SCANNER_OBJECTS_SCANNED: &str = "rustfs_scanner_objects_scanned_total"; -const OTEL_SCANNER_DIRECTORIES_SCANNED: &str = "rustfs_scanner_directories_scanned_total"; const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total"; const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total"; const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds"; const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds"; -fn emit_otel_counter(metric: usize, count: u64) { - match Metric::from_index(metric) { - Some(Metric::ScanObject) => { - metrics::counter!(OTEL_SCANNER_OBJECTS_SCANNED).increment(count); - } - Some(Metric::ScanFolder) => { - metrics::counter!(OTEL_SCANNER_DIRECTORIES_SCANNED).increment(count); - } - _ => {} - } -} - fn scan_cycle_result_label(result: u8) -> &'static str { match result { SCAN_CYCLE_RESULT_SUCCESS => SCAN_CYCLE_RESULT_SUCCESS_LABEL, @@ -1960,7 +1946,6 @@ impl Metrics { let duration = SystemTime::now().duration_since(start).unwrap_or_default(); global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed); global_metrics().record_source_work_for_metric(metric, 1); - emit_otel_counter(metric_idx, 1); if metric_idx < Metric::LastRealtime as usize { global_metrics().latency[metric_idx].add(duration); } @@ -1976,7 +1961,6 @@ impl Metrics { let duration = SystemTime::now().duration_since(start).unwrap_or_default(); global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed); global_metrics().record_source_work_for_metric(metric, 1); - emit_otel_counter(metric_idx, 1); if metric_idx < Metric::LastRealtime as usize { global_metrics().latency[metric_idx].add_size(duration, size); } @@ -1992,7 +1976,6 @@ impl Metrics { let duration = SystemTime::now().duration_since(start).unwrap_or_default(); global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed); global_metrics().record_source_work_for_metric(metric, 1); - emit_otel_counter(metric_idx, 1); if metric_idx < Metric::LastRealtime as usize { global_metrics().latency[metric_idx].add(duration); } @@ -2010,7 +1993,6 @@ impl Metrics { let count = usize_to_u64_saturated(count); global_metrics().operations[metric_idx].fetch_add(count, Ordering::Relaxed); global_metrics().record_source_work_for_metric(metric, count); - emit_otel_counter(metric_idx, count); if metric_idx < Metric::LastRealtime as usize { global_metrics().latency[metric_idx].add(duration); } @@ -2031,7 +2013,6 @@ impl Metrics { let duration = SystemTime::now().duration_since(start).unwrap_or_default(); let metric_idx = Metric::Ilm as usize; global_metrics().operations[metric_idx].fetch_add(versions, Ordering::Relaxed); - emit_otel_counter(metric_idx, versions); global_metrics().actions[a_idx].fetch_add(versions, Ordering::Relaxed); global_metrics().actions_latency[a_idx].add(duration); }) @@ -2044,7 +2025,6 @@ impl Metrics { let metric_idx = metric as usize; global_metrics().operations[metric_idx].fetch_add(1, Ordering::Relaxed); global_metrics().record_source_work_for_metric(metric, 1); - emit_otel_counter(metric_idx, 1); if metric_idx < Metric::LastRealtime as usize { global_metrics().latency[metric_idx].add(duration); } diff --git a/crates/io-metrics/Cargo.toml b/crates/io-metrics/Cargo.toml index 6124de8ca..0591aa4e7 100644 --- a/crates/io-metrics/Cargo.toml +++ b/crates/io-metrics/Cargo.toml @@ -58,7 +58,7 @@ sysinfo = { workspace = true } [dev-dependencies] criterion = { workspace = true, features = ["html_reports"] } -metrics-util = { version = "0.20", features = ["debugging"] } +metrics-util = { workspace = true, features = ["debugging"] } tokio = { workspace = true, features = ["test-util", "macros", "fs", "rt-multi-thread"] } [lints] diff --git a/crates/kms/Cargo.toml b/crates/kms/Cargo.toml index 2883667c1..f1a41c888 100644 --- a/crates/kms/Cargo.toml +++ b/crates/kms/Cargo.toml @@ -94,7 +94,7 @@ aws-smithy-types = { workspace = true } [dev-dependencies] anyhow = { workspace = true } # Debugging recorder for asserting emitted metrics in tests. -metrics-util = { version = "0.20", features = ["debugging"] } +metrics-util = { workspace = true, features = ["debugging"] } insta = { workspace = true, features = ["yaml", "json"] } tempfile = { workspace = true } temp-env = { workspace = true } diff --git a/crates/lifecycle/Cargo.toml b/crates/lifecycle/Cargo.toml index 8f5f63556..ae8232f20 100644 --- a/crates/lifecycle/Cargo.toml +++ b/crates/lifecycle/Cargo.toml @@ -67,7 +67,7 @@ url.workspace = true uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] } [dev-dependencies] -metrics-util = { version = "0.20", features = ["debugging"] } +metrics-util = { workspace = true, features = ["debugging"] } proptest = "1" serial_test.workspace = true temp-env.workspace = true diff --git a/crates/object-data-cache/Cargo.toml b/crates/object-data-cache/Cargo.toml index 704f3df6f..152b73992 100644 --- a/crates/object-data-cache/Cargo.toml +++ b/crates/object-data-cache/Cargo.toml @@ -58,7 +58,7 @@ tracing = { workspace = true, optional = true } [dev-dependencies] criterion = { workspace = true, features = ["html_reports"] } -metrics-util = { version = "0.20", features = ["debugging"] } +metrics-util = { workspace = true, features = ["debugging"] } # `rt-multi-thread` lets the concurrency stress tests run tasks on real worker # threads, so they exercise true parallelism on the shared singleflight/index # state rather than only cooperative interleaving. diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index eeec4ee02..17ba113ed 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -163,4 +163,5 @@ libc = { workspace = true } [dev-dependencies] tempfile = { workspace = true } temp-env = { workspace = true } -log = "0.4" +log.workspace = true +metrics-util = { workspace = true, features = ["debugging"] } diff --git a/crates/obs/src/metrics/collectors/scanner.rs b/crates/obs/src/metrics/collectors/scanner.rs index db220ca96..56f1db556 100644 --- a/crates/obs/src/metrics/collectors/scanner.rs +++ b/crates/obs/src/metrics/collectors/scanner.rs @@ -465,6 +465,57 @@ fn bool_metric_value(enabled: bool) -> f64 { mod tests { use super::*; use crate::metrics::report::report_metrics; + use metrics_util::debugging::DebuggingRecorder; + use rustfs_common::metrics::{Metric, Metrics}; + + fn prometheus_counter_name(name: &str) -> String { + if name.ends_with("_total") { + name.to_string() + } else { + format!("{name}_total") + } + } + + #[test] + fn scanner_lifetime_counters_have_one_prometheus_producer() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let scanner_metrics = collect_scanner_metrics(&ScannerStats { + directories_scanned: 3, + objects_scanned: 7, + ..Default::default() + }); + + metrics::with_local_recorder(&recorder, || { + Metrics::time(Metric::ScanObject)(); + Metrics::time(Metric::ScanFolder)(); + report_metrics(&scanner_metrics); + }); + + let normalized_counter_names: Vec<_> = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter_map(|(composite, _, _, value)| { + matches!(value, metrics_util::debugging::DebugValue::Counter(_)) + .then(|| prometheus_counter_name(composite.key().name())) + }) + .collect(); + + for name in [ + "rustfs_scanner_objects_scanned_total", + "rustfs_scanner_directories_scanned_total", + ] { + assert_eq!( + normalized_counter_names + .iter() + .filter(|candidate| candidate.as_str() == name) + .count(), + 1, + "scanner lifetime counter must have exactly one producer after Prometheus name normalization" + ); + } + } #[test] fn test_collect_scanner_metrics() { diff --git a/crates/protos/src/lib.rs b/crates/protos/src/lib.rs index d23224b93..a9e59b8a0 100644 --- a/crates/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -2433,7 +2433,7 @@ mod tests { json_field: "opts", bin_field: "opts_bin", }, - json_encoder: "let opts_str = compat_json(opts)?;", + json_encoder: "let encoded_opts = compat_json(opts).and_then(|opts_str| encode_msgpack(opts).map(|opts_bin| (opts_str, opts_bin)));", policy: RequestJsonPolicy::MsgpackOnlyEligible, }, RequestCompatSendSite { diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 69916207e..2e1952c34 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -350,7 +350,7 @@ proptest = "1" tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] } temp-env = { workspace = true, features = ["async_closure"] } tracing-subscriber = { workspace = true, features = ["env-filter", "time"] } -metrics-util = { version = "0.20", features = ["debugging"] } +metrics-util = { workspace = true, features = ["debugging"] } opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] } rsa = { workspace = true } rcgen = { workspace = true } From 5a1b0fe9df44ebee09ab5af1609993d4143fda76 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 19 Aug 2026 22:04:25 +0800 Subject: [PATCH 25/27] docs(admin): pin two authorization semantics against a dedup rewrite (#6279) --- rustfs/src/admin/handlers/is_admin.rs | 6 ++++++ rustfs/src/admin/handlers/service_account.rs | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/rustfs/src/admin/handlers/is_admin.rs b/rustfs/src/admin/handlers/is_admin.rs index 4bbc1026a..7e29c4bff 100644 --- a/rustfs/src/admin/handlers/is_admin.rs +++ b/rustfs/src/admin/handlers/is_admin.rs @@ -46,6 +46,12 @@ impl Operation for IsAdminHandler { let access_key_to_check = input_cred.access_key.clone(); + // This endpoint reports a capability; it does not gate on one. The + // `is_allowed` result below becomes the `is_admin` field of a 200 + // response — a caller without admin rights gets `{"is_admin": false}`, + // not a 403. Turning this into a rejection would change the API + // contract, so it must stay out of any shared-gate normalisation + // (backlog#1886). // Check if the user is admin: root user check, then evaluate through the policy engine let is_admin = if let Some(sys_cred) = current_action_credentials() { constant_time_eq(&access_key_to_check, &sys_cred.access_key) diff --git a/rustfs/src/admin/handlers/service_account.rs b/rustfs/src/admin/handlers/service_account.rs index ac5075068..938ba7836 100644 --- a/rustfs/src/admin/handlers/service_account.rs +++ b/rustfs/src/admin/handlers/service_account.rs @@ -359,6 +359,13 @@ impl Operation for AddServiceAccount { return Err(s3_error!(InvalidRequest, "iam not init")); }; + // This family deliberately calls `is_allowed` directly instead of going + // through `validate_admin_request`, and must keep doing so + // (backlog#1886). The shared helper returns as soon as *any* candidate + // action is allowed — an OR. The checks here are an AND: each one must + // pass, and a later stage additionally needs `owner` for the GHSA-5354 + // parent-scope guard and drives `deny_only` dynamically. Replacing these + // with the shared gate would widen authorization. if !iam_store .is_allowed(&Args { account: &cred.access_key, From 3bde70d5b414383c79258d1651a57212bec4e593 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 19 Aug 2026 22:38:49 +0800 Subject: [PATCH 26/27] chore(protocols): narrow the SessionDiag blanket to its one unread field (#6266) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last item-level bare allow of backlog#1823 step 10. `SessionDiag` itself is live — `sftp/server.rs` constructs one per accepted connection and `wedge_watchdog` reads `session_id`, `peer` and `last_activity_ms` off it — so the struct-level blanket was covering exactly one field: `accepted_at`, which is written at accept time and never read back. The allow moves onto that field with a reason. The three remaining `#![allow(dead_code)]` in this crate (`sftp/test_support.rs`, `common/dummy_storage.rs`) are module-root blankets in test-support files, which belong to steps 1-5 rather than step 10. Refs backlog#1823 --- crates/protocols/src/sftp/lifecycle.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/protocols/src/sftp/lifecycle.rs b/crates/protocols/src/sftp/lifecycle.rs index 4bd3f1aad..64c05a950 100644 --- a/crates/protocols/src/sftp/lifecycle.rs +++ b/crates/protocols/src/sftp/lifecycle.rs @@ -84,11 +84,11 @@ const TCP_STATE_RADIX: u32 = 16; /// and the SftpDriver, registered weakly into the SessionRegistry so an /// outside observer can enumerate live sessions without holding their /// lifetime. -#[allow(dead_code)] pub struct SessionDiag { pub session_id: u64, pub local: SocketAddr, pub peer: SocketAddr, + #[allow(dead_code, reason = "written at accept time but never read back (backlog#1823)")] pub accepted_at: Instant, pub last_activity_ms: AtomicU64, } From be7f684718976c62bf3bc845778d9828aa670737 Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 19 Aug 2026 22:52:36 +0800 Subject: [PATCH 27/27] refactor(heal): remove the dead MRF heal-type path (#6275) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HealType::MRF (a #1664-era "metadata repair file" task kind) had no production construction site left: its only builder lived in the HealEvent -> HealRequest converter, and the HealEvent/HealEventHandler queue itself had zero production references — both were superseded by the MrfIntent pipeline (mrf_queue.rs), which produces Object/Metadata/ ECDecode requests and never an MRF task. The dead path nevertheless carried ~700 lines: the whole event.rs module, the heal_mrf executor, a dedup-key arm, an overlap arm with the "\u{0}mrf" sentinel bucket hack, per-kind labels, and an empty MrfRuntime::record_accept shell. Deleting the variant is compile-time safe: HealType has no Serialize derive, the protos wire enums carry no heal-type discriminant (the receiver rebuilds it from HealChannelRequest fields), the MRF journal encodes MrfKind (1/2/3), and the scanner pending-heal ledger uses its own kind enum — none of them can name an MRF task. Also resolves the in-crate naming clash where "MRF" denoted both the dead task kind and the live mission-repair-feed loop; the loop stays, the task kind goes. Co-authored-by: heihutu --- crates/heal/src/heal/event.rs | 683 ----------------------- crates/heal/src/heal/manager.rs | 129 ++++- crates/heal/src/heal/mod.rs | 1 - crates/heal/src/heal/mrf_queue.rs | 136 ++++- crates/heal/src/heal/task.rs | 157 +----- crates/heal/tests/heal_bug_fixes_test.rs | 82 +-- 6 files changed, 223 insertions(+), 965 deletions(-) delete mode 100644 crates/heal/src/heal/event.rs diff --git a/crates/heal/src/heal/event.rs b/crates/heal/src/heal/event.rs deleted file mode 100644 index 0d46b859b..000000000 --- a/crates/heal/src/heal/event.rs +++ /dev/null @@ -1,683 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::heal::{HealOptions, HealPriority, HealRequest, HealType}; -use crate::{Error, Result}; -use serde::{Deserialize, Serialize}; -use std::time::SystemTime; - -use super::Endpoint; - -/// Corruption type -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum CorruptionType { - /// Data corruption - DataCorruption, - /// Metadata corruption - MetadataCorruption, - /// Partial corruption - PartialCorruption, - /// Complete corruption - CompleteCorruption, -} - -/// Severity level -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -pub enum Severity { - /// Low severity - Low = 0, - /// Medium severity - Medium = 1, - /// High severity - High = 2, - /// Critical severity - Critical = 3, -} - -/// Heal event -#[derive(Debug, Clone)] -pub enum HealEvent { - /// Object corruption event - ObjectCorruption { - bucket: String, - object: String, - version_id: Option, - corruption_type: CorruptionType, - severity: Severity, - }, - /// Object missing event - ObjectMissing { - bucket: String, - object: String, - version_id: Option, - expected_locations: Vec, - available_locations: Vec, - }, - /// Metadata corruption event - MetadataCorruption { - bucket: String, - object: String, - corruption_type: CorruptionType, - }, - /// Disk status change event - DiskStatusChange { - endpoint: Endpoint, - old_status: String, - new_status: String, - }, - /// EC decode failure event - ECDecodeFailure { - bucket: String, - object: String, - version_id: Option, - missing_shards: Vec, - available_shards: Vec, - }, - /// Checksum mismatch event - ChecksumMismatch { - bucket: String, - object: String, - version_id: Option, - expected_checksum: String, - actual_checksum: String, - }, - /// Bucket metadata corruption event - BucketMetadataCorruption { - bucket: String, - corruption_type: CorruptionType, - }, - /// MRF metadata corruption event - MRFMetadataCorruption { - meta_path: String, - corruption_type: CorruptionType, - }, -} - -impl HealEvent { - /// Convert HealEvent to HealRequest - pub fn to_heal_request(&self) -> Result { - match self { - HealEvent::ObjectCorruption { - bucket, - object, - version_id, - severity, - .. - } => Ok(HealRequest::new( - HealType::Object { - bucket: bucket.clone(), - object: object.clone(), - version_id: version_id.clone(), - }, - HealOptions::default(), - Self::severity_to_priority(severity), - )), - HealEvent::ObjectMissing { - bucket, - object, - version_id, - .. - } => Ok(HealRequest::new( - HealType::Object { - bucket: bucket.clone(), - object: object.clone(), - version_id: version_id.clone(), - }, - HealOptions::default(), - HealPriority::High, - )), - HealEvent::MetadataCorruption { bucket, object, .. } => Ok(HealRequest::new( - HealType::Metadata { - bucket: bucket.clone(), - object: object.clone(), - }, - HealOptions::default(), - HealPriority::High, - )), - HealEvent::DiskStatusChange { endpoint, .. } => { - // Convert disk status change to erasure set heal - // Note: This requires access to storage to get bucket list, which is not available here - // The actual bucket list will need to be provided by the caller or retrieved differently - let set_disk_id = crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx) - .ok_or_else(|| Error::InvalidHealType { - heal_type: format!("erasure-set(pool={}, set={})", endpoint.pool_idx, endpoint.set_idx), - })?; - Ok(HealRequest::new( - HealType::ErasureSet { - buckets: vec![], // Empty bucket list - caller should populate this - set_disk_id, - }, - HealOptions::default(), - HealPriority::High, - )) - } - HealEvent::ECDecodeFailure { - bucket, - object, - version_id, - .. - } => Ok(HealRequest::new( - HealType::ECDecode { - bucket: bucket.clone(), - object: object.clone(), - version_id: version_id.clone(), - }, - HealOptions::default(), - HealPriority::Urgent, - )), - HealEvent::ChecksumMismatch { - bucket, - object, - version_id, - .. - } => Ok(HealRequest::new( - HealType::Object { - bucket: bucket.clone(), - object: object.clone(), - version_id: version_id.clone(), - }, - HealOptions::default(), - HealPriority::High, - )), - HealEvent::BucketMetadataCorruption { bucket, .. } => Ok(HealRequest::new( - HealType::Bucket { bucket: bucket.clone() }, - HealOptions::default(), - HealPriority::High, - )), - HealEvent::MRFMetadataCorruption { meta_path, .. } => Ok(HealRequest::new( - HealType::MRF { - meta_path: meta_path.clone(), - }, - HealOptions::default(), - HealPriority::High, - )), - } - } - - /// Convert severity to priority - fn severity_to_priority(severity: &Severity) -> HealPriority { - match severity { - Severity::Low => HealPriority::Low, - Severity::Medium => HealPriority::Normal, - Severity::High => HealPriority::High, - Severity::Critical => HealPriority::Urgent, - } - } - - /// Get event description - pub fn description(&self) -> String { - match self { - HealEvent::ObjectCorruption { - bucket, - object, - corruption_type, - .. - } => { - format!("Object corruption detected: {bucket}/{object} - {corruption_type:?}") - } - HealEvent::ObjectMissing { bucket, object, .. } => { - format!("Object missing: {bucket}/{object}") - } - HealEvent::MetadataCorruption { - bucket, - object, - corruption_type, - .. - } => { - format!("Metadata corruption: {bucket}/{object} - {corruption_type:?}") - } - HealEvent::DiskStatusChange { - endpoint, - old_status, - new_status, - .. - } => { - format!("Disk status changed: {endpoint:?} {old_status} -> {new_status}") - } - HealEvent::ECDecodeFailure { - bucket, - object, - missing_shards, - .. - } => { - format!("EC decode failure: {bucket}/{object} - missing shards: {missing_shards:?}") - } - HealEvent::ChecksumMismatch { - bucket, - object, - expected_checksum, - actual_checksum, - .. - } => { - format!("Checksum mismatch: {bucket}/{object} - expected: {expected_checksum}, actual: {actual_checksum}") - } - HealEvent::BucketMetadataCorruption { - bucket, corruption_type, .. - } => { - format!("Bucket metadata corruption: {bucket} - {corruption_type:?}") - } - HealEvent::MRFMetadataCorruption { - meta_path, - corruption_type, - .. - } => { - format!("MRF metadata corruption: {meta_path} - {corruption_type:?}") - } - } - } - - /// Get event severity - pub fn severity(&self) -> Severity { - match self { - HealEvent::ObjectCorruption { severity, .. } => severity.clone(), - HealEvent::ObjectMissing { .. } => Severity::High, - HealEvent::MetadataCorruption { .. } => Severity::High, - HealEvent::DiskStatusChange { .. } => Severity::High, - HealEvent::ECDecodeFailure { .. } => Severity::Critical, - HealEvent::ChecksumMismatch { .. } => Severity::High, - HealEvent::BucketMetadataCorruption { .. } => Severity::High, - HealEvent::MRFMetadataCorruption { .. } => Severity::High, - } - } - - /// Get event timestamp - pub fn timestamp(&self) -> SystemTime { - SystemTime::now() - } -} - -/// Heal event handler -pub struct HealEventHandler { - /// Event queue - events: Vec, - /// Maximum number of events - max_events: usize, -} - -impl HealEventHandler { - pub fn new(max_events: usize) -> Self { - Self { - events: Vec::new(), - max_events, - } - } - - /// Add event - pub fn add_event(&mut self, event: HealEvent) { - if self.events.len() >= self.max_events { - // Remove oldest event - self.events.remove(0); - } - self.events.push(event); - } - - /// Get all events - pub fn get_events(&self) -> &[HealEvent] { - &self.events - } - - /// Clear events - pub fn clear_events(&mut self) { - self.events.clear(); - } - - /// Get event count - pub fn event_count(&self) -> usize { - self.events.len() - } - - /// Filter events by severity - pub fn filter_by_severity(&self, min_severity: Severity) -> Vec<&HealEvent> { - self.events.iter().filter(|event| event.severity() >= min_severity).collect() - } - - /// Filter events by type - pub fn filter_by_type(&self, event_type: &str) -> Vec<&HealEvent> { - self.events - .iter() - .filter(|event| match event { - HealEvent::ObjectCorruption { .. } => event_type == "ObjectCorruption", - HealEvent::ObjectMissing { .. } => event_type == "ObjectMissing", - HealEvent::MetadataCorruption { .. } => event_type == "MetadataCorruption", - HealEvent::DiskStatusChange { .. } => event_type == "DiskStatusChange", - HealEvent::ECDecodeFailure { .. } => event_type == "ECDecodeFailure", - HealEvent::ChecksumMismatch { .. } => event_type == "ChecksumMismatch", - HealEvent::BucketMetadataCorruption { .. } => event_type == "BucketMetadataCorruption", - HealEvent::MRFMetadataCorruption { .. } => event_type == "MRFMetadataCorruption", - }) - .collect() - } -} - -impl Default for HealEventHandler { - fn default() -> Self { - Self::new(1000) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::heal::task::{HealPriority, HealType}; - - #[test] - fn test_heal_event_object_corruption_to_request() { - let event = HealEvent::ObjectCorruption { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }; - - let request = event.to_heal_request().unwrap(); - assert!(matches!(request.heal_type, HealType::Object { .. })); - assert_eq!(request.priority, HealPriority::High); - } - - #[test] - fn test_heal_event_object_missing_to_request() { - let event = HealEvent::ObjectMissing { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - version_id: Some("v1".to_string()), - expected_locations: vec![0, 1], - available_locations: vec![2, 3], - }; - - let request = event.to_heal_request().unwrap(); - assert!(matches!(request.heal_type, HealType::Object { .. })); - assert_eq!(request.priority, HealPriority::High); - } - - #[test] - fn test_heal_event_metadata_corruption_to_request() { - let event = HealEvent::MetadataCorruption { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - corruption_type: CorruptionType::MetadataCorruption, - }; - - let request = event.to_heal_request().unwrap(); - assert!(matches!(request.heal_type, HealType::Metadata { .. })); - assert_eq!(request.priority, HealPriority::High); - } - - #[test] - fn test_heal_event_ec_decode_failure_to_request() { - let event = HealEvent::ECDecodeFailure { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - version_id: None, - missing_shards: vec![0, 1], - available_shards: vec![2, 3, 4], - }; - - let request = event.to_heal_request().unwrap(); - assert!(matches!(request.heal_type, HealType::ECDecode { .. })); - assert_eq!(request.priority, HealPriority::Urgent); - } - - #[test] - fn test_heal_event_checksum_mismatch_to_request() { - let event = HealEvent::ChecksumMismatch { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - version_id: None, - expected_checksum: "abc123".to_string(), - actual_checksum: "def456".to_string(), - }; - - let request = event.to_heal_request().unwrap(); - assert!(matches!(request.heal_type, HealType::Object { .. })); - assert_eq!(request.priority, HealPriority::High); - } - - #[test] - fn test_heal_event_bucket_metadata_corruption_to_request() { - let event = HealEvent::BucketMetadataCorruption { - bucket: "test-bucket".to_string(), - corruption_type: CorruptionType::MetadataCorruption, - }; - - let request = event.to_heal_request().unwrap(); - assert!(matches!(request.heal_type, HealType::Bucket { .. })); - assert_eq!(request.priority, HealPriority::High); - } - - #[test] - fn test_heal_event_mrf_metadata_corruption_to_request() { - let event = HealEvent::MRFMetadataCorruption { - meta_path: "test-bucket/test-object".to_string(), - corruption_type: CorruptionType::MetadataCorruption, - }; - - let request = event.to_heal_request().unwrap(); - assert!(matches!(request.heal_type, HealType::MRF { .. })); - assert_eq!(request.priority, HealPriority::High); - } - - #[test] - fn test_heal_event_severity_to_priority() { - let event_low = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::Low, - }; - let request = event_low.to_heal_request().unwrap(); - assert_eq!(request.priority, HealPriority::Low); - - let event_medium = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::Medium, - }; - let request = event_medium.to_heal_request().unwrap(); - assert_eq!(request.priority, HealPriority::Normal); - - let event_high = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }; - let request = event_high.to_heal_request().unwrap(); - assert_eq!(request.priority, HealPriority::High); - - let event_critical = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::Critical, - }; - let request = event_critical.to_heal_request().unwrap(); - assert_eq!(request.priority, HealPriority::Urgent); - } - - #[test] - fn test_heal_event_description() { - let event = HealEvent::ObjectCorruption { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }; - - let desc = event.description(); - assert!(desc.contains("Object corruption detected")); - assert!(desc.contains("test-bucket/test-object")); - assert!(desc.contains("DataCorruption")); - } - - #[test] - fn test_heal_event_severity() { - let event = HealEvent::ECDecodeFailure { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - missing_shards: vec![], - available_shards: vec![], - }; - assert_eq!(event.severity(), Severity::Critical); - - let event = HealEvent::ObjectMissing { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - expected_locations: vec![], - available_locations: vec![], - }; - assert_eq!(event.severity(), Severity::High); - } - - #[test] - fn test_heal_event_handler_new() { - let handler = HealEventHandler::new(10); - assert_eq!(handler.event_count(), 0); - assert_eq!(handler.max_events, 10); - } - - #[test] - fn test_heal_event_handler_default() { - let handler = HealEventHandler::default(); - assert_eq!(handler.max_events, 1000); - } - - #[test] - fn test_heal_event_handler_add_event() { - let mut handler = HealEventHandler::new(3); - let event = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }; - - handler.add_event(event.clone()); - assert_eq!(handler.event_count(), 1); - - handler.add_event(event.clone()); - handler.add_event(event); - assert_eq!(handler.event_count(), 3); - } - - #[test] - fn test_heal_event_handler_max_events() { - let mut handler = HealEventHandler::new(2); - let event = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }; - - handler.add_event(event.clone()); - handler.add_event(event.clone()); - handler.add_event(event); // Should remove oldest - - assert_eq!(handler.event_count(), 2); - } - - #[test] - fn test_heal_event_handler_get_events() { - let mut handler = HealEventHandler::new(10); - let event = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }; - - handler.add_event(event.clone()); - handler.add_event(event); - - let events = handler.get_events(); - assert_eq!(events.len(), 2); - } - - #[test] - fn test_heal_event_handler_clear_events() { - let mut handler = HealEventHandler::new(10); - let event = HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }; - - handler.add_event(event); - assert_eq!(handler.event_count(), 1); - - handler.clear_events(); - assert_eq!(handler.event_count(), 0); - } - - #[test] - fn test_heal_event_handler_filter_by_severity() { - let mut handler = HealEventHandler::new(10); - handler.add_event(HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::Low, - }); - handler.add_event(HealEvent::ECDecodeFailure { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - missing_shards: vec![], - available_shards: vec![], - }); - - let high_severity = handler.filter_by_severity(Severity::High); - assert_eq!(high_severity.len(), 1); // Only ECDecodeFailure is Critical >= High - } - - #[test] - fn test_heal_event_handler_filter_by_type() { - let mut handler = HealEventHandler::new(10); - handler.add_event(HealEvent::ObjectCorruption { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - corruption_type: CorruptionType::DataCorruption, - severity: Severity::High, - }); - handler.add_event(HealEvent::ObjectMissing { - bucket: "test".to_string(), - object: "test".to_string(), - version_id: None, - expected_locations: vec![], - available_locations: vec![], - }); - - let corruption_events = handler.filter_by_type("ObjectCorruption"); - assert_eq!(corruption_events.len(), 1); - - let missing_events = handler.filter_by_type("ObjectMissing"); - assert_eq!(missing_events.len(), 1); - } -} diff --git a/crates/heal/src/heal/manager.rs b/crates/heal/src/heal/manager.rs index 4c3c622c7..2f03b30d4 100644 --- a/crates/heal/src/heal/manager.rs +++ b/crates/heal/src/heal/manager.rs @@ -86,7 +86,7 @@ fn unblock_replacement_recovery_sets_after_validation( } } -// Admission/scheduler outcomes for per-object requests (Object/Metadata/MRF/ +// Admission/scheduler outcomes for per-object requests (Object/Metadata/ // ECDecode) log via demote_to_debug_when! — MRF, autoheal, and scanner // recovery loops submit those per object, so a full queue or a retry storm // would otherwise emit one warn! per object (rustfs/rustfs#5716). The @@ -143,6 +143,16 @@ async fn pause_duplicate_admission_after_active_lock(request_id: &str) { type WorkloadSnapshotProviderRef = Arc; +/// Per-key bookkeeping for the queued-request dedup index: how many queued +/// requests hold the key, and the id of the first request that opened it — +/// the O(1) stand-in for the former heap scan when a merge receipt needs to +/// name a queued representative. +#[derive(Debug)] +struct DedupKeyEntry { + refcount: usize, + representative_request_id: String, +} + /// Priority queue wrapper for heal requests /// Uses BinaryHeap for priority-based ordering while maintaining FIFO for same-priority items #[derive(Debug)] @@ -151,8 +161,8 @@ struct PriorityHealQueue { heap: BinaryHeap, /// Sequence counter for FIFO ordering within same priority sequence: u64, - /// Deduplication key reference counts for queued requests - dedup_keys: HashMap, + /// Deduplication index for queued requests + dedup_keys: HashMap, } /// Wrapper for heap items to implement proper ordering @@ -402,8 +412,16 @@ impl PriorityHealQueue { return QueuePushOutcome::Merged; } // Track dedup keys for both normal and forced requests so queued forced work - // also reserves the dedup key for later non-forced duplicates. - *self.dedup_keys.entry(key).or_insert(0) += 1; + // also reserves the dedup key for later non-forced duplicates. The first + // request that opens the key becomes the named representative for merge + // receipts (taken before `request` moves into the heap). + self.dedup_keys + .entry(key) + .or_insert_with(|| DedupKeyEntry { + refcount: 0, + representative_request_id: request.id.clone(), + }) + .refcount += 1; self.sequence += 1; self.heap.push(PriorityQueueItem { priority: request.priority, @@ -447,6 +465,7 @@ impl PriorityHealQueue { let displaced = displaced.map(|item| { let key = Self::make_dedup_key(&item.request); Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); + self.refresh_dedup_representative(&key); item.request }); @@ -559,9 +578,6 @@ impl PriorityHealQueue { HealType::Metadata { bucket, object } => { format!("metadata:{bucket}:{object}") } - HealType::MRF { meta_path } => { - format!("mrf:{meta_path}") - } HealType::ECDecode { bucket, object, @@ -572,12 +588,12 @@ impl PriorityHealQueue { } } - fn decrement_or_remove_dedup_key(dedup_keys: &mut HashMap, key: &str) { - if let Some(count) = dedup_keys.get_mut(key) { - if *count <= 1 { + fn decrement_or_remove_dedup_key(dedup_keys: &mut HashMap, key: &str) { + if let Some(entry) = dedup_keys.get_mut(key) { + if entry.refcount <= 1 { dedup_keys.remove(key); } else { - *count -= 1; + entry.refcount -= 1; } } } @@ -610,10 +626,32 @@ impl PriorityHealQueue { .any(|item| item.request.id == request_id && heal_type_matches_path(&item.request.heal_type, heal_path)) } - fn request_for_dedup_key(&self, key: &str) -> Option<&HealRequest> { - self.heap + fn queued_request_id_for_dedup_key(&self, key: &str) -> Option<&str> { + self.dedup_keys.get(key).map(|entry| entry.representative_request_id.as_str()) + } + + /// Re-elect the representative for `key` from the queue entries holding + /// it. Needed after a holder leaves the queue *without* becoming active + /// (canceled by id, or displaced): the former opener may be the request + /// that just left, and a merge receipt must never name an id that + /// resolves nowhere. The scheduler pop path does not need this — the + /// popped request surfaces in `active_heals` under the same id and the + /// duplicate pre-check consults active heals before the queue. No-op for + /// released keys; the survivor scan only runs when a key still has + /// holders, which under forced duplicates is the rare admin path. + fn refresh_dedup_representative(&mut self, key: &str) { + if !self.dedup_keys.contains_key(key) { + return; + } + if let Some(id) = self + .heap .iter() - .find_map(|item| (Self::make_dedup_key(&item.request) == key).then_some(&item.request)) + .find(|item| Self::make_dedup_key(&item.request) == key) + .map(|item| item.request.id.clone()) + && let Some(entry) = self.dedup_keys.get_mut(key) + { + entry.representative_request_id = id; + } } fn contains_matching(&self, mut matches: F) -> bool @@ -638,6 +676,9 @@ impl PriorityHealQueue { } self.heap = retained; + if let Some(removed) = removed.as_ref() { + self.refresh_dedup_representative(&Self::make_dedup_key(removed)); + } removed } @@ -647,11 +688,13 @@ impl PriorityHealQueue { { let mut retained = BinaryHeap::new(); let mut removed = Vec::new(); + let mut affected_keys = Vec::new(); while let Some(item) = self.heap.pop() { if should_remove(&item.request) { let key = Self::make_dedup_key(&item.request); Self::decrement_or_remove_dedup_key(&mut self.dedup_keys, &key); + affected_keys.push(key); removed.push(item.request); } else { retained.push(item); @@ -659,6 +702,9 @@ impl PriorityHealQueue { } self.heap = retained; + for key in &affected_keys { + self.refresh_dedup_representative(key); + } removed } } @@ -686,7 +732,6 @@ fn heal_type_matches_path(heal_type: &HealType, heal_path: &str) -> bool { HealType::Bucket { bucket } => heal_path == bucket, HealType::Prefix { bucket, prefix } => heal_path_matches_bucket_child(heal_path, bucket, prefix), HealType::ErasureSet { set_disk_id, .. } => heal_path == set_disk_id, - HealType::MRF { meta_path } => heal_path == meta_path.trim_matches('/'), } } @@ -781,9 +826,6 @@ fn heal_type_path_view(heal_type: &HealType) -> (Option<&str>, &str) { HealType::Object { bucket, object, .. } | HealType::Metadata { bucket, object } | HealType::ECDecode { bucket, object, .. } => (Some(bucket), object), - // MRF/MetaPath heal keys on a meta path; treat the whole set of - // buckets as one namespace so it only overlaps itself exactly. - HealType::MRF { meta_path } => (Some("\u{0}mrf"), meta_path), // Erasure-set heal: the set id is the overlap dimension. HealType::ErasureSet { set_disk_id, .. } => (Some("\u{0}set"), set_disk_id), } @@ -1974,8 +2016,8 @@ impl HealManager { .map(|(task_id, _)| (task_id, "active")) .or_else(|| { queue - .request_for_dedup_key(&dedup_key) - .map(|queued| (queued.id.clone(), "queued")) + .queued_request_id_for_dedup_key(&dedup_key) + .map(|queued_id| (queued_id.to_string(), "queued")) }) .or_else(|| retrying_heal_for_dedup_key(&retrying_heals, &dedup_key).map(|(task_id, _)| (task_id, "retrying"))) }); @@ -2093,9 +2135,9 @@ impl HealManager { let mut task_id = request.id.clone(); let admission = Self::admit_request_to_queue(&mut queue, request, &config, "submit"); if admission == HealAdmissionResult::Merged - && let Some(queued) = queue.request_for_dedup_key(&dedup_key) + && let Some(queued_id) = queue.queued_request_id_for_dedup_key(&dedup_key) { - task_id.clone_from(&queued.id); + task_id = queued_id.to_owned(); } let should_notify = matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable; drop(retrying_heals); @@ -3712,7 +3754,6 @@ fn heal_request_type_label(request: &HealRequest) -> &'static str { HealType::Prefix { .. } => "prefix", HealType::ErasureSet { .. } => "erasure_set", HealType::Metadata { .. } => "metadata", - HealType::MRF { .. } => "mrf", HealType::ECDecode { .. } => "ec_decode", } } @@ -4104,6 +4145,46 @@ mod tests { assert_eq!(queue.len(), 0); } + #[test] + fn queued_request_id_for_dedup_key_tracks_the_representative() { + let mut queue = PriorityHealQueue::new(); + + let first = HealRequest::object("bucket".to_string(), "object".to_string(), None); + let first_id = first.id.clone(); + let first_key = PriorityHealQueue::make_dedup_key(&first); + assert_eq!(queue.push(first), QueuePushOutcome::Accepted); + + // A forced duplicate of the same target opens a second entry under + // the same key; the representative stays the request that opened it. + let mut second = HealRequest::object("bucket".to_string(), "object".to_string(), None); + second.force_start = true; + let second_id = second.id.clone(); + assert_eq!(queue.push(second), QueuePushOutcome::Accepted); + + let representative = queue + .queued_request_id_for_dedup_key(&first_key) + .expect("key must be reserved while either request is queued"); + assert_eq!(representative, first_id); + + // A holder leaving WITHOUT becoming active (canceled by id) must + // re-elect the representative to the surviving queued request, or a + // later merge receipt would name an id that resolves nowhere. The + // scheduler pop path needs no re-election: the popped request + // surfaces in active_heals under the same id and the duplicate + // pre-check consults active heals before the queue. + queue.remove_request_id(&first_id); + assert_eq!( + queue.queued_request_id_for_dedup_key(&first_key), + Some(second_id.as_str()), + "canceling the opener must re-elect the surviving queued holder" + ); + + // Pop the last holder: the key is released entirely. + let last = queue.pop_next().expect("second request must be queued"); + assert_eq!(last.id, second_id); + assert!(queue.queued_request_id_for_dedup_key(&first_key).is_none()); + } + #[test] fn test_priority_queue_ordering() { let mut queue = PriorityHealQueue::new(); diff --git a/crates/heal/src/heal/mod.rs b/crates/heal/src/heal/mod.rs index ff910ed70..35080e8e9 100644 --- a/crates/heal/src/heal/mod.rs +++ b/crates/heal/src/heal/mod.rs @@ -14,7 +14,6 @@ pub mod channel; pub mod erasure_healer; -pub mod event; pub mod manager; pub mod mrf_queue; pub mod progress; diff --git a/crates/heal/src/heal/mrf_queue.rs b/crates/heal/src/heal/mrf_queue.rs index c1144435e..5101a5e12 100644 --- a/crates/heal/src/heal/mrf_queue.rs +++ b/crates/heal/src/heal/mrf_queue.rs @@ -276,20 +276,26 @@ async fn read_journal() -> Option> { None } -async fn write_journal(data: &[u8]) { +/// Write the snapshot to every local disk; returns true when at least one +/// disk accepted it, so a total write failure keeps the runtime dirty and +/// the next tick retries the persist. +async fn write_journal(data: &[u8]) -> bool { let payload = bytes::Bytes::copy_from_slice(data); + let mut any_persisted = false; for disk in journal_disks().await { - if let Err(err) = disk + match disk .write_all(super::RUSTFS_META_BUCKET, MRF_JOURNAL_PATH, payload.clone()) .await { - warn_mrf_journal_write(&err); + Ok(()) => any_persisted = true, + Err(err) => warn_mrf_journal_write(&err), } } if !data.is_empty() { counter!("rustfs_heal_mrf_journal_fsync_total").increment(1); } gauge!("rustfs_heal_mrf_journal_bytes").set(data.len() as f64); + any_persisted } async fn delete_journal() { @@ -351,6 +357,12 @@ struct MrfRuntime { queue: MrfQueue, config: MrfConsumerConfig, new_since_flush: usize, + /// True while the in-memory pending set has changed since the last + /// journal flush (push, pop, or an attempts bump that alters the encoded + /// bytes). Only a dirty state rewrites the snapshot: a steady backlog + /// waiting out an admission backoff must not re-fsync every local disk + /// twice a second. + dirty: bool, /// True while a journal snapshot exists on disk that no longer reflects /// an all-consumed pending set; the next idle tick removes it (MinIO /// deletes its `list.bin` after replay for the same reason). @@ -360,11 +372,6 @@ struct MrfRuntime { } impl MrfRuntime { - fn record_accept(&mut self) { - // Accepted intents leave the pending set; the next flush persists the - // smaller snapshot, which is the journal's compaction. - } - fn snapshot(&self) -> Vec { let mut buf = Vec::new(); for intent in self.queue.intents() { @@ -374,8 +381,14 @@ impl MrfRuntime { } async fn flush(&mut self) { - write_journal(&self.snapshot()).await; + let persisted = write_journal(&self.snapshot()).await; self.new_since_flush = 0; + // Keep the dirty flag when every disk write failed: a clean backlog + // would otherwise never rewrite, losing the periodic persist retry a + // non-empty queue used to provide. + if persisted { + self.dirty = false; + } self.journal_on_disk = true; } @@ -389,9 +402,15 @@ impl MrfRuntime { self.backoff_until = None; } while let Some(mut intent) = self.queue.pop_front() { + // Leaving the pending set (consumed or re-queued with a bumped + // attempts counter) changes the encoded snapshot; mark it dirty + // either way. + self.dirty = true; let request = build_heal_request(&intent); match manager.submit_heal_request(request).await { - Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => self.record_accept(), + // Accepted intents leave the pending set; the next flush persists the + // smaller snapshot, which is the journal's compaction. + Ok(HealAdmissionResult::Accepted) | Ok(HealAdmissionResult::Merged) => {} Ok(HealAdmissionResult::Full) | Ok(HealAdmissionResult::Dropped(HealAdmissionDropReason::QueueFull)) => { intent.attempts = intent.attempts.saturating_add(1); if intent.attempts >= MRF_MAX_ATTEMPTS { @@ -519,6 +538,7 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive queue: MrfQueue::new(config.queue_capacity, config.journal_max_bytes), config: config.clone(), new_since_flush: 0, + dirty: false, journal_on_disk: false, backoff_until: None, }; @@ -526,6 +546,10 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive // Replay: read the journal, re-arm intents (duplicates are merged by the // manager's dedup key), then drop the file so the next flush starts clean. replay_into(&manager, &mut runtime.queue, &mut runtime.backoff_until).await; + // The replay deleted the journal file; anything still pending (e.g. the + // manager was full and backoff armed) must be re-persisted by the next + // flush or a crash before it would lose those intents. + runtime.dirty = runtime.queue.depth() > 0; let mut flush_tick = tokio::time::interval(runtime.config.flush_interval); flush_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); @@ -535,8 +559,13 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive tokio::select! { received = receiver.recv_many(&mut batch, runtime.config.replay_batch) => { if received == 0 { - // Channel closed: flush once more and stop. - runtime.flush().await; + // Channel closed: flush once more unless the snapshot is + // provably current AND idle (a dirty or pending state + // gets one last persist attempt, matching the shutdown + // retry the unconditional flush used to provide). + if runtime.dirty || runtime.queue.depth() > 0 { + runtime.flush().await; + } tracing::info!( target: "rustfs::heal::mrf", "MRF channel closed; consumer stopped after final flush" @@ -544,8 +573,10 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive return; } for intent in batch.drain(..) { - runtime.queue.try_push(intent); - runtime.new_since_flush += 1; + if runtime.queue.try_push(intent) { + runtime.new_since_flush += 1; + runtime.dirty = true; + } } runtime.dispatch(manager.as_ref()).await; if runtime.new_since_flush >= runtime.config.flush_threshold { @@ -553,15 +584,26 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive } } _ = flush_tick.tick() => { - if runtime.new_since_flush > 0 || runtime.queue.depth() > 0 { - runtime.flush().await; - runtime.dispatch(manager.as_ref()).await; - } else if runtime.journal_on_disk { - // All intents consumed: remove the journal so a restart - // replays nothing (mirrors MinIO's post-replay unlink). - delete_journal().await; - runtime.journal_on_disk = false; - gauge!("rustfs_heal_mrf_journal_bytes").set(0.0); + match tick_action(runtime.dirty, runtime.queue.depth(), runtime.journal_on_disk) { + TickAction::Flush => { + runtime.flush().await; + runtime.dispatch(manager.as_ref()).await; + } + TickAction::Retry => { + // Pending set unchanged since the last flush (a + // backlog waiting out an admission backoff): skip the + // rewrite but keep dispatching so the retry fires on + // time. + runtime.dispatch(manager.as_ref()).await; + } + TickAction::DeleteJournal => { + // All intents consumed: remove the journal so a restart + // replays nothing (mirrors MinIO's post-replay unlink). + delete_journal().await; + runtime.journal_on_disk = false; + gauge!("rustfs_heal_mrf_journal_bytes").set(0.0); + } + TickAction::Idle => {} } gauge!("rustfs_heal_mrf_queue_depth").set(runtime.queue.depth() as f64); } @@ -569,6 +611,33 @@ async fn run_mrf_consumer(manager: Arc, mut receiver: mpsc::Receive } } +/// What the periodic tick should do, as a pure function of the runtime state +/// so the decision table is unit-testable. +enum TickAction { + /// The pending set changed since the last snapshot: rewrite it, then + /// drain. + Flush, + /// Pending intents exist but the snapshot is current: only drain (an + /// admission backoff may have expired). + Retry, + /// Nothing pending and a stale journal file remains: remove it. + DeleteJournal, + /// Quiescent: nothing to do. + Idle, +} + +fn tick_action(dirty: bool, depth: usize, journal_on_disk: bool) -> TickAction { + if dirty { + TickAction::Flush + } else if depth > 0 { + TickAction::Retry + } else if journal_on_disk { + TickAction::DeleteJournal + } else { + TickAction::Idle + } +} + #[cfg(test)] mod tests { use super::*; @@ -586,6 +655,27 @@ mod tests { } } + #[test] + fn tick_action_table() { + use TickAction::*; + + // Dirty dominates: a changed pending set flushes even when idle + // otherwise. + assert!(matches!(tick_action(true, 0, false), Flush)); + assert!(matches!(tick_action(true, 3, true), Flush)); + + // Clean backlog: no rewrite, but keep draining so an expired + // admission backoff retries on time. + assert!(matches!(tick_action(false, 1, false), Retry)); + assert!(matches!(tick_action(false, 2, true), Retry)); + + // Quiescent with a stale journal file on disk: remove it. + assert!(matches!(tick_action(false, 0, true), DeleteJournal)); + + // Fully quiescent: nothing to do. + assert!(matches!(tick_action(false, 0, false), Idle)); + } + #[test] fn queue_enforces_count_and_byte_ceilings() { let mut queue = MrfQueue::new(2, usize::MAX); diff --git a/crates/heal/src/heal/task.rs b/crates/heal/src/heal/task.rs index 7b1d33348..4db62f271 100644 --- a/crates/heal/src/heal/task.rs +++ b/crates/heal/src/heal/task.rs @@ -54,7 +54,7 @@ const MAX_BUCKET_OBJECT_HEAL_RETRIES: u32 = 3; const MAX_BUCKET_FAILURE_LOG_SAMPLES: u64 = 5; /// Emits at `$level`, demoted to `debug!` when `$demote` is true. Keeps -/// per-object heal work — Object/Metadata/MRF/ECDecode tasks queued per +/// per-object heal work — Object/Metadata/ECDecode tasks queued per /// object by MRF/autoheal/scanner loops, and per-object sweep failures past /// a sample cap — from amplifying into one info!/warn!/error! line per /// object during mass recovery (rustfs/rustfs#5716). Aggregate task kinds @@ -75,8 +75,6 @@ const EVENT_HEAL_BUCKET_STAGE: &str = "heal_bucket_stage"; const EVENT_HEAL_BUCKET_RESULT: &str = "heal_bucket_result"; const EVENT_HEAL_METADATA_STAGE: &str = "heal_metadata_stage"; const EVENT_HEAL_METADATA_RESULT: &str = "heal_metadata_result"; -const EVENT_HEAL_MRF_STAGE: &str = "heal_mrf_stage"; -const EVENT_HEAL_MRF_RESULT: &str = "heal_mrf_result"; const EVENT_HEAL_EC_DECODE_STAGE: &str = "heal_ec_decode_stage"; const EVENT_HEAL_EC_DECODE_RESULT: &str = "heal_ec_decode_result"; const EVENT_HEAL_ERASURE_SET_STAGE: &str = "heal_erasure_set_stage"; @@ -101,8 +99,6 @@ pub enum HealType { ErasureSet { buckets: Vec, set_disk_id: String }, /// Metadata heal Metadata { bucket: String, object: String }, - /// MRF heal - MRF { meta_path: String }, /// EC decode heal ECDecode { bucket: String, @@ -120,21 +116,18 @@ impl HealType { Self::Prefix { .. } => "prefix", Self::ErasureSet { .. } => "erasure_set", Self::Metadata { .. } => "metadata", - Self::MRF { .. } => "mrf", Self::ECDecode { .. } => "ec_decode", } } /// Task kinds enqueued at per-object granularity (MRF, autoheal, scanner, - /// read-repair loops). Their lifecycle and admission logs stay at `debug!` + /// read-repair loops; the MRF loop queues Object/ECDecode/Metadata + /// tasks). Their lifecycle and admission logs stay at `debug!` /// so a recovery loop queuing hundreds of thousands of object heal tasks /// cannot amplify into per-object `info!`/`warn!` lines; aggregate kinds /// (cluster/bucket/prefix/erasure-set) keep operator-visible levels. pub(crate) fn is_per_object(&self) -> bool { - matches!( - self, - Self::Object { .. } | Self::Metadata { .. } | Self::MRF { .. } | Self::ECDecode { .. } - ) + matches!(self, Self::Object { .. } | Self::Metadata { .. } | Self::ECDecode { .. }) } } @@ -504,7 +497,6 @@ impl HealTask { HealType::Prefix { .. } => "prefix", HealType::ErasureSet { .. } => "erasure_set", HealType::Metadata { .. } => "metadata", - HealType::MRF { .. } => "mrf", HealType::ECDecode { .. } => "ec_decode", } } @@ -579,7 +571,6 @@ impl HealTask { None => event, } } - HealType::MRF { meta_path } => event.with_object(meta_path.as_str()), }; match error { @@ -821,7 +812,6 @@ impl HealTask { HealType::Prefix { bucket, prefix } => self.heal_prefix(bucket, prefix).await, HealType::Metadata { bucket, object } => self.heal_metadata(bucket, object).await, - HealType::MRF { meta_path } => self.heal_mrf(meta_path).await, HealType::ECDecode { bucket, object, @@ -2020,139 +2010,6 @@ impl HealTask { } } - async fn heal_mrf(&self, meta_path: &str) -> Result<()> { - debug!( - target: "rustfs::heal::task", - event = EVENT_HEAL_MRF_STAGE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_TASK, - task_id = %self.id, - meta_path, - stage = "start", - "Heal MRF started" - ); - - // update progress - { - let mut progress = self.progress.write().await; - progress.set_current_object(Some(format!("mrf: {meta_path}"))); - progress.update_progress(0, 2, 0, 0); - } - - // Parse meta_path to extract bucket and object - let parts: Vec<&str> = meta_path.split('/').collect(); - if parts.len() < 2 { - return Err(Error::TaskExecutionFailed { - message: format!("Invalid meta path format: {meta_path}"), - }); - } - - let bucket = parts[0]; - let object = parts[1..].join("/"); - - // Step 1: Perform MRF heal using ecstore - debug!( - target: "rustfs::heal::task", - event = EVENT_HEAL_MRF_STAGE, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_TASK, - task_id = %self.id, - meta_path, - bucket, - object = %object, - stage = "heal_with_ecstore", - "Heal MRF stage entered" - ); - let heal_opts = HealOpts { - recursive: true, - dry_run: self.options.dry_run, - remove: self.options.remove_corrupted, - recreate: self.options.recreate_missing, - scan_mode: HealScanMode::Deep, - update_parity: true, - no_lock: self.options.no_lock, - pool: None, - set: None, - }; - - let heal_result = self - .await_with_control(self.storage.heal_object(bucket, &object, None, &heal_opts)) - .await; - - match heal_result { - Ok((result, error)) => { - if let Some(e) = error { - error!( - target: "rustfs::heal::task", - event = EVENT_HEAL_MRF_RESULT, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_TASK, - task_id = %self.id, - meta_path, - bucket, - object = %object, - result = "failed", - error = %e, - "Heal MRF failed" - ); - { - let mut progress = self.progress.write().await; - progress.update_progress(2, 2, 0, 0); - } - return Err(Error::TaskExecutionFailed { - message: format!("Failed to heal MRF {meta_path}: {e}"), - }); - } - - debug!( - target: "rustfs::heal::task", - event = EVENT_HEAL_MRF_RESULT, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_TASK, - task_id = %self.id, - meta_path, - bucket, - object = %object, - drives_healed = result.drives_healed(), - drives_total = result.drives_reported(), - result = "ok", - "Heal MRF repaired" - ); - - { - let mut progress = self.progress.write().await; - progress.update_progress(2, 2, 0, 0); - } - self.record_result_item(result).await; - Ok(()) - } - Err(Error::TaskCancelled) => Err(Error::TaskCancelled), - Err(Error::TaskTimeout) => Err(Error::TaskTimeout), - Err(e) => { - error!( - target: "rustfs::heal::task", - event = EVENT_HEAL_MRF_RESULT, - component = LOG_COMPONENT_HEAL, - subsystem = LOG_SUBSYSTEM_TASK, - task_id = %self.id, - meta_path, - bucket, - object = %object, - result = "failed", - error = %e, - "Heal MRF failed" - ); - { - let mut progress = self.progress.write().await; - progress.update_progress(2, 2, 0, 0); - } - Err(Error::TaskExecutionFailed { - message: format!("Failed to heal MRF {meta_path}: {e}"), - }) - } - } - } - async fn heal_ec_decode(&self, bucket: &str, object: &str, version_id: Option<&str>) -> Result<()> { debug!( target: "rustfs::heal::task", @@ -3389,12 +3246,6 @@ mod tests { } .is_per_object() ); - assert!( - HealType::MRF { - meta_path: "p".to_string(), - } - .is_per_object() - ); assert!( HealType::ECDecode { bucket: "b".to_string(), diff --git a/crates/heal/tests/heal_bug_fixes_test.rs b/crates/heal/tests/heal_bug_fixes_test.rs index 1bdb00348..59ff1bb3f 100644 --- a/crates/heal/tests/heal_bug_fixes_test.rs +++ b/crates/heal/tests/heal_bug_fixes_test.rs @@ -13,93 +13,13 @@ // limitations under the License. use rustfs_heal::heal::{ - event::{HealEvent, Severity}, task::{HealPriority, HealType}, utils, }; mod storage_api; -use storage_api::bug_fixes::{BucketInfo, DiskStore, Endpoint}; - -#[test] -fn test_heal_event_to_heal_request_no_panic() { - // Test that invalid pool/set indices don't cause panic - // Create endpoint using try_from or similar method - let endpoint_result = Endpoint::try_from("http://localhost:9000"); - if let Ok(mut endpoint) = endpoint_result { - endpoint.pool_idx = -1; - endpoint.set_idx = -1; - endpoint.disk_idx = 0; - - let event = HealEvent::DiskStatusChange { - endpoint, - old_status: "ok".to_string(), - new_status: "offline".to_string(), - }; - - // Should return error instead of panicking - let result = event.to_heal_request(); - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("Invalid heal type")); - } -} - -#[test] -fn test_heal_event_to_heal_request_valid_indices() { - // Test that valid indices work correctly - let endpoint_result = Endpoint::try_from("http://localhost:9000"); - if let Ok(mut endpoint) = endpoint_result { - endpoint.pool_idx = 0; - endpoint.set_idx = 1; - endpoint.disk_idx = 0; - - let event = HealEvent::DiskStatusChange { - endpoint, - old_status: "ok".to_string(), - new_status: "offline".to_string(), - }; - - let result = event.to_heal_request(); - assert!(result.is_ok()); - let request = result.unwrap(); - assert!(matches!(request.heal_type, HealType::ErasureSet { .. })); - } -} - -#[test] -fn test_heal_event_object_corruption() { - let event = HealEvent::ObjectCorruption { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - version_id: None, - corruption_type: rustfs_heal::heal::event::CorruptionType::DataCorruption, - severity: Severity::High, - }; - - let result = event.to_heal_request(); - assert!(result.is_ok()); - let request = result.unwrap(); - assert!(matches!(request.heal_type, HealType::Object { .. })); - assert_eq!(request.priority, HealPriority::High); -} - -#[test] -fn test_heal_event_ec_decode_failure() { - let event = HealEvent::ECDecodeFailure { - bucket: "test-bucket".to_string(), - object: "test-object".to_string(), - version_id: None, - missing_shards: vec![0, 1], - available_shards: vec![2, 3], - }; - - let result = event.to_heal_request(); - assert!(result.is_ok()); - let request = result.unwrap(); - assert!(matches!(request.heal_type, HealType::ECDecode { .. })); - assert_eq!(request.priority, HealPriority::Urgent); -} +use storage_api::bug_fixes::{BucketInfo, DiskStore}; #[test] fn test_format_set_disk_id_from_i32_negative() {