From 0a975f2fe2513585de67d7ba77b5d5233f06f3ed Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Wed, 2 Sep 2026 08:26:59 +0800 Subject: [PATCH] docs(knowledge-base): prune stale content and add agent-facing index (#7035) --- .agents/skills/arch-checks/SKILL.md | 9 +- .config/nextest.toml | 5 +- .gitignore | 3 - AGENTS.md | 1 + ARCHITECTURE.md | 2 +- CLAUDE.md | 1 + crates/e2e_test/README.md | 14 +- docs/architecture/README.md | 95 ++- .../admin-route-action-snapshot.md | 148 +---- .../background-controller-contract.md | 206 ++----- .../background-services-inventory.md | 89 --- docs/architecture/compat-cleanup-register.md | 9 +- .../architecture/config-model-boundary-adr.md | 198 +----- docs/architecture/crate-boundaries.md | 290 +-------- .../decommission-compatibility.md | 322 ++-------- .../ecstore-api-facade-inventory.md | 103 ++-- .../ecstore-config-consumer-inventory.md | 199 ------ docs/architecture/ecstore-layout-boundary.md | 62 +- .../architecture/ecstore-module-split-plan.md | 360 ++--------- docs/architecture/erasure-coding.md | 30 +- .../global-state-crate-split-plan.md | 201 +------ docs/architecture/global-state-inventory.md | 170 ++---- docs/architecture/heal-concurrency-model.md | 77 +++ docs/architecture/kms-bulk-rekey-contract.md | 205 +++---- docs/architecture/minio-file-format-compat.md | 468 +++------------ .../minio-rustfs-router-compatibility.md | 225 +------ .../obs-ecstore-dependency-inventory.md | 70 +-- docs/architecture/overview.md | 60 +- .../placement-repair-invariants.md | 101 ++-- docs/architecture/readiness-matrix.md | 7 +- .../runtime-capability-contracts.md | 86 +-- docs/architecture/runtime-lifecycle.md | 143 ++--- docs/architecture/s3-compatibility-matrix.md | 60 +- docs/architecture/s3-tables-support-matrix.md | 355 +++-------- docs/architecture/scanner-heal-admission.md | 29 - .../storage-control-data-plane.md | 12 +- .../architecture/unified-object-generation.md | 422 +++---------- .../workload-admission-contracts.md | 140 +---- docs/operations/atomic-object-undo.md | 3 + docs/operations/authing-oidc-integration.md | 240 -------- .../container-resource-detection.md | 229 ++----- docs/operations/dial9-runtime-profiling.md | 8 +- docs/operations/drive-timeout-tuning.md | 35 +- docs/operations/durability-modes.md | 35 +- .../get-path-experimental-switches.md | 35 -- .../heal-concurrency-safety-notes-zh.md | 115 ---- .../heal-scanner-parity-notes-zh.md | 109 ---- docs/operations/hotpath-warp-ab-runbook.md | 270 +++++---- docs/operations/hotpath-warp-abba-runbook.md | 296 --------- .../internode-grpc-benchmark-runbook.md | 178 ++---- ...ernode-msgpack-json-convergence-runbook.md | 176 ++---- docs/operations/keycloak-oidc-integration.md | 305 ---------- docs/operations/kms-admin-contract.md | 96 ++- docs/operations/kms-backend-security.md | 278 ++++----- .../kms-cryptographic-compliance.md | 106 +--- .../operations/kms-disaster-recovery-drill.md | 11 +- docs/operations/kms-observability-runbook.md | 175 +++--- docs/operations/kms-per-key-authorization.md | 7 +- docs/operations/log-diagnose.md | 102 ++-- docs/operations/nats-jetstream.md | 393 +++--------- docs/operations/no-parity-bitrot-recovery.md | 99 +-- docs/operations/object-io-tuning-ab-matrix.md | 155 ++--- docs/operations/oidc-console-integration.md | 217 +++++++ docs/operations/oidc-provider-requirements.md | 26 + .../openid-compatibility-issues-for-vendor.md | 134 ----- docs/operations/outbound-connection-policy.md | 148 ++--- docs/operations/pool-metadata-recovery.md | 58 +- .../presigned-multipart-size-limit.md | 55 -- docs/operations/presigned-put-size-limit.md | 35 -- docs/operations/presigned-size-limits.md | 57 ++ .../rebalance-stored-representation-impact.md | 225 ++----- docs/operations/replication-check.md | 52 +- .../replication-object-size-limits.md | 6 +- docs/operations/reverse-proxy.md | 124 ++-- docs/operations/rolling-restart.md | 147 ++--- ...minio-comprehensive-analysis-2026-08-16.md | 568 ------------------ ...io-comprehensive-analysis-2026-08-16_zh.md | 568 ------------------ docs/operations/s3-tables-cutover-runbook.md | 60 ++ docs/operations/scanner-benchmark-runbook.md | 351 +++-------- docs/operations/scanner-excess-alerts.md | 34 +- docs/operations/scanner-excess-alerts_zh.md | 37 -- docs/operations/scanner-runtime-controls.md | 543 +++++++---------- docs/operations/sftp.md | 307 +++------- docs/operations/tier-ilm-debugging.md | 154 +++-- docs/operations/two-factor-auth.md | 256 ++------ docs/operations/vault-kms-authentication.md | 44 +- docs/testing/README.md | 238 ++------ docs/testing/ci-gates.md | 188 +++--- docs/testing/e2e-suite-inventory.md | 107 ---- .../ecstore-validation-suite-design.md | 470 ++++----------- docs/testing/issue-2007-coalescer-delay.md | 58 -- docs/testing/metrics-1649-prometheus-smoke.md | 44 -- docs/testing/security-regressions.md | 102 +--- rustfs/src/storage/rpc/http_service.rs | 3 +- scripts/README.md | 6 +- scripts/check_architecture_migration_rules.sh | 2 +- scripts/check_doc_paths.sh | 15 +- .../issue_2007_coalescer_prometheus_report.py | 17 + scripts/prometheus_metrics_1649_smoke.py | 13 + 99 files changed, 3312 insertions(+), 10590 deletions(-) delete mode 100644 docs/architecture/background-services-inventory.md delete mode 100644 docs/architecture/ecstore-config-consumer-inventory.md create mode 100644 docs/architecture/heal-concurrency-model.md delete mode 100644 docs/architecture/scanner-heal-admission.md delete mode 100644 docs/operations/authing-oidc-integration.md delete mode 100644 docs/operations/get-path-experimental-switches.md delete mode 100644 docs/operations/heal-concurrency-safety-notes-zh.md delete mode 100644 docs/operations/heal-scanner-parity-notes-zh.md delete mode 100644 docs/operations/hotpath-warp-abba-runbook.md delete mode 100644 docs/operations/keycloak-oidc-integration.md create mode 100644 docs/operations/oidc-console-integration.md create mode 100644 docs/operations/oidc-provider-requirements.md delete mode 100644 docs/operations/openid-compatibility-issues-for-vendor.md delete mode 100644 docs/operations/presigned-multipart-size-limit.md delete mode 100644 docs/operations/presigned-put-size-limit.md create mode 100644 docs/operations/presigned-size-limits.md delete mode 100644 docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md delete mode 100644 docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md create mode 100644 docs/operations/s3-tables-cutover-runbook.md delete mode 100644 docs/operations/scanner-excess-alerts_zh.md delete mode 100644 docs/testing/e2e-suite-inventory.md delete mode 100644 docs/testing/issue-2007-coalescer-delay.md delete mode 100644 docs/testing/metrics-1649-prometheus-smoke.md diff --git a/.agents/skills/arch-checks/SKILL.md b/.agents/skills/arch-checks/SKILL.md index a95347830..76ba69008 100644 --- a/.agents/skills/arch-checks/SKILL.md +++ b/.agents/skills/arch-checks/SKILL.md @@ -50,10 +50,11 @@ consider adding it to the script's `checked_files` list. ## `check_doc_paths.sh` -Instruction/architecture docs (`AGENTS.md`, `CLAUDE.md`, `ARCHITECTURE.md`, -`docs/architecture/*.md`) must not reference repo file paths that no longer -exist. If your refactor moved code, update the docs that point at it — the -error message lists `doc -> stale-path` pairs. +Instruction docs (`AGENTS.md`, `CLAUDE.md`, `ARCHITECTURE.md`) and every +Markdown file under `docs/` (architecture, operations, testing, index) must not +reference repo file paths that no longer exist. If your refactor moved code, +update the docs that point at it — the error message lists `doc -> stale-path` +pairs. Cite paths plus symbol names, never line numbers (see `docs/README.md`). ## `check_no_planning_docs.sh` diff --git a/.config/nextest.toml b/.config/nextest.toml index d4da98b79..8c08956fc 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -355,7 +355,8 @@ test-group = 'ecstore-serial-flaky' # allowlist", so any new replication test lands in nightly by default (never # silently unrun) until it is explicitly blessed as fast here. Keep the two # regexes byte-identical. The committed profile selection digests make changes -# visible in CI; current counts live in docs/testing/e2e-suite-inventory.md. +# visible in CI; list current membership with `cargo nextest list -p e2e_test +# --profile ` (platform-dependent; see docs/testing/README.md). # HISTORY (2026-07-11): the 20 fast tests were briefly pulled out of this lane # (#4724) because they set a loopback (127.0.0.1) replication target that the # SSRF egress guard rejected on every PR after repl-1 (#4712). That is fixed — @@ -508,7 +509,7 @@ path = "junit.xml" # quota, checksum, encryption, # security-boundary, ... — that the fast PR `e2e-smoke` subset deliberately # skips. Budget <= 45 min; authority for the suite count is `cargo nextest list -# --profile e2e-full` (see docs/testing/e2e-suite-inventory.md). +# --profile e2e-full -p e2e_test` (platform-dependent; see docs/testing/README.md). # # The filter is "the whole e2e_test crate MINUS the sets owned by other lanes": # * protocols:: — FTPS/SFTP/WebDAV, run from the dedicated protocol profile diff --git a/.gitignore b/.gitignore index 0d321c00a..c6d670f50 100644 --- a/.gitignore +++ b/.gitignore @@ -57,9 +57,6 @@ docs/* !docs/operations/** !docs/testing/ !docs/testing/** -docs/heal-scanner-logging-governance.md -docs/benchmark/rustfs-target-bench/ -docs/benchmark/*.md .codegraph/* .docker/test/compat/data/* .docker/test/compat/kms/* diff --git a/AGENTS.md b/AGENTS.md index d581061c1..acba2d9b9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,6 +86,7 @@ This file contains repository-wide rules. Use the nearest subdirectory - CI gates: `.github/workflows/ci.yml`. - PR format: `.github/pull_request_template.md`. - Architecture routing: `ARCHITECTURE.md` and `docs/architecture/README.md`. +- Knowledge-base index and documentation rules: `docs/README.md`. - Agent skills: `.agents/skills/*/SKILL.md`. Do not commit one-shot plans, trackers, migration ledgers, benchmark snapshots, diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 5aaf9163f..4d85c8f23 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -62,7 +62,7 @@ rustfs/ # Workspace root (virtual manifest) │ ├── utils/ # Pure utility functions │ ├── ... # (see "Crate Reference" below) │ └── e2e_test/ # End-to-end integration tests -└── docs/ # Design documents and analysis +└── docs/ # Agent knowledge base: contracts, runbooks, testing rules (index: docs/README.md) ``` ### Main Crate Layers (`rustfs/src/`) diff --git a/CLAUDE.md b/CLAUDE.md index eca1def46..7059380d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,6 +27,7 @@ make build-docker BUILD_OS=ubuntu22.04 ## Where to look (do not duplicate here) +- Agent knowledge base index and doc-writing rules: [docs/README.md](docs/README.md) - Crate membership: `Cargo.toml` `[workspace].members` - Architecture, layering, crate map: [ARCHITECTURE.md](ARCHITECTURE.md) - Migration guardrails & readiness contracts: [docs/architecture/](docs/architecture/README.md) diff --git a/crates/e2e_test/README.md b/crates/e2e_test/README.md index 634df77bb..2685661b3 100644 --- a/crates/e2e_test/README.md +++ b/crates/e2e_test/README.md @@ -233,8 +233,8 @@ spawn error. Install the pinned CI version before running their profiles. [`src/policy/README.md`](src/policy/README.md), [`src/protocols/README.md`](src/protocols/README.md), [`src/reliant/README.md`](src/reliant/README.md) -- Authoritative per-module counts: - [`docs/testing/e2e-suite-inventory.md`](../../docs/testing/e2e-suite-inventory.md) +- Per-module counts: `cargo nextest list -p e2e_test --profile ` + (one-liner in [`docs/testing/README.md`](../../docs/testing/README.md)) - Test pyramid & flake policy: [`docs/testing/README.md`](../../docs/testing/README.md) ## CI smoke subset (`--profile e2e-smoke`) @@ -271,12 +271,12 @@ Note on `#[serial]`: nextest runs each test in its own process, so parallel-safe by construction (random port + isolated temp dir), which the current subset is. -### Authoritative test inventory +### Test inventory -`docs/testing/e2e-suite-inventory.md` records the per-module test counts as -listed by `cargo nextest list -p e2e_test`. Regenerate it when adding or -moving e2e tests so acceptance numbers in the test-strategy issues -(backlog#1147–#1155) stay auditable. When a profile membership change is +Per-module counts are not committed; list them with +`cargo nextest list -p e2e_test --profile ` (the result is +platform-dependent because some modules are linux-only; the `jq` one-liner is +in `docs/testing/README.md`). When a profile membership change is intentional, review its JSON listing before updating the matching `.config/e2e-*-selection.txt` test-ID digest. Update only the platform that produced the listing: diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 0a317ab0e..0a5f7bf58 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -1,66 +1,57 @@ # Architecture Documentation -Durable architecture reference for RustFS: migration guardrails, runtime -contracts, boundary rules, and support matrices. +**Use this when:** you need the contract, invariant, or boundary rule that governs a change, and you want the one document that owns it. +**Source of truth:** the code and the guards. `scripts/check_architecture_migration_rules.sh` enforces the CI-anchored documents below; `scripts/check_doc_paths.sh` fails the pre-commit gate when any doc under `docs/` cites a repository path that no longer exists. Two rules keep this directory healthy: -1. **Durable reference only.** One-shot implementation plans, task trackers, - and PR templates do not belong in the repository — keep them in the issue - tracker or your local worktree. When their work closes, delete them rather - than archiving them here. -2. **No copies of other sources of truth.** Crate lists come from - `Cargo.toml`, CI steps from `.github/workflows/ci.yml`, code structure from - the code. `scripts/check_doc_paths.sh` fails the pre-commit gate when a - doc here references a file path that no longer exists. +1. **Durable reference only.** One-shot plans, task trackers, dated analyses, status snapshots, and PR-scoped notes do not belong in the repository; keep them in the issue tracker or a local worktree and delete them when the work closes. +2. **No copies of other sources of truth.** Crate lists come from `Cargo.toml`, CI steps from `.github/workflows/`, code structure from the code. Cite a file path plus a symbol name, never a line number, and never paste counts or tables that a command can regenerate. -## Start here +Every document starts with a `**Use this when:**` line so an agent can decide in one glance whether to read further. The index below repeats those lines. -- [overview.md](overview.md) — migration baseline, phase order, core principles +## CI-anchored core -## CI-enforced core (required by `scripts/check_architecture_migration_rules.sh`) +Required headings and strings in these files are asserted by `scripts/check_architecture_migration_rules.sh`; rename a heading only together with the guard. -- [crate-boundaries.md](crate-boundaries.md) — dependency direction, PR types, re-export contracts -- [runtime-lifecycle.md](runtime-lifecycle.md) — startup/shutdown sequencing, readiness guarantees -- [readiness-matrix.md](readiness-matrix.md) — request/dependency behavior, probe semantics -- [storage-control-data-plane.md](storage-control-data-plane.md) — storage API contracts, control-plane boundaries -- [global-state-crate-split-plan.md](global-state-crate-split-plan.md) — remaining global-state owners and split evaluation -- [ecstore-module-split-plan.md](ecstore-module-split-plan.md) — ECStore decomposition rules and facade contracts +| Document | Use this when | +|---|---| +| [crate-boundaries.md](crate-boundaries.md) | you add a crate dependency, move code across crates, touch a `storage_api.rs` boundary file, or need the change-type vocabulary the architecture guard enforces | +| [runtime-lifecycle.md](runtime-lifecycle.md) | moving or reordering anything in `rustfs/src/startup_*.rs`, changing readiness publication, or touching shutdown ordering | +| [readiness-matrix.md](readiness-matrix.md) | changing what a request surface does before storage or IAM is ready, changing probe semantics, or adding a runtime dependency that readiness must wait for | +| [storage-control-data-plane.md](storage-control-data-plane.md) | adding a storage API surface, a cluster read model, or a background-service status/reconcile surface, and you need to know which layer owns it | +| [global-state-crate-split-plan.md](global-state-crate-split-plan.md) | business logic needs runtime state (object store, endpoints, lock clients, lifecycle state, config) and you must pick the right boundary, or you are evaluating a crate split out of ECStore | +| [global-state-inventory.md](global-state-inventory.md) | you meet a `GLOBAL_*` static or an `OnceLock` and need to know whether it is a runtime ownership handle, an owner-local static, or process-global by design | +| [ecstore-module-split-plan.md](ecstore-module-split-plan.md) | you add lifecycle or replication logic and need to know which crate it belongs in, plan to move an operation family out of `SetDisks`, or the guard fails on one of the split rules | +| [ecstore-api-facade-inventory.md](ecstore-api-facade-inventory.md) | you need something from `rustfs_ecstore` in another crate, you are narrowing a `rustfs_ecstore::api` facade group, or the guard reports a facade bypass | +| [obs-ecstore-dependency-inventory.md](obs-ecstore-dependency-inventory.md) | adding, removing, or moving any `rustfs_ecstore` or `rustfs_storage_api` reference inside `crates/obs` | +| [compat-cleanup-register.md](compat-cleanup-register.md) | you add, review, or remove a temporary compatibility path and need the `RUSTFS_COMPAT_TODO` marker format and its removal condition | +| [overview.md](overview.md) | you need the historical framing of the architecture-migration program or the phase names that other contracts refer to | -## Contracts & invariants +## Contracts and invariants -- [erasure-coding.md](erasure-coding.md) — normative erasure-coding algorithm and on-disk (`xl.meta`) compatibility contract; the frozen invariants for all user-data read/write, encode/decode, quorum, heal, and decode tolerance -- [placement-repair-invariants.md](placement-repair-invariants.md) -- [unified-object-generation.md](unified-object-generation.md) — single per-object generation authority (fencing epoch, transport/encoding/proto/mixed-version contracts) -- [runtime-capability-contracts.md](runtime-capability-contracts.md) -- [workload-admission-contracts.md](workload-admission-contracts.md) -- [background-controller-contract.md](background-controller-contract.md) -- [config-model-boundary-adr.md](config-model-boundary-adr.md) -- [ecstore-layout-boundary.md](ecstore-layout-boundary.md) -- [decommission-compatibility.md](decommission-compatibility.md) -- [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) — object-side DEK re-wrap job: work unit, idempotency model, exclusion rules, and the never-destroy-old-key-versions constraint +| Document | Use this when | +|---|---| +| [erasure-coding.md](erasure-coding.md) | changing anything under `crates/ecstore/src/erasure/`, `crates/filemeta/`, `crates/ecstore/src/set_disk/`, storage-class or layout code, or any decode, quorum, or heal boundary (normative spec) | +| [placement-repair-invariants.md](placement-repair-invariants.md) | changing anything that resolves an object to a pool, set, or disk, or that admits scanner or heal work | +| [heal-concurrency-model.md](heal-concurrency-model.md) | changing heal, PUT/multipart commit, delete, lifecycle expiry, or data-movement code that shares the `(bucket, object)` commit surface, or asking whether RustFS needs a persistent healing marker | +| [unified-object-generation.md](unified-object-generation.md) | adding or changing anything that fences a commit, scopes a read lease, gates old-directory cleanup, binds prepared pool reads, or settles quota against the current object version | +| [decommission-compatibility.md](decommission-compatibility.md) | changing pool decommission or rebalance behavior, its admin API shape, the persisted `PoolMeta` fields, or how tier free versions move between pools | +| [ecstore-layout-boundary.md](ecstore-layout-boundary.md) | touching endpoint expansion, `FormatV3`, pool/set layout, or moving files between ECStore's internal directories | +| [runtime-capability-contracts.md](runtime-capability-contracts.md) | changing the read-only observability or topology snapshot contracts in `rustfs-storage-api`, their providers, or the `storage_classes` payload of `GET /rustfs/admin/v4/runtime/capabilities` | +| [workload-admission-contracts.md](workload-admission-contracts.md) | adding a workload class or snapshot provider, or consuming admission state from a background job | +| [background-controller-contract.md](background-controller-contract.md) | adding a status snapshot or reconcile surface for a background service, or being tempted to fold several services into a generic controller | +| [config-model-boundary-adr.md](config-model-boundary-adr.md) | touching the server-config model (`Config`, `KV`, `KVS`) or its persistence, or asking which crate owns which part of server configuration | +| [admin-route-action-snapshot.md](admin-route-action-snapshot.md) | adding, moving, or re-authorizing an admin route and needing to know where the route → handler → `AdminAction` contract is enforced | +| [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) | changing the bulk envelope re-wrap sweep, its admin endpoints, the re-wrap primitive, or which objects a rekey may touch | -## Support matrices (release-facing, keep current) +## Support and compatibility matrices (release-facing, keep current) -- [s3-compatibility-matrix.md](s3-compatibility-matrix.md) -- [s3-tables-support-matrix.md](s3-tables-support-matrix.md) -- [minio-rustfs-router-compatibility.md](minio-rustfs-router-compatibility.md) -- [minio-file-format-compat.md](minio-file-format-compat.md) +| Document | Use this when | +|---|---| +| [s3-compatibility-matrix.md](s3-compatibility-matrix.md) | writing or checking a user-facing S3 compatibility claim, or moving a Ceph s3tests case between lists | +| [s3-tables-support-matrix.md](s3-tables-support-matrix.md) | writing a release note or client-compatibility statement about S3 Tables / Iceberg REST Catalog (cutover procedure: [../operations/s3-tables-cutover-runbook.md](../operations/s3-tables-cutover-runbook.md)) | +| [minio-rustfs-router-compatibility.md](minio-rustfs-router-compatibility.md) | a client or `mc` call that works against MinIO fails against RustFS and you need to know whether the endpoint is missing, stubbed, or deliberately different | +| [minio-file-format-compat.md](minio-file-format-compat.md) | deciding whether a MinIO drive set, bucket-metadata blob, or SSE object can be read or imported by a given RustFS build, or before touching a listed version anchor | -## Inventories & baselines (snapshots that feed migration work) - -- [global-state-inventory.md](global-state-inventory.md) -- [ecstore-api-facade-inventory.md](ecstore-api-facade-inventory.md) -- [ecstore-config-consumer-inventory.md](ecstore-config-consumer-inventory.md) -- [obs-ecstore-dependency-inventory.md](obs-ecstore-dependency-inventory.md) -- [background-services-inventory.md](background-services-inventory.md) -- [scanner-heal-admission.md](scanner-heal-admission.md) -- [admin-route-action-snapshot.md](admin-route-action-snapshot.md) -- [compat-cleanup-register.md](compat-cleanup-register.md) - -Historical plans and trackers (rebalance/decommission phases, -migration-progress ledger, and the one-shot migration snapshots that fed it — -startup timeline, scheduler baseline, profiling/NUMA capability inventory, KMS -development defaults inventory) were retired in 2026-07 once the -architecture-review ledger they served closed out (backlog#660/#665). Planning -documents are no longer kept in the repository. +Operations runbooks live in [../operations/](../README.md#operations) and testing references in [../testing/README.md](../testing/README.md). diff --git a/docs/architecture/admin-route-action-snapshot.md b/docs/architecture/admin-route-action-snapshot.md index be64ad6ac..b73fdcd1e 100644 --- a/docs/architecture/admin-route-action-snapshot.md +++ b/docs/architecture/admin-route-action-snapshot.md @@ -1,145 +1,27 @@ # Admin Route Action Snapshot -This snapshot records the current admin routing and authorization surface before -directory moves or crate extraction. It is a migration guardrail: later pure -move PRs must preserve the route, handler, authorization action, public -exception, and compatibility alias semantics listed here unless the PR is -explicitly scoped as a behavior change. +**Use this when:** you add, move, or re-authorize an admin route and need to know where the route → handler → `AdminAction` contract is enforced. +**Source of truth:** `rustfs/src/admin/route_policy.rs` (the `AdminRouteSpec` matrix, checked by `validate_admin_route_policy_specs`), `rustfs/src/admin/route_registration_test.rs` (registration coverage), `rustfs/src/admin/router.rs` (dispatch and credential checks), `rustfs/src/admin/handlers/*.rs` (handler-level authorization calls). -## Source Of Truth - -- Router assembly: `rustfs/src/admin/mod.rs::make_admin_route` -- Route registration coverage: `rustfs/src/admin/route_registration_test.rs` -- Runtime dispatch: `rustfs/src/admin/router.rs` -- Admin auth helpers: `rustfs/src/admin/auth.rs` -- Handler route/action ownership: `rustfs/src/admin/handlers/*.rs` - -The route registration test intentionally covers representative paths for every -registered route family. This document uses route patterns from the registration -functions and action names from the handler authorization calls. +This page is a pointer, not a route table. The machine-checked matrix in `route_policy.rs` lists every admin route with its `AdminAction` and `RouteRiskLevel`; routes that are registered but answered by policy instead of a handler are declared there too through `DeferredRoutePolicyReason`. The `AdminRouteSpec` type lives in `crates/security-governance/src/admin_matrix.rs`. ## Prefix And Alias Contract -| Prefix | Current behavior | Migration rule | +| Prefix | Behavior | Rule | |---|---|---| -| `/rustfs/admin` | Canonical admin API prefix used by route registration | Keep as the single registered admin prefix | -| `/minio/admin` | Compatibility alias accepted by `S3Router::is_match`; dispatch canonicalizes it to `/rustfs/admin` | Do not duplicate registrations; preserve canonicalization | -| `/iceberg/v1` table catalog prefix | Registered through `table_catalog::register_table_catalog_route` and accepted by `is_admin_path` | Keep outside `/rustfs/admin` and document auth separately | -| `/health` and `/health/ready` | Public health endpoints when `ENV_HEALTH_ENDPOINT_ENABLE` allows registration | Preserve unauthenticated health bypass | -| `/profile/cpu` and `/profile/memory` | Registered by health handler but guarded by profile auth | Do not couple to health endpoint enablement | - -The compatibility alias is not a second route table. `canonicalize_admin_path` -maps `/minio/admin/...` to `/rustfs/admin/...` immediately before route lookup. - -## Dispatch And Auth Shape - -```mermaid -flowchart LR - A["Incoming request"] --> B{"S3Router::is_match"} - B -->|"Replication or misc extension"| X["Extension handler"] - B -->|"Health path"| H["Public health"] - B -->|"OIDC public path"| O["OIDC public handler"] - B -->|"POST / STS form"| S["STS handler"] - B -->|"Admin or console path"| C{"S3Router::check_access"} - C -->|"public exception"| P["No SigV4 required"] - C -->|"admin route"| D["Credential required"] - D --> E["canonicalize /minio/admin to /rustfs/admin"] - E --> F["matchit route lookup"] - F --> G["AdminOperation handler"] - G --> I["handler-level validate_admin_request"] -``` - -Route-level credential presence and handler-level policy authorization are -separate contracts. The router enforces credential presence for ordinary admin -routes. Handler rows below record whether the current handler performs a -precise `AdminAction` or `S3Action` check, or only repeats a credential -presence check. +| `/rustfs/admin` | Canonical admin prefix used by `make_admin_route` (`rustfs/src/admin/mod.rs`) | The only registered admin prefix | +| `/minio/admin` | Compatibility alias accepted by `S3Router::is_match`; `canonicalize_admin_path` rewrites it to `/rustfs/admin` immediately before route lookup (`rustfs/src/admin/router.rs`) | Never register routes twice; preserve canonicalization | +| `/iceberg/v1` | Table catalog prefix registered by `register_table_catalog_route` (`rustfs/src/admin/handlers/table_catalog/routes.rs`) and accepted by `is_admin_path` | Stays outside `/rustfs/admin`; table actions are authorized per handler | +| `/health`, `/health/ready` | Public health endpoints, registered only when `ENV_HEALTH_ENDPOINT_ENABLE` allows | Preserve the unauthenticated bypass | +| `/profile/cpu`, `/profile/memory` | Registered by the health handler but guarded by profile authorization | Never couple to health-endpoint enablement | ## Public Exceptions -| Method | Path pattern | Handler | Auth contract | -|---|---|---|---| -| `GET`, `HEAD` | `/health` | `HealthCheckHandler` | Public when health routes are registered | -| `GET`, `HEAD` | `/health/ready` | `HealthCheckHandler` | Public when health routes are registered | -| Registered as `GET`; auth bypass is path-based | `/rustfs/admin/v3/oidc/providers` and `/minio/admin/v3/oidc/providers` | `ListOidcProvidersHandler` | Public OIDC bootstrap path; `check_access` bypasses SigV4 for any method matching this path | -| Registered as `GET`; auth bypass is path-prefix-based | `/rustfs/admin/v3/oidc/authorize/{provider_id}` and `/minio/admin/v3/oidc/authorize/{provider_id}` | `OidcAuthorizeHandler` | Public OIDC bootstrap path; `check_access` bypasses SigV4 for any method matching this path prefix | -| Registered as `GET`; auth bypass is path-prefix-based | `/rustfs/admin/v3/oidc/callback/{provider_id}` and `/minio/admin/v3/oidc/callback/{provider_id}` | `OidcCallbackHandler` | Public OIDC bootstrap path; `check_access` bypasses SigV4 for any method matching this path prefix | -| Registered as `GET`; auth bypass is path-based | `/rustfs/admin/v3/oidc/logout` and `/minio/admin/v3/oidc/logout` | `OidcLogoutHandler` | Public OIDC logout path; `check_access` bypasses SigV4 for any method matching this path | -| `POST` | `/` with `application/x-www-form-urlencoded` | `AssumeRoleHandle` | Public only for unsigned STS web identity form requests; handler validates JWT/action | -| Any matched method | `/favicon.ico` and `/rustfs/console...` | Console router | Public only when `console_enabled` is true; router bypasses SigV4 before handing off to the console router | +Router-level credential checks (`S3Router::check_access`) are bypassed only for: -## Registered Route Families +- health routes, when they are registered; +- OIDC bootstrap paths matched by `is_oidc_path` (`providers`, `authorize/{provider_id}`, `callback/{provider_id}`, `logout`); the bypass is path-based, so it applies to any method on those paths; +- unsigned STS web-identity form posts to `/` with `application/x-www-form-urlencoded`, which the STS handler validates itself; +- console assets (`/favicon.ico`, `/rustfs/console...`), only while the console is enabled. -All rows with `/rustfs/admin` also accept the `/minio/admin` compatibility alias -through router canonicalization unless the row explicitly says otherwise. - -| Area | Methods and path patterns | Handler ownership | Authorization contract | -|---|---|---|---| -| STS and admin probe | `POST /`; `GET /rustfs/admin/v3/is-admin` | `sts.rs`, `is_admin.rs` | STS dispatch validates request action; is-admin checks `AllAdminActions` | -| User lifecycle | `GET /v3/list-users`; `GET /v3/user-info`; `PUT /v3/add-user`; `PUT /v3/set-user-status`; `DELETE /v3/remove-user` | `user_lifecycle.rs`, `user.rs` | `ListUsersAdminAction`, `GetUserAdminAction`, `CreateUserAdminAction`, `EnableUserAdminAction`, `DeleteUserAdminAction` | -| Group management | `GET /v3/groups`; `GET /v3/group`; `DELETE /v3/group/{group}`; `PUT /v3/set-group-status`; `PUT /v3/update-group-members` | `group.rs` | `ListGroupsAdminAction`, `GetGroupAdminAction`, `RemoveUserFromGroupAdminAction`, `EnableGroupAdminAction`, `AddUserToGroupAdminAction` | -| Service accounts | `PUT /v3/add-service-account(s)`; `POST /v3/update-service-account`; `GET /v3/info-service-account`; `GET /v3/temporary-account-info`; `GET /v3/info-access-key`; `GET /v3/list-service-accounts`; `GET /v3/list-access-keys-bulk`; `DELETE /v3/delete-service-account(s)` | `service_account.rs` | create/update/list/temp-info/user-list/remove service account actions as checked in handler context | -| IAM import/export | `GET /v3/export-iam`; `PUT /v3/import-iam` | `user_iam.rs`, `user.rs` | `ExportIAMAction`, `ImportIAMAction` | -| IAM policies | `GET /v3/list-canned-policies`; `GET /v3/info-canned-policy`; `PUT /v3/add-canned-policy`; `DELETE /v3/remove-canned-policy`; `PUT /v3/set-user-or-group-policy`; `PUT /v3/set-policy`; `POST /v3/idp/builtin/policy/attach`; `POST /v3/idp/builtin/policy/detach`; `GET /v3/idp/builtin/policy-entities` | `policies.rs` | list/create/get/delete/attach policy actions; policy-entities combines list groups, users, and policies | -| Account info | `GET /v3/accountinfo` | `account_info.rs` | S3 action checks for account-scoped bucket and object probes | -| System info | `GET /v3/info`; `GET /v3/storageinfo`; `GET /v3/datausageinfo` | `system.rs` | `ServerInfoAdminAction`, `StorageInfoAdminAction`, `DataUsageInfoAdminAction` plus `ListBucketAction` for data usage | -| Metrics stream | `GET /v3/metrics` | `metrics.rs` through `system.rs` | Router credential presence plus handler credential check; no handler-level `AdminAction` is currently enforced | -| System service placeholders | `POST /v3/service`; `GET|POST /v3/inspect-data` | `system.rs` | Currently registered but handler returns `NotImplemented`; migration must preserve this unless behavior changes | -| Pools | `GET /v3/pools/list`; `GET /v3/pools/status`; `POST /v3/pools/decommission`; `POST /v3/pools/cancel` | `pools.rs` | list/status accept server-info or decommission; decommission/cancel use `DecommissionAdminAction` | -| Rebalance | `POST /v3/rebalance/start`; `GET /v3/rebalance/status`; `POST /v3/rebalance/stop` | `rebalance.rs` | `RebalanceAdminAction` | -| Heal | `POST /v3/heal/`; `POST /v3/heal/{bucket}`; `POST /v3/heal/{bucket}/{prefix}`; `POST /v3/background-heal/status`; `GET /v4/heal/replacement-recovery` | `heal.rs` | `HealAdminAction` | -| Tier | `GET /v3/tier`; `GET /v3/tier-stats`; `GET /v3/tier/{tier}`; `DELETE /v3/tier/{tiername}`; `PUT /v3/tier`; `POST /v3/tier/{tiername}`; `POST /v3/tier/clear` | `tier.rs` | `ListTierAction` for reads/status; `SetTierAction` for add/edit/remove/clear | -| Quota legacy and bucket-scoped | `PUT /v3/set-bucket-quota`; `GET /v3/get-bucket-quota`; `PUT|GET|DELETE /v3/quota/{bucket}`; `GET /v3/quota-stats/{bucket}`; `POST /v3/quota-check/{bucket}` | `quota.rs` | `SetBucketQuotaAdminAction` for writes; `GetBucketQuotaAction` for bucket-scoped reads/stats/checks | -| Bucket metadata | `GET /export-bucket-metadata`; `GET /v3/export-bucket-metadata`; `PUT /import-bucket-metadata`; `PUT /v3/import-bucket-metadata` | `bucket_meta.rs` | `ExportBucketMetadataAction`, `ImportBucketMetadataAction` | -| Server config | `GET /v3/get-config-kv`; `PUT /v3/set-config-kv`; `DELETE /v3/del-config-kv`; `GET /v3/help-config-kv`; `GET /v3/list-config-history-kv`; `DELETE /v3/clear-config-history-kv`; `PUT /v3/restore-config-history-kv`; `GET|PUT /v3/config` | `config_admin.rs` | `ConfigUpdateAdminAction` helper path; read/write handlers preserve current per-handler checks | -| Scanner | `GET /v3/scanner/status` | `scanner.rs` | `ServerInfoAdminAction` | -| Notification targets | `GET /v3/target/list`; `GET /v3/target/arns`; `PUT /v3/target/{target_type}/{target_name}`; `DELETE /v3/target/{target_type}/{target_name}/reset` | `event.rs` through `user_policy_binding.rs` | `GetBucketTargetAction` for list/ARNs; `SetBucketTargetAction` for put/delete | -| Audit targets | `GET /v3/audit/target/list`; `PUT /v3/audit/target/{target_type}/{target_name}`; `DELETE /v3/audit/target/{target_type}/{target_name}/reset` | `audit.rs` | `GetBucketTargetAction` for list; `SetBucketTargetAction` for put/delete | -| Module switches | `GET|PUT /v3/module-switches` | `module_switch.rs` | `ServerInfoAdminAction` for get; `ConfigUpdateAdminAction` for update | -| Plugin catalog | `GET /v4/plugins/catalog` | `plugins_catalog.rs` | `ServerInfoAdminAction` | -| Plugin instances | `GET /v4/plugins/instances`; `GET|PUT|DELETE /v4/plugins/instances/{id}` | `plugins_instances.rs` | read uses `GetBucketTargetAction`; write/delete use `SetBucketTargetAction` | -| Replication target list | `GET /v3/list-remote-targets` | `replication.rs` | Router credential presence plus handler credential check; no handler-level `AdminAction` is currently enforced | -| Replication target metrics/mutation | `GET /v3/replicationmetrics`; `PUT /v3/set-remote-target`; `DELETE /v3/remove-remote-target` | `replication.rs` | `GetReplicationMetricsAction` for metrics; `SetBucketTargetAction` for target mutation | -| Site replication | `PUT /v3/site-replication/add`; `PUT /v3/site-replication/remove`; `GET /v3/site-replication/info`; `GET /v3/site-replication/metainfo`; `GET /v3/site-replication/status`; `POST /v3/site-replication/devnull`; `POST /v3/site-replication/netperf`; `PUT /v3/site-replication/edit`; `PUT /v3/site-replication/peer/join`; `PUT /v3/site-replication/peer/bucket-ops`; `PUT /v3/site-replication/peer/iam-item`; `PUT /v3/site-replication/peer/bucket-meta`; `GET /v3/site-replication/peer/idp-settings`; `PUT /v3/site-replication/peer/edit`; `PUT /v3/site-replication/peer/remove`; `PUT /v3/site-replication/resync/op`; `PUT /v3/site-replication/state/edit` | `site_replication.rs` | add/remove/info/operation/resync actions selected per handler | -| Admin profiling | `GET /rustfs/admin/debug/pprof/profile`; `GET /rustfs/admin/debug/pprof/status` | `profile_admin.rs`, `profile.rs` | `ProfilingAdminAction` | -| TLS debug | `GET /rustfs/admin/debug/tls/status` | `tls_debug.rs`, `profile.rs` | `ProfilingAdminAction` via shared profile authorization | -| KMS legacy management | `POST /v3/kms/create-key`; `POST /v3/kms/key/create`; `GET /v3/kms/describe-key`; `GET /v3/kms/key/status`; `GET /v3/kms/list-keys`; `POST /v3/kms/generate-data-key`; `GET|POST /v3/kms/status`; `GET /v3/kms/config`; `POST /v3/kms/clear-cache` | `kms_management.rs`, `kms_keys.rs` | dedicated `kms:*` actions throughout; `kms:ServiceControl` for the status paths, `kms:Configure` for config, `kms:ClearCache` for cache. No `ServerInfoAdminAction` fallback remains on any KMS route | -| KMS dynamic control | `POST /v3/kms/configure`; `POST /v3/kms/start`; `POST /v3/kms/stop`; `GET /v3/kms/service-status`; `POST /v3/kms/reconfigure` | `kms_dynamic.rs` | `kms:Configure` for configure/reconfigure; `kms:ServiceControl` for start/stop/service-status | -| KMS keys | `POST /v3/kms/keys`; `DELETE /v3/kms/keys/delete`; `POST /v3/kms/keys/cancel-deletion`; `GET /v3/kms/keys`; `GET /v3/kms/keys/{key_id}` | `kms_keys.rs` | dedicated `kms:*` actions per handler | -| OIDC public | `GET /v3/oidc/providers`; `GET /v3/oidc/authorize/{provider_id}`; `GET /v3/oidc/callback/{provider_id}`; `GET /v3/oidc/logout` | `oidc.rs` | Public OIDC exception in `is_oidc_path` | -| OIDC config | `GET /v3/oidc/config`; `PUT|DELETE /v3/oidc/config/{provider_id}`; `POST /v3/oidc/validate` | `oidc.rs` | `ServerInfoAdminAction` for read/validate; `ConfigUpdateAdminAction` for mutation | - -## Table Catalog Routes - -The table catalog API is registered by the admin router but is not under -`/rustfs/admin`. It has its own prefix and Iceberg-style route shape. - -| Method | Path pattern | Handler | Authorization action | -|---|---|---|---| -| `GET` | `/iceberg/v1/config` | `GET_CONFIG_HANDLER` | `GetTableCatalogAction` | -| `GET` | `/iceberg/v1/{warehouse}/namespaces` | `LIST_NAMESPACES_HANDLER` | `GetTableNamespaceAction` | -| `POST` | `/iceberg/v1/{warehouse}/namespaces` | `CREATE_NAMESPACE_HANDLER` | `SetTableNamespaceAction` | -| `GET` | `/iceberg/v1/{warehouse}/namespaces/{namespace}` | `GET_NAMESPACE_HANDLER` | `GetTableNamespaceAction` | -| `DELETE` | `/iceberg/v1/{warehouse}/namespaces/{namespace}` | `DROP_NAMESPACE_HANDLER` | `DeleteTableNamespaceAction` | -| `GET` | `/iceberg/v1/{warehouse}/namespaces/{namespace}/tables` | `LIST_TABLES_HANDLER` | `GetTableAction` | -| `POST` | `/iceberg/v1/{warehouse}/namespaces/{namespace}/tables` | `CREATE_TABLE_HANDLER` | `CreateTableAction` | -| `POST` | `/iceberg/v1/{warehouse}/namespaces/{namespace}/register` | `REGISTER_TABLE_HANDLER` | `RegisterTableAction` | -| `GET` | `/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}` | `LOAD_TABLE_HANDLER` | `GetTableAction` | -| `POST` | `/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}` | `COMMIT_TABLE_HANDLER` | `CommitTableAction` | -| `DELETE` | `/iceberg/v1/{warehouse}/namespaces/{namespace}/tables/{table}` | `DROP_TABLE_HANDLER` | `DeleteTableAction` | - -## Migration Rules - -1. Pure move PRs may move handler modules, but must not change registered - methods, patterns, handler ownership, alias canonicalization, or public - exception behavior. -2. If an admin handler is wrapped to cut a dependency direction, the wrapper - must preserve the same `AdminAction` or `S3Action` check and keep response - compatibility unchanged. -3. Do not duplicate `/minio/admin` registrations. The alias remains a router - canonicalization concern. -4. Do not move table catalog routes under `/rustfs/admin` during route cleanup. -5. Registered-but-`NotImplemented` routes are behavior contracts too. Removing - or implementing them requires a behavior-change PR type. -6. Future route matrix automation should compare against this document and - `route_registration_test.rs` before crate extraction begins. +Every other admin route requires credentials at the router and a precise `AdminAction` or `S3Action` check in the handler (metrics routes, for example, authorize `GetMetricsAction`). The MinIO alias contract is specified in [minio-rustfs-router-compatibility.md](minio-rustfs-router-compatibility.md). diff --git a/docs/architecture/background-controller-contract.md b/docs/architecture/background-controller-contract.md index 8557e570f..768b4fac9 100644 --- a/docs/architecture/background-controller-contract.md +++ b/docs/architecture/background-controller-contract.md @@ -1,187 +1,55 @@ # Background Controller Contract -This document defines `BGC-002` for -[`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660). It turns -the background service inventory into a shared vocabulary for future read-only -status work. It does not add a Rust trait, a scheduler, a service registry, or -any worker start/stop behavior. +**Use this when:** you add a status snapshot or reconcile surface for a background service (scanner, heal, lifecycle, replication, config reload, capacity, metrics, memory observability, allocator reclaim, auto-tuner), or you are tempted to fold several of them into a generic controller. +**Source of truth:** the shipped reference surfaces — `MemoryObservabilityReconcilePlan` and `reconcile()` in `rustfs/src/memory_observability.rs`, `AllocatorReclaimControllerSnapshot` and `AllocatorReclaimReconcilePlan` in `rustfs/src/allocator_reclaim.rs`, `MetricsRuntimeReconcilePlan` in `crates/obs/src/metrics/scheduler.rs`. Startup and shutdown ordering is owned by [runtime-lifecycle.md](runtime-lifecycle.md); the plane-level overview is in [storage-control-data-plane.md](storage-control-data-plane.md). -## Scope +There is no `BackgroundController` trait, scheduler, or service registry. Each service exposes its own typed snapshot and reconcile plan; this page fixes the vocabulary and the rules those surfaces follow. -- PR type: `docs-only`. -- Baseline: `upstream/main` at - `f9a5e6d7e67322ac6f626b6f437a5e722fbe22e2`. -- Applies to future controller work for scanner, heal, lifecycle, replication, - dynamic config reload, capacity, metrics, memory observability, allocator - reclaim, and auto-tuning. -- Out of scope: worker creation, worker shutdown, queue resizing, storage - writes, readiness changes, peer signaling changes, scheduler replacement, and - crate splitting. +## Vocabulary -## Contract Vocabulary - -| Term | Meaning | BGC-002 boundary | +| Term | Meaning | Boundary | |---|---|---| -| Desired | Static intent from env, persisted config, module switches, feature flags, bucket config, or admin configuration. | Read only. Do not normalize or mutate config while collecting desired state. | -| Current | Observed local runtime state such as configured, disabled, running, degraded, stopping, or unknown. | Read only. Do not infer state by starting probes that create storage or network side effects. | -| Status | Human-readable and machine-checkable snapshot of runtime counters, worker counts, queue pressure, last successful cycle, last error, cancellation source, and shutdown handle shape. | Side-effect-free. Missing status surfaces must be reported as `unknown`, not guessed. | -| Reconcile | Future comparison between desired, current, and status that can produce a recommendation. | No action in `BGC-002`; future reconcile must not start or stop workers until a tested pilot PR allows it. | -| Side effects | Writes, deletes, queue admission, target activation, external I/O, metrics emission, readiness publication, peer signal, or config reload fanout. | Must be declared before any controller migration touches that service. | +| Desired | Static intent from env, persisted config, module switches, feature flags, bucket config, or admin configuration. | Read only; collecting desired state never normalizes or mutates config. | +| Current | Observed local runtime state: configured, disabled, running, degraded, stopping, or unknown. | Read only; never inferred by probes that create storage or network side effects. | +| Status | Machine-checkable snapshot of counters, worker counts, queue pressure, last cycle, last error, cancellation source, and shutdown-handle shape. | Side-effect-free; a missing surface is reported as `unknown`, never guessed. | +| Reconcile | Comparison of desired, current, and status that yields a plan. | Shipped plans only report; the only worker mutation they may request is `none`. | +| Side effects | Writes, deletes, queue admission, target activation, external I/O, metrics emission, readiness publication, peer signals, config reload fanout. | Declared per service before any controller touches it. | ## State Model -Future status snapshots should use the narrowest state that the current code can -prove: +Snapshots use the narrowest state the code can prove: | State | Meaning | Notes | |---|---|---| -| NotConfigured | No valid desired source exists for this service. | Use when config/module switches/features make the service absent. | -| Disabled | Desired source exists and explicitly disables the service. | Do not use for missing config. | -| Starting | Startup was requested and has not reached steady state. | Only expose when current code has a start boundary. | -| Running | The service is active according to existing runtime state. | Do not use merely because config is enabled. | -| Degraded | The service is active but current status exposes known error, partial, or stalled state. | Do not introduce new failure classification in docs-only work. | -| Stopping | Shutdown was requested and the service has not fully exited. | Only expose where shutdown can be observed. | -| Stopped | The service was started before and is now fully stopped. | Do not confuse with `Disabled` or `NotConfigured`. | -| Unknown | Current code lacks a safe status surface. | Preferred over speculative status. | - -## Lifecycle Boundary - -```mermaid -flowchart LR - D["Desired source"] - C["Current runtime state"] - S["Read-only status snapshot"] - R["Future reconcile recommendation"] - W["Workers and side effects"] - - D --> S - C --> S - S --> R - R -. "future tested pilot only" .-> W -``` - -`BGC-002` stops at the read-only contract. The arrow from reconcile to workers is -intentionally dotted because this PR does not allow any implementation to start, -stop, resize, or reconfigure workers. - -## Service Boundaries - -| Service area | Desired source | Current/status inputs | Side effects to preserve | -|---|---|---|---| -| Data scanner | Scanner env and runtime scanner config. | Admin scanner status, scanner metrics, scanner cancellation token, checkpoint/yield/alert counters. | Data usage cache updates, lifecycle evaluation, replication heal admission, scanner heal admission, alerts, and scanner metrics. | -| Heal/AHM | Heal enablement and scanner-driven heal admission. | Heal manager global channel, active task atomics, queue length atomics, AHM cancellation token. | Heal queue consumption, heal storage writes, and channel close semantics. | -| Lifecycle expiry/transition | Bucket lifecycle config and scanner event source. | Lifecycle worker counts, active tasks, queue send timeouts, transition stats, expiry/transition queues. | Object deletes, transition queueing, stale multipart cleanup, and lifecycle metrics. | -| Replication pool | Bucket/site replication config and resync admin requests. | Global replication stats, worker pool sizes, queue counters, persisted resync state, per-bucket cancel tokens. | Object replication, delete replication, queue resizing by channel close, persisted resync metadata, and admin-triggered cancel paths. | -| Dynamic config reload | Persisted server config, admin config calls, and peer snapshot signals. | Last local reload result, per-subsystem reload errors, peer reload signal result. | Scanner/heal runtime config updates, audit reload, notification reload, peer signaling, and config snapshot fanout. | -| Capacity manager | Local disk inventory and capacity feature state. | Capacity manager cache age, scheduled refresh state, last refresh result, runtime summary loop. | Global capacity cache refresh and runtime summary metrics/logging. | -| Metrics runtime | Observability metrics feature state and collector configuration. | Collector intervals, last collection result, cancellation token state, collector grouping. | Metrics collection and emission only. | -| Memory observability | Observability feature state and memory sampling config. | Sampler loop state, last sample time, last sample error, runtime cancellation token. | Memory metric emission. This is the preferred first BGC-003 status candidate. | -| Allocator reclaim | Allocator reclaim env/config and backend support. | Enabled flag, idle streak, active request gauge, scanner/heal activity gauges, last reclaim result. | Backend-specific allocator reclaim and metrics. | -| Auto-tuner | `RUSTFS_AUTOTUNER_ENABLED` and tuning inputs. | Last tuning attempt, last tuning error, 60-second loop state. | Runtime concurrency tuning. Treat as behavior-sensitive. | - -The following areas stay outside the first controller migrations: - -- deferred IAM recovery, because it can publish readiness; -- optional protocol servers, because they already have protocol shutdown handles; -- ECStore endpoint monitor and disk health monitor, because they are storage- - adjacent and can affect disk state; -- notification and audit runtime coupling, because live streams, replay, target - activation, and reload behavior need dedicated preservation tests. +| NotConfigured | No valid desired source exists. | Config, module switches, or features make the service absent. | +| Disabled | A desired source exists and explicitly disables the service. | Not for missing config. | +| Starting | Start requested, steady state not reached. | Only where a start boundary exists. | +| Running | Active according to existing runtime state. | Not merely because config is enabled. | +| Degraded | Active with known error, partial, or stalled status. | No new failure classification is invented for a snapshot. | +| Stopping | Shutdown requested, not fully exited. | Only where shutdown is observable. | +| Stopped | Started earlier, now fully stopped. | Distinct from `Disabled` and `NotConfigured`. | +| Unknown | No safe status surface exists. | Preferred over speculation. | ## Read-Only Snapshot Requirements -Any future `BGC-003` status implementation must satisfy all of these: +- Status collection never starts, stops, resizes, or wakes a worker. +- Status collection never writes storage data, object metadata, target state, queue entries, persisted config, or resync metadata. +- Status collection never publishes readiness or peer reload signals. +- Missing fields are `unknown` or omitted with a documented reason. +- Cancellation source and shutdown-handle shape are reported separately from desired enabled/disabled state. +- Repeated `reconcile` calls over the same snapshot return the same plan. +- Scanner, heal, lifecycle, and replication status must not hide their queue and admission coupling. -- status collection must not start, stop, resize, or wake a worker; -- status collection must not write storage data, object metadata, target state, - queue entries, persisted config, or resync metadata; -- status collection must not publish readiness or peer reload signals; -- missing fields must be represented as `unknown` or omitted with a documented - reason; -- cancellation source and shutdown handle shape must be reported separately from - desired enabled/disabled state; -- scanner, heal, lifecycle, and replication status must not hide their queue and - admission coupling. +## Coupling Notes -## BGC-003 Snapshot Pilot +The services below share state or shutdown contracts and must not be folded into a generic controller without service-specific preservation tests: -The first read-only snapshot is memory observability status. It reports the -service name, whether observability metrics currently enable the sampler, the -configured sampler interval, runtime-token cancellation state, and the absence -of a dedicated shutdown handle. - -This snapshot intentionally does not define an admin route, scheduler, service -registry, worker start/stop path, readiness signal, peer signal, storage write, -or metrics emission change. - -## BGC-004 Controller Pilot - -The first controller pilot is also memory observability. It converts the -existing desired inputs and status snapshot into a typed reconcile plan. The -pilot reports desired state, current state, and worker mutation intent. - -The only allowed worker mutation for this pilot is `none`. Repeated reconcile -calls must return the same plan for the same snapshot and must not request a -worker start, stop, resize, wakeup, storage write, readiness signal, peer -signal, or metrics emission. - -## BGC-005 Allocator Reclaim Status And Controller Surface - -The second low-risk controller/status surface is allocator reclaim. It reports -the service name, desired enablement, configured force flag, backend-specific -effective force, idle interval settings, runtime-token cancellation state, and -the absence of a dedicated shutdown handle. - -The only allowed worker mutation for this surface is `none`. Reconcile output is -read-only and must not start, stop, resize, wake, or otherwise drive the -allocator reclaim loop. Existing backend-specific force handling, idle-streak -logic, metrics emission, and runtime-token shutdown behavior remain owned by the -current loop. - -## BGC-006 Metrics Runtime Status And Controller Surface - -The third low-risk controller/status surface is metrics runtime. It reports the -service name, observability metrics enablement, collector task count, configured -collector intervals, replication bandwidth zero-tombstone cycle count, -runtime-token cancellation state, and the absence of a dedicated shutdown -handle. - -The only allowed worker mutation for this surface is `none`. Reconcile output is -read-only and must not start, stop, resize, wake, or otherwise drive metrics -collector tasks. Existing collector grouping, interval parsing, metrics -emission, replication bandwidth tombstone handling, and runtime-token shutdown -behavior remain owned by the current loops. - -## Future Reconcile Rules - -Future reconcile work is allowed only after a read-only status snapshot exists. -The first reconcile pilot must: - -- choose one low-risk service; -- compare desired/current/status without side effects; -- prove idempotence under repeated calls; -- prove no duplicate workers are created; -- preserve existing shutdown order and cancellation source; -- include rollback guidance that removes the pilot without changing existing - worker behavior. - -Memory observability is the recommended first candidate because it already has a -simple runtime cancellation loop and no storage writes. Scanner, heal, -replication, lifecycle, disk health, deferred IAM recovery, and auto-tuning must -wait for focused preservation tests. - -## Verification Expectations - -For this docs-only contract: - -- architecture migration guard scripts must pass; -- layer dependency and metrics reference guards must pass; -- no Rust source, Cargo metadata, CI workflow, Makefile, or runtime config file - may change. - -For the next implementation PRs: - -- add focused tests before changing behavior; -- do not modify production logic only to make tests pass; -- keep compatibility comments searchable with `RUSTFS_COMPAT_TODO()` - whenever temporary old paths are retained for later deletion. +- Scanner implies heal: the loop started by `init_data_scanner` (`rustfs/src/startup_lifecycle.rs`) enqueues heal work, so scanner status must separate scheduler state from work-source accounting. +- Heal/AHM owns its own token: `create_ahm_services_cancel_token` and `init_heal_manager` run in `rustfs/src/startup_background.rs`; `shutdown_ahm_services` runs in `rustfs/src/startup_shutdown.rs`. Heal admission and channel-close semantics stay intact. +- Replication has two shutdown contracts: the pool started by `init_background_replication` (`rustfs/src/startup_storage.rs`) stops workers by closing channels, while resync started by `init_resync` (`rustfs/src/startup_bucket_metadata.rs`) uses cancellation tokens, and admin-triggered resync uses per-bucket tokens. +- Lifecycle expiry, transition, and stale-multipart cleanup are started by `ECStore::init` (`init_background_expiry`, `init_background_stale_multipart_upload_cleanup` in `crates/ecstore/src/store/init.rs`), which binds the runtime token through `bind_background_cancel_token`; the scanner is their event source, so they are not a separate periodic controller. +- Notification and audit share a runtime pattern but not a lifecycle: `init_event_notifier` and `start_audit_system` (`rustfs/src/startup_audit.rs`), `shutdown_event_notifier` and `stop_audit_system` (`rustfs/src/startup_shutdown.rs`). Live event streams stay separate from target-delivery enablement. +- Dynamic config reload is admin-triggered fanout (`apply_dynamic_config_for_subsystem`, `signal_dynamic_config_reload`, `signal_config_snapshot_reload` in `rustfs/src/admin/service/config.rs`), not a loop; per-subsystem validation and error boundaries are preserved. +- Capacity refresh tasks are owned through `CapacityBackgroundTasks` returned by `init_capacity_management_managed` (`rustfs/src/capacity/capacity_integration.rs`, called from `rustfs/src/startup_entrypoint.rs`); scheduled interval defaults and singleflight refresh stay unchanged. +- Storage-adjacent monitors (`monitor_and_connect_endpoints` in `crates/ecstore/src/core/sets.rs`, `enable_health_check` in `crates/ecstore/src/disk/disk_store.rs`) change disk state and stay outside controller work. +- Deferred IAM recovery (`spawn_iam_recovery_task`, `rustfs/src/startup_iam.rs`) publishes readiness; optional protocol servers already own `ShutdownHandle`s; the auto-tuner (`init_auto_tuner` in `rustfs/src/init.rs`) changes runtime concurrency. All three stay outside generic controllers. diff --git a/docs/architecture/background-services-inventory.md b/docs/architecture/background-services-inventory.md deleted file mode 100644 index d484a1004..000000000 --- a/docs/architecture/background-services-inventory.md +++ /dev/null @@ -1,89 +0,0 @@ -# Background Services Inventory - -This document records the current background service surface before -BackgroundController work. It is a behavior-preservation inventory only; it does -not define a new scheduler, controller framework, or shutdown contract. - -## Scope - -- Related migration task: `BGC-001`. -- PR type: `docs-only`. -- Baseline: `upstream/main` at - `03eb10b07f5f968c531151ae667dfe218050493d`. -- Out of scope: changing startup order, shutdown order, readiness, storage - writes, heal admission, scanner scheduling, replication queues, config reload - behavior, metrics intervals, or worker counts. - -## Startup And Shutdown Owners - -| Area | Startup owner | Shutdown owner | Current cancellation source | -|---|---|---|---| -| Main runtime token | `rustfs/src/main.rs::run` creates `ctx` after HTTP listeners start and before ECStore creation. | `rustfs/src/main.rs::handle_shutdown` calls `ctx.cancel()` before service-specific shutdown. | Shared `tokio_util::sync::CancellationToken`. | -| Scanner | `rustfs/src/main.rs::run` calls `init_data_scanner(ctx.clone(), store.clone())` after successful startup log and global init time. | Main shutdown calls `ctx.cancel()`; if scanner was enabled it also calls `shutdown_background_services()`. | Scanner loop receives the main runtime token. | -| Heal/AHM | Main creates `create_ahm_services_cancel_token()` before scanner/heal feature checks and calls `init_heal_manager(...)` when heal or scanner is enabled. | Main shutdown calls `shutdown_ahm_services()` when heal or scanner was enabled. | Global AHM token plus channel/worker-local state. | -| Replication pool | Main calls `init_background_replication(store.clone())` after global config init, then `pool.init_resync(ctx.clone(), buckets.clone())` after bucket listing. | No direct main shutdown call for the replication pool; resync receives the main runtime token. | Resync routine uses the main runtime token; per-bucket resync uses registered cancel tokens. | -| Lifecycle expiry/transition | `ECStore::init` calls `init_background_expiry(self.clone())` and `init_background_stale_multipart_upload_cleanup(self.clone())`. | Expiry workers read `get_background_services_cancel_token()` and fall back to a private token if none exists. Stale multipart cleanup exits when the weak ECStore reference cannot upgrade. | `ECStore::init` binds the main runtime token into the instance context with `bind_background_cancel_token(ctx)` before expiry starts, so the private-token fallback is a defensive path rather than the normal one. | -| Notification runtime | Main calls `init_event_notifier()` after buffer profile init. | Main shutdown calls `shutdown_event_notifier().await`. | Notification runtime owns target/replay shutdown internally. | -| Audit runtime | Main calls `start_audit_system().await`. | Main shutdown calls `stop_audit_system().await`. | Audit runtime owns target/replay shutdown internally. | -| Metrics and memory loops | Main calls `init_metrics_runtime(ctx.clone())`, `init_memory_observability(ctx.clone())`, and `init_auto_tuner(ctx.clone())` when observability metrics are enabled. | Main shutdown only cancels the shared runtime token. | Shared runtime token. | -| Allocator reclaim | Main calls `init_allocator_reclaim(ctx.clone())` unconditionally. | Main shutdown cancels the shared runtime token. | Shared runtime token. | -| Capacity manager | Main calls `init_capacity_management().await` before HTTP listener startup and ECStore creation. | No direct main shutdown call. | Current scheduled capacity and metrics loops do not receive a shutdown token. | -| Optional protocol servers | Main calls feature-gated FTP, FTPS, WebDAV, and SFTP init functions. | Main shutdown calls each stored `ShutdownHandle` and waits for all protocol shutdown futures. | Per-protocol broadcast shutdown handles. | -| Deferred IAM recovery | `bootstrap_or_defer_iam_init(...)` may spawn a deferred recovery loop. | Main shutdown cancels the shared runtime token. | Shared runtime token. | - -## Service Inventory - -| Service | Trigger and workers | Side effects | Status and metrics | Migration notes | -|---|---|---|---|---| -| Capacity background refresh | `rustfs/src/capacity/capacity_integration.rs::init_capacity_management` delegates to `init_capacity_management_for_local_disks`, then `crates/object-capacity/src/capacity_manager.rs::start_background_task` spawns a scheduled refresh loop and a runtime summary loop. | Refreshes global capacity cache from local disks and logs runtime summaries. | Uses the object-capacity manager state and log summaries; no explicit shutdown status surface is exposed here. | Add read-only status before any controller migration. A future controller must not change scheduled interval defaults or singleflight refresh behavior. | -| ECStore endpoint monitor | `crates/ecstore/src/core/sets.rs::new` spawns `monitor_and_connect_endpoints`. | Monitors endpoint connectivity and reconnect behavior for erasure sets. | Logs monitor start, cancellation, and exit. | This is storage-adjacent and must stay outside broad controller movement until storage shutdown semantics are explicitly covered. | -| Local disk health monitor | `crates/ecstore/src/store/init.rs::init` enables disk health checks after store initialization; `crates/ecstore/src/disk/disk_store.rs::enable_health_check` spawns writable and recovery monitors. | Periodically probes disk writability, can create test objects named `health-check-*`, and updates disk runtime health state. | Disk info includes runtime health metrics and waiting counts. | Do not merge this with scanner/heal controller work; probes affect disk health semantics. | -| Data scanner | `crates/scanner/src/scanner.rs::init_data_scanner` configures scanner defaults, applies runtime config, waits the initial scanner delay, then loops `run_data_scanner`. | Updates data usage cache, scans buckets/sets, evaluates lifecycle rules, queues replication heal, queues scanner heal, and emits scanner alerts. | Scanner runtime config/status is exposed through admin scanner status; scanner metrics record ILM, replication admission, heal admission, checkpoints, yields, and alerts. | Scanner implies heal because scanner can enqueue heal requests. Future controller status must separate scheduler state from scanner work-source accounting. | -| Heal/AHM | `crates/heal/src/lib.rs::init_heal_manager` starts `HealManager`, initializes the shared heal channel, and spawns `HealChannelProcessor`. | Consumes heal requests from the global heal channel and drives heal work through the configured heal storage API. | Global active-task and queue-length atomics track current heal pressure. | Keep heal admission and channel semantics intact. Controller work should first expose queue/active status and shutdown state. | -| Bucket replication pool | `crates/ecstore/src/bucket/replication/replication_pool.rs::init_background_replication` creates global replication stats and the global pool; pool resizing spawns regular, large-object, and failed-object workers. | Replicates object and delete operations, updates queue stats, and maintains replication worker pools. | Replication stats expose active worker counts and queue accounting. | Worker resize behavior currently closes channels to stop workers. Do not replace this with a generic controller until queue close semantics are captured by tests. | -| Bucket replication resync | Main calls `get_global_replication_pool().init_resync(ctx.clone(), buckets.clone())`; the pool spawns `start_resync_routine`. Admin site-replication handlers can start or cancel per-bucket resync with dedicated tokens. | Loads persisted resync state, starts bucket resync, persists status, and can cancel per-target resync. | Admin site-replication status surfaces resync state. | Preserve the split between startup resync and admin-triggered resync operations. | -| Lifecycle expiry and transition | `ECStore::init` calls `init_background_expiry(self.clone())`. Scanner evaluates lifecycle events and queues expiry/transition work through `apply_expiry_rule` and `apply_transition_rule`. | Deletes expired objects, queues transitions, updates lifecycle stats, and accounts scanner ILM actions only when work is queued. | Lifecycle state tracks worker counts, active tasks, queue send timeouts, compensation tasks, and transition stats. | This is not a separate periodic controller today; scanner is the main event source for object lifecycle evaluation. | -| Stale multipart cleanup | `ECStore::init` calls `init_background_stale_multipart_upload_cleanup(self.clone())`. | Periodically deletes stale multipart upload data. | Logs cleanup passes when objects are deleted. | Current loop has no explicit cancellation token and exits when ECStore is dropped. Future controller work needs an explicit lifecycle decision before changing it. | -| Notification runtime | `rustfs/src/server/event.rs::init_event_notifier` initializes live event stream support even when notification targets are disabled; when enabled, it loads server config and activates targets. Config reload uses `NotificationConfigManager::reload_config`. | Installs ECStore event dispatch hook, activates notification targets, manages replay/runtime target state, and supports live event streams. | Notification module state is refreshed from persisted module switches; target health is available through runtime target status. | Keep live event stream support separate from target delivery enablement. Reload must remain admin-triggered and peer-signaled. | -| Audit runtime | `rustfs/src/server/audit.rs::start_audit_system` starts audit only when module switches and configured targets allow it. `AuditSystem::reload_config` replaces runtime targets. | Dispatches audit events to configured targets and manages replay workers. | Audit observability records config reloads and target delivery metrics. | Do not couple audit lifecycle to notification lifecycle even though the runtime patterns are similar. | -| Dynamic config reload | Admin config handlers call `apply_dynamic_config_for_subsystem`, then `signal_dynamic_config_reload` or `signal_config_snapshot_reload` through the global notification system. | Applies scanner/heal runtime config, audit reloads, notification reloads, and peer reload signals. | Logs local and peer reload failures. Audit reload increments audit config reload metrics. | This is admin-triggered fanout, not a background scheduler. Controller work should preserve per-subsystem validation and error boundaries. | -| Metrics runtime | `crates/obs/src/metrics/scheduler.rs::init_metrics_runtime` spawns multiple interval loops for cluster, bucket, node, resource, audit, notification, and replication bandwidth metrics. | Periodically collects and reports metrics. | Reports through the metrics runtime, logs cancellation warnings, and exposes a typed read-only status snapshot plus a no-op reconcile plan for enablement, collector task count, intervals, replication bandwidth tombstone cycles, cancellation source, and shutdown handle shape. | Keep intervals and collector grouping stable. The current controller surface does not mutate workers. | -| Memory observability | `rustfs/src/memory_observability.rs::init_memory_observability` spawns a token-cancelled sampler. | Periodically records memory snapshots. | Emits memory observability metrics and exposes a read-only status snapshot plus a no-op reconcile plan for metrics enablement, interval, cancellation source, and shutdown handle shape. | This is the first low-risk pilot for controller status because it already has a simple token loop and the pilot does not mutate workers. | -| Allocator reclaim | `rustfs/src/allocator_reclaim.rs::init_allocator_reclaim` spawns a token-cancelled reclaim loop when enabled. | Observes reclaimable work and may run allocator reclaim after idle intervals. | Emits reclaim enabled/backend counters, active-request gauges, scanner/heal activity gauges, and reclaim result counters. Exposes a typed read-only status snapshot plus a no-op reconcile plan for enablement, backend, effective force, intervals, cancellation source, and shutdown handle shape. | A controller must preserve idle-streak logic and backend-specific force behavior. The current controller surface does not mutate workers. | -| Auto-tuner | `rustfs/src/init.rs::init_auto_tuner` optionally spawns a 60-second loop when `RUSTFS_AUTOTUNER_ENABLED` is true. | Tunes concurrency manager settings from performance metrics. | Logs iteration success/failure. | Treat as behavior-sensitive; a future controller needs explicit rollback because it can change runtime concurrency. | -| Update check | `rustfs/src/init.rs::init_update_check` spawns one async task with a 30-second timeout when update checks are enabled. | Performs version check network I/O and logs available updates. | Logs result only. | This is a one-shot task, not a controller candidate for the first BGC PRs. | -| Deferred IAM recovery | `rustfs/src/startup_iam.rs::spawn_iam_recovery_task` retries IAM init with backoff and finalizes readiness when successful. | Can initialize IAM later, initialize AppContext if needed, mark `IamReady`, and publish `FullReady`. | Readiness state reflects deferred recovery progress. | Keep this lifecycle-critical path separate from generic background controllers. | -| Optional protocol servers | `rustfs/src/init.rs` starts FTP, FTPS, WebDAV, and SFTP with per-protocol `ShutdownHandle`s when features and config enable them. | Serve protocol traffic in background tasks. | Shutdown logs per protocol. | Protocol servers already have explicit handles; do not fold them into BGC until the service registry owns shutdown ordering. | - -## Current Gaps To Preserve Before Controller Work - -- ECStore background-service cancellation has a public global token API, but this - inventory found no current startup call to create that token. Lifecycle expiry - workers therefore use their fallback token when no global token exists. -- Capacity manager loops do not receive the main runtime cancellation token. -- Replication worker pools stop some workers by closing channels, while resync - uses cancellation tokens. These are different shutdown contracts. -- Scanner, lifecycle, replication, and heal are coupled by work queues and - metrics. Moving one without status snapshots for the others risks hiding work - admission failures. -- Dynamic config reload is admin-triggered and peer-signaled, not a periodic - background loop. - -## BGC-002 Contract Inputs - -These inputs are formalized in -[`background-controller-contract.md`](background-controller-contract.md). - -Future controller contract work should start with a read-only shape: - -- `desired`: enabled/disabled plus static config source. -- `current`: started, stopped, running, degraded, or disabled. -- `status`: worker counts, queue lengths, last cycle/reload time, and last error. -- `shutdown`: cancellation source and whether the service has an explicit stop - handle. -- `side_effects`: storage writes, target activation, external I/O, metrics, and - readiness changes. - -The first pilot should use a service with an existing simple cancellation loop -and no storage writes, such as memory observability. Scanner, heal, replication, -lifecycle, and disk health must wait for focused preservation tests. diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 7e4af6c47..d598bd553 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -1,8 +1,7 @@ # Compatibility Cleanup Register -Use this file to track temporary compatibility code introduced by architecture -migration PRs. Entries are required only for compatibility paths that are planned -for later deletion. +**Use this when:** you add, review, or remove a temporary compatibility path (fallback, wrapper, re-export, legacy codec) and need the required marker and its removal condition. +**Source of truth:** the `RUSTFS_COMPAT_TODO()` source markers, matched in both directions against `## Open Items` by `scripts/check_architecture_migration_rules.sh`. Entries exist only for compatibility paths planned for later deletion. ## Required Source Marker @@ -14,7 +13,7 @@ for later deletion. - `tokio-tar-extension-limits` bounded archive parser hardening: Snowball extraction depends on per-entry and cumulative GNU long-name, GNU long-link, and PAX extension limits; physical-entry, GNU sparse-map, and sparse-continuation limits; cancellation-safe sparse parsing; and fused entry streams after parser errors. The released tokio-tar API does not provide this complete boundary. Keep the reviewed fork pin until astral-sh/tokio-tar#118 is merged and one published tokio-tar release contains every listed capability with the Snowball regression fixtures passing against that release. - `backlog-2102` rc.2/rc.3 empty scanner usage floor recovery: old DeleteBucket cleanup could synthesize an empty incomplete v2 usage primary/backup before leadership added an epoch, while newer scanners require a durable authoritative baseline identity. New scanners recognize only that exact serialized empty-fence shape, preserve its epoch through a CAS-protected recovery marker, and rebuild namespace coverage without treating zero usage as authoritative. Remove this recovery path and marker after rc.2 and rc.3 are no longer supported direct-upgrade sources. -- `s3gate-metadata-xml` persisted bucket XML migration: mixed-version site-replication peers, retained `.metadata.bin` objects, and backup archives can all carry XML written by the s3s codec, so the gateway migration must keep the legacy codec available until every stored form has crossed a verified rewrite boundary. Remove the legacy s3s parser and serializer only after the minimum supported direct-upgrade release reads and writes every persisted XML configuration family through the gateway codec, the four-way D1-D5 gate has remained clean for one full support window, every supported mixed-version site-replication topology has completed its writer upgrade, and migration tooling has verified or rewritten every retained bucket metadata object and restorable backup archive. +- `s3gate-metadata-xml` persisted bucket XML migration: mixed-version site-replication peers, retained `.metadata.bin` objects, and backup archives can all carry XML written by the s3s codec, so the gateway migration must keep the legacy codec available until every stored form has crossed a verified rewrite boundary. Remove the legacy s3s parser and serializer only after the minimum supported direct-upgrade release reads and writes every persisted XML configuration family through the gateway codec, every supported mixed-version site-replication topology has completed its writer upgrade, and migration tooling has verified or rewritten every retained bucket metadata object and restorable backup archive. - `rustfs-6339` legacy bucket policy ID casing: earlier RustFS releases persisted the top-level policy identifier as "ID", while current writes use the S3-compatible "Id" spelling. Readers accept both spellings so retained bucket metadata remains usable after upgrade. Remove the legacy alias after migration tooling has rewritten every retained bucket policy using "ID". - `table-publication-fence-v1` table publication fencing: nodes that predate table and table-bucket publication fences can mutate live files while a new node is publishing a catalog pointer. New nodes retain exact object guards until the operator confirms that every serving node uses the new fences. Fleet confirmation also requires non-overlapping active warehouse prefixes and lifecycle workers that exclude table buckets. Remove the exact live-file fallback and the fleet-confirmation gate after the minimum supported RustFS release acquires table fences for registered-table mutations and table-bucket fences for unresolved-prefix mutations. - `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: version 1 writes continue during mixed-version rollout until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2. @@ -29,7 +28,7 @@ for later deletion. - `rustfs-5416-zero-retry-delay` startup retry-delay validation: releases before bounded topology convergence accept RUSTFS_STARTUP_TOPOLOGY_RETRY_MAX_DELAY values of 0 or 0ms. New servers replace those values with the safe nonzero default so a direct upgrade neither fails startup nor enters a busy loop. Reject zero after the minimum supported direct-upgrade release validates or rewrites this setting before rollout. - `scanner-usage-v2` persisted scanner usage migration: pre-v2 scanners write `.usage.json`, so upgraded clusters read that primary/backup pair only while `.usage.v2.json` is absent and continue removing deleted buckets from legacy copies that still exist. The additive usage_snapshot_complete field in `.usage.v2.json` must remain optional while mixed-version clusters are supported; a missing field means the snapshot is not authoritative. The legacy read also feeds the degraded quota-admission baseline (issue #5716): while no authoritative usage exists, quota checks admit against the pre-discard sizes of the last loaded snapshot, including a legacy one. Remove the legacy object fallback and cleanup only after every supported direct-upgrade source writes `.usage.v2.json`; the baseline then feeds from incomplete v2 snapshots alone. - `ns-scanner-rpc-v3` namespace scanner capability and activity handshake: old peers and legacy internode transports lack the authenticated startup-epoch handshake. The oldest peers send an empty activity request and receive a field-empty protocol-0 response. Protocol v4 binds the challenge and response topology but cannot authenticate distributed dirty-usage state. Protocol v5 binds the request version, acknowledgement target and generation, and the response dirty-usage state, but predates set-scoped scanner cache locks. Protocol v6 additionally fences scanner cache lock-domain changes. Current protocol v7 binds the storage-owned movement generation and publication-blocked state, so distributed scanner cycles publish usage only after every peer reports a complete v7 activity proof; v6 responses remain readable but are treated as unverified for publication. Servers retain protocol-0, protocol-v4, and protocol-v6 codecs alongside the current v7 codec for rolling upgrades, while protocol-v5 peers are treated as previous-version peers that cannot safely participate in the new cache lock domain. Scanner selection treats HTTP 404/405/426 and the legacy MethodNotAllowed default as an explicit lack of remote scanner v3 support and assigns those disks to coordinator-driven workers; transient capability failures remain incomplete and do not activate the fallback. Remove the coordinator fallback after the minimum supported RustFS peer version implements namespace scanner protocol v3, remove protocol-0 activity requests and responses after every supported peer implements authenticated scanner activity protocol v4, remove the protocol-v4 activity codec after every supported peer implements protocol v5, and remove protocol-v5 previous-version rejection after every supported peer implements protocol v6; future protocol revisions must keep the same dual-version server/codec window before changing the advertised version. -- `#4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability. +- `rustfs-4648` walk-dir stream completion capability: old clients can append fallback output to an already-used metacache writer after a terminal body error, so servers emit terminal walk errors only to clients that sign the `walk_dir_stream_completion=error-v1` query capability and its request-body digest. Remove the legacy clean-EOF path after the minimum supported RustFS peer version always advertises this capability. - `heal-rpc-auth-v2` internode gRPC authentication: servers temporarily accept legacy prefix signatures so old peers remain available during rolling upgrades. Remove the legacy fallback after the minimum supported RustFS peer version sends v2 authentication on every internode gRPC request. - `put-file-auth-epoch-strict` internode put_file epoch compatibility: rc.2 peers can cache a remote put_file capability before that remote node restarts, then continue sending v1 authenticated uploads with the old server epoch; those peers cannot recover from the 409 conflict used by newer clients to trigger a re-probe. Servers temporarily accept signed, non-nil stale put_file epochs while legacy put_file auth remains non-strict so mixed-version rolling upgrades can finish multipart/object writes. Remove the stale-epoch fallback after the minimum supported RustFS peer version re-probes put_file capability after server-epoch conflicts and legacy put_file auth is no longer accepted. - `disk-mutation-body-digest` internode mutating disk RPCs: servers temporarily accept mutating disk RPCs (RenameData, DeleteVersion, DeleteVersions, WriteMetadata, UpdateMetadata, WriteAll, Delete, DeletePaths, RenameFile, RenamePart, DeleteVolume, MakeVolume, MakeVolumes) that carry no signature-bound canonical body digest, so peers from releases that predate body-digest signing remain available during rolling upgrades. Accepted digestless mutations increment the internode body-digest fallback counter; that counter must read zero fleet-wide across a release window before RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT is enabled. Because body-bound requests now consume replay-cache nonces on the receiver, deploy the raised RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY default fleet-wide before enabling strict mode, and watch the internode replay-cache overflow counter for undersized capacity during the rollout. Remove the digestless fallback after the minimum supported RustFS peer version body-binds every mutating disk RPC. diff --git a/docs/architecture/config-model-boundary-adr.md b/docs/architecture/config-model-boundary-adr.md index 6fe7cbb00..74e2199b6 100644 --- a/docs/architecture/config-model-boundary-adr.md +++ b/docs/architecture/config-model-boundary-adr.md @@ -1,185 +1,49 @@ # Config Model Boundary ADR -Related issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) - -Task: `CFG-002` +**Use this when:** you touch the server-config model (`Config`, `KV`, `KVS`) or its persistence, or you need to know which crate owns which part of server configuration. +**Source of truth:** `crates/config/src/server_config.rs` (model, default registration, process-global snapshot) and `crates/ecstore/src/config/` (`ConfigSys`, persistence, migration, storage-class runtime state). ## Decision -Use the existing `crates/config` package (`rustfs-config`) as the target owner -for the pure server-config model. Do not create a new config-model crate for -the first extraction. +`rustfs-config` (`crates/config`) owns the pure server-config model and the process-global server-config snapshot. ECStore keeps config persistence, migration, default-registration wiring, startup initialization, and storage-class runtime state. There is no separate config-model crate, and `rustfs_ecstore::config` does not re-export the model or the snapshot accessors. -The next model extraction PR should introduce the model under: - -```text -crates/config/src/server_config.rs -``` - -The exported path should be: - -```rust -rustfs_config::server_config::{Config, KV, KVS} -``` - -The extraction kept the existing path available through a temporary -compatibility re-export: - -```rust -rustfs_ecstore::config::{Config, KV, KVS} -``` - -That re-export included `RUSTFS_COMPAT_TODO(CFG-004)` and a matching entry in -[`compat-cleanup-register.md`](compat-cleanup-register.md) until the model -consumers were migrated. The CFG-004 cleanup removed this old model path after -code scans showed consumers import the model directly from `rustfs-config`. - -Follow-up `CFG-008` moved the process-global server-config snapshot accessors -to `rustfs_config::server_config` after the model path stabilized. Its temporary -`rustfs_ecstore::config::{get_global_server_config, set_global_server_config}` -compatibility re-export was removed after in-repo runtime consumers migrated to -the `rustfs-config` owner. +Import path: `rustfs_config::server_config::{Config, KV, KVS}`. The model sits behind the `server-config-model` feature of `rustfs-config` (`crates/config/Cargo.toml`), which enables `serde` and `serde_json`. ## Why `rustfs-config` -`rustfs-config` is already the lowest RustFS crate for configuration constants -and subsystem identifiers used by ECStore, notify, audit, targets, scanner, IAM, -and admin code. The current `ecstore::config::{Config, KV, KVS}` model already -uses `rustfs-config` constants, so moving the pure model upward to -`rustfs-config` cuts the wrong dependency direction without adding another crate. +- It is already the lowest RustFS crate for configuration constants and subsystem identifiers used by ECStore, notify, audit, targets, scanner, IAM, and admin code, and the model needs only those constants. +- Moving the model upward removes the wrong-direction dependency (outer crates importing ECStore for a plain data type) without adding another crate or a second config namespace. -Creating a new crate now would add a second config namespace before consumers -are migrated. That would increase re-export and compatibility surface while not -removing any storage or runtime dependency by itself. +## Ownership -## Allowed Dependencies +| Item | Owner | Notes | +|---|---|---| +| `KV`, `KVS`, `Config` and their methods (`get_value`, `set_defaults`, `marshal`, `unmarshal`, `merge`) | `crates/config/src/server_config.rs` | Pure data model with serde roundtrip | +| `DEFAULT_KVS`, `register_default_kvs` | `crates/config/src/server_config.rs` | Registration surface; ECStore still calls it from `init()` in `crates/ecstore/src/config/mod.rs` | +| `GLOBAL_SERVER_CONFIG`, `get_global_server_config`, `set_global_server_config` | `crates/config/src/server_config.rs` | Process-global snapshot accessors | +| `ConfigSys`, `init()`, `try_migrate_server_config` | `crates/ecstore/src/config/mod.rs` | Startup order and caller unchanged | +| `read_config_without_migrate`, `save_server_config`, other config-object helpers | `crates/ecstore/src/config/com.rs` | Persistence over the object store | +| `GLOBAL_STORAGE_CLASS` and storage-class parsing | `crates/ecstore/src/config/mod.rs`, `crates/ecstore/src/config/storageclass.rs` | Storage behavior stays in ECStore | -The server-config model module may use only: +## Allowed Dependencies Of The Model Module -- `std::collections::HashMap` -- `std::sync::{LazyLock, OnceLock, RwLock}` for the default `KVS` registration - surface and process-global server-config snapshot -- `serde` for `KV` and `KVS` serialization compatibility -- `serde_json` for `Config::marshal` and `Config::unmarshal` -- existing `rustfs-config` constants and subsystem modules +- `std::collections::HashMap` and `std::sync::{LazyLock, OnceLock, RwLock}` for `DEFAULT_KVS` and `GLOBAL_SERVER_CONFIG`; +- `serde` for `KV`/`KVS` and `serde_json` for `Config::marshal` / `Config::unmarshal`, gated by `server-config-model`; +- existing `rustfs-config` constants and subsystem modules. -If `serde` and `serde_json` are added to `rustfs-config`, they should be attached -only to a model feature such as `server-config-model` unless the implementation -PR proves that making them non-optional is simpler and harmless for downstream -builds. +## Forbidden Dependencies Of The Model Module -## Forbidden Dependencies +- `rustfs-ecstore`, `rustfs`, storage-api traits, or object persistence helpers; +- notify, audit, targets, IAM, scanner, KMS, or admin handler crates; +- async runtimes, HTTP/router crates, object-store crates, or runtime lifecycle state; +- `ConfigSys`, `read_config_without_migrate`, `save_server_config`, or any `com.rs` helper. -The model module must not depend on: +## Shape Preservation -- `rustfs-ecstore` -- `rustfs` -- `StorageAPI` or object persistence helpers -- notify, audit, targets, IAM, scanner, KMS, or admin handler crates -- async runtimes, HTTP/router crates, object-store crates, or runtime lifecycle - state -- unrelated runtime global state outside the process-global server-config - snapshot -- `ConfigSys`, `read_config_without_migrate`, `save_server_config`, or any - `com.rs` persistence helper +Persisted server-config JSON must keep decoding unchanged: -## Boundary Split - -Move in the first extraction: - -- `KV` -- `KVS` -- `Config` -- `DEFAULT_KVS` -- `register_default_kvs` -- `Config::new` -- `Config::get_value` -- `Config::set_defaults` -- `Config::marshal` -- `Config::unmarshal` -- `Config::merge` - -Keep in `ecstore`: - -- `ConfigSys` -- `init_global_config_sys` -- `try_migrate_server_config` -- `read_config_without_migrate` -- `save_server_config` -- generic `com.rs` config-object helpers -- storage-class runtime global state - -Keep default registration wiring in `ecstore::config::init` until a later PR -extracts a dedicated default-registration contract. The values may be registered -through the moved `rustfs_config::server_config::register_default_kvs`, but the -startup order and caller remain unchanged. - -Move in `CFG-008`: - -- `GLOBAL_SERVER_CONFIG` -- `get_global_server_config` -- `set_global_server_config` - -The temporary ECStore compatibility re-export for these accessors was removed -after code scans showed in-repo consumers use `rustfs_config::server_config` -directly. - -## Required Shape Preservation - -The extraction PR must preserve: - -- `KV { key, value, hidden_if_empty }` -- `#[serde(default, alias = "hiddenIfEmpty")]` on `KV::hidden_if_empty` -- `KVS(pub Vec)` -- `Config(pub HashMap>)` -- `KVS::new`, `get`, `lookup`, `is_empty`, `keys`, `insert`, and `extend` -- `Config::new`, `get_value`, `set_defaults`, `marshal`, `unmarshal`, and - `merge` -- `Config::new()` default application after `ecstore::config::init()` -- existing persisted server-config JSON shape -- existing target, notify, audit, scanner, OIDC, and admin interpretation of - `Config` and `KVS` - -## Next PR Requirements - -`CFG-003` should be a pure model extraction or narrow `api-extraction` PR. It -must not migrate consumers, change persistence helpers, or alter runtime -behavior. - -`CFG-004` kept the old `rustfs_ecstore::config::*` path as a temporary -compatibility shim, registered its removal condition, and removed the shim after -all in-repo consumers migrated. - -`CFG-005` should migrate external consumers one group at a time after the model -and compatibility path are stable. - -`CFG-008` moves only the global server-config snapshot accessors to -`rustfs-config` and migrates in-repo direct consumers. It must not move -`ConfigSys`, storage-class global state, persistence helpers, default -registration wiring, startup order, or storage behavior. - -## Verification Gate - -Before pushing an extraction PR, run: - -- serde roundtrip tests for old and new paths -- tests for `hiddenIfEmpty` alias compatibility -- tests for `KVS` insertion, lookup, extension, and keys behavior -- tests for `Config::new`, `set_defaults`, `marshal`, `unmarshal`, and `merge` -- a cleanup scan proving in-repo consumers no longer use the old - `rustfs_ecstore::config::{Config, KV, KVS}` model path before removing the - compatibility shim -- `cargo tree -p rustfs-config --edges normal` -- `cargo tree -p rustfs-ecstore --edges normal` -- `./scripts/check_layer_dependencies.sh` -- `./scripts/check_architecture_migration_rules.sh` -- `cargo fmt --all --check` -- `make pre-commit` - -## Non-Goals - -- No consumer migration in `CFG-002`. -- No code movement in `CFG-002`. -- No new crate in `CFG-002`. -- No `com.rs` or `StorageAPI` movement in the first model extraction. -- No global server-config state migration until the model path is stable. +- `KV { key, value, hidden_if_empty }` with `#[serde(default, alias = "hiddenIfEmpty")]` on `hidden_if_empty`; +- `KVS(pub Vec)` and `Config(pub HashMap>)`; +- `KVS::{get, lookup, is_empty, keys, insert, extend}` and `Config::{get_value, set_defaults, marshal, unmarshal, merge}` keep their semantics; +- `Config::new()` applies the defaults registered by `ecstore::config::init()`; +- target, notify, audit, scanner, OIDC, and admin code keep interpreting `Config` and `KVS` the same way. diff --git a/docs/architecture/crate-boundaries.md b/docs/architecture/crate-boundaries.md index 909305556..542c7582f 100644 --- a/docs/architecture/crate-boundaries.md +++ b/docs/architecture/crate-boundaries.md @@ -1,7 +1,7 @@ # Crate Boundaries And Migration Guardrails -These rules apply to architecture-migration PRs linked to -[`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660). +**Use this when:** you add a crate dependency, move code across crates, touch a `storage_api.rs` boundary file, or need the change-type vocabulary the architecture guard enforces. +**Source of truth:** `scripts/check_architecture_migration_rules.sh` (the enumerated rules; this file is its boundary document) and `scripts/check_layer_dependencies.sh` (layer and edge checks). Extend those guards instead of adding a parallel system. ## PR Types @@ -22,275 +22,51 @@ Do not mix directory movement, security tightening, and behavior changes in one ## Dependency Direction -Contract crates must stay below implementation crates. Initial forbidden edges: +Contract crates stay below implementation crates. Forbidden edges: -- `storage-api -> ecstore` -- `security-governance -> rustfs` -- `extension-schema -> rustfs` -- `extension-schema -> ecstore` - -`rustfs-storage-api` may only expose storage-facing replication status/state -contracts through `crates/storage-api/src/replication.rs` while the underlying -wire types still live in `rustfs-filemeta`. This keeps the temporary dependency -centralized until those wire contracts can move without introducing a -`rustfs-replication` / `rustfs-storage-api` cycle. - -Leaf crates carry exactly one adjudicated allowed edge: -`io-metrics -> rustfs-s3-ops` (transitively `rustfs-s3-types`). Both are pure -contract crates — types and enums only, no I/O, no global state, no non-contract -internal dependencies — so `io-metrics` reuses the `S3Operation` vocabulary -instead of copying it. `madmin` is no longer counted a leaf: since #6166 it is -the SigV4-signed admin SDK client and deliberately depends on `rustfs-signer`; -the guard pins its internal dependency surface to exactly that edge so it cannot -quietly grow storage-side dependencies. The leaf-crate allowlist in -`scripts/check_architecture_migration_rules.sh` fails any other `rustfs-*` -dependency in `config`, `credentials`, `crypto`, `io-metrics`, or `madmin`, in -either TOML spelling (`rustfs-x = ...` or `rustfs-x.workspace = true`). -Adjudicated in -[`rustfs/backlog#1834`](https://github.com/rustfs/backlog/issues/1834); a further -leaf exception must meet the pure-contract criterion — types and enums only, no -I/O, no globals, no non-contract internal dependencies — and land its guard -allowlist entry alongside the dependency. - -Dependency direction also applies to compile-time source reads: -`include_str!`/`include!` of a `.rs` file must not resolve outside the -including crate's own directory (`scripts/check_layer_dependencies.sh` -enforces this). A source-text tripwire belongs in the crate that owns the -asserted file; shared expectations move into a contract surface such as -`rustfs_protos::compat_manifest` and are asserted by each owning crate. - -Existing migration checks live in: - -- `scripts/check_layer_dependencies.sh` -- `scripts/check_architecture_migration_rules.sh` - -Extend these guardrails instead of adding a parallel system. - -## Required Architecture Documents - -The migration guard must keep these baseline documents present and anchored to -their required sections: - -- `docs/architecture/overview.md`: Baseline, Core Principle, Phase Order. -- `docs/architecture/runtime-lifecycle.md`: Startup And Readiness, Shutdown - Lifecycle Boundary, AppContext Foundation. -- `docs/architecture/storage-control-data-plane.md`: Storage API Contracts, - Cluster Control Plane, Background Controllers. -- `docs/architecture/crate-boundaries.md`: PR Types, Dependency Direction, - Required Architecture Documents. -- `docs/architecture/readiness-matrix.md`: Request Behavior Matrix, Runtime - Dependency Matrix, Probe Semantics. -- `docs/architecture/global-state-crate-split-plan.md`: Remaining Global - Owners, Runtime Source Boundaries, Fallback Removal Plan, Crate Split - Evaluation. - -## Pre-Push Expert Review - -Before pushing any PR branch, record three expert reviews in the task notes: - -| Expert | Required focus | +| Edge | Why | |---|---| -| Quality/architecture | Structure, naming, dependency direction, PR type, scope, and over-abstraction risk | -| Migration preservation | Startup order, readiness, quorum, reader semantics, AppContext/global fallback, notify/audit lifecycle, IAM/KMS boundaries, and compatibility | -| Testing/verification | Focused tests, regression tests, commands run, missing coverage, and whether tests are forcing business-logic drift | +| `storage-api -> ecstore` | Storage contracts must not depend on the storage implementation | +| `security-governance -> rustfs` | Governance contracts stay below the binary crate | +| `extension-schema -> rustfs` | The extension schema is consumed by the binary, never the reverse | +| `extension-schema -> ecstore` | The extension schema must not reach storage internals | -Push is allowed only when all three experts return `pass` or -`pass-with-nonblocking-follow-up`. Any `blocker` prevents push until the issue is -fixed and the relevant review is repeated. +- `rustfs-storage-api` exposes storage-facing replication status/state contracts only through `crates/storage-api/src/replication.rs`, so its temporary dependency on `rustfs-filemeta` wire types stays centralized and no `rustfs-replication` / `rustfs-storage-api` cycle appears. +- Leaf crates (`config`, `credentials`, `crypto`, `io-metrics`, `madmin`) may not depend on other `rustfs-*` crates, in either TOML spelling, except the adjudicated edges pinned in the guard's leaf allowlist: `io-metrics -> rustfs-s3-ops` (pure contract crates sharing the `S3Operation` vocabulary) and `madmin -> rustfs-signer` (the SigV4-signed admin SDK client). A new leaf exception must be a pure contract dependency (types and enums only, no I/O, no globals, no non-contract internal dependencies) and land together with its allowlist entry. +- Compile-time source reads follow the same direction: `include_str!` / `include!` of a `.rs` file must not resolve outside the including crate (`scripts/check_layer_dependencies.sh`). Shared source-text expectations belong in a contract surface such as `rustfs_protos::compat_manifest` (`crates/protos/src/compat_manifest.rs`) and are asserted by each owning crate. -## Temporary Compatibility Code +## ECStore Access Boundary -Temporary compatibility code that must be removed later must include a searchable -source comment and a cleanup-register entry. +Outer crates reach ECStore only through `rustfs_ecstore::api`, and only from one local boundary file per owner (`storage_api.rs`). Boundary files and facade groups are inventoried in [ecstore-api-facade-inventory.md](ecstore-api-facade-inventory.md). -Use this source-comment format: - -```rust -// RUSTFS_COMPAT_TODO(API-005): keep old ecstore::store_api path during storage-api migration. Remove after all consumers use rustfs-storage-api. -``` - -Rules: - -- Add the marker only to temporary compatibility paths, not permanent APIs. -- Include the task ID in the marker. -- State why the compatibility path exists and when it can be removed. -- Use this for temporary re-exports, wrappers, fallbacks, legacy action mappings, - and old endpoint compatibility layers. -- Delete compatibility layers in their own cleanup PR. - -## Config Model First - -`ecstore::config::{Config, KV, KVS}` should move before extension config adapters -or config-schema work. First inventory consumers, then decide whether existing -`crates/config` is enough or whether a smaller model crate is required. - -The current decision is recorded in -[`config-model-boundary-adr.md`](config-model-boundary-adr.md): use the existing -`rustfs-config` package for the pure server-config model and global -server-config snapshot accessors, while ECStore keeps config persistence, -storage-class global state, default wiring, and startup initialization. - -The old `rustfs_ecstore::config::{Config, KV, KVS, register_default_kvs, -get_global_server_config, set_global_server_config}` compatibility path must -not be restored after the Phase 1a cleanup. Consumers use -`rustfs_config::server_config` for the moved model and accessors; ECStore public -facades must not re-export those symbols. +- Inside a boundary file, raw `rustfs_ecstore::api::...` paths are centralized behind local `ecstore_*` module aliases; code outside the boundary sees local type aliases, constants, traits, or wrapper functions, never the raw facade path. +- Non-trait ECStore surfaces (metadata, object-lock, lifecycle journal, monitor, notification types) stay behind local aliases; boundary function signatures do not expose raw ECStore facade types once narrowed. Object and error aliases anchor on storage-api associated object types and a local `StorageError`. +- Outer consumers use `rustfs-storage-api` operation traits (`ObjectIO`, `ObjectOperations`, `ListOperations`, `MultipartOperations`, `HealOperations`, `NamespaceLocking`) and generic list responses (`ListObjectsV2Info`, `ListObjectVersionsInfo`, `ObjectInfoOrErr`) directly; ECStore keeps concrete aliases only for internal implementation and compatibility. +- Bucket lifecycle, replication, versioning, object-lock, restore-request, disk, RPC peer client, and warm-backend trait methods are reached through owner-local compatibility traits or wrapper functions, not by importing ECStore traits outside the boundary. +- The old `StorageAPI` aggregate facade must not reappear in production `crates/ecstore/src` or `rustfs/src` code. +- Facade-covered ECStore root modules (layout, `endpoints`, `disks_layout`, bitrot, erasure, object DTO/reader, event, list, batch processor, `global`) stay crate-private; public access goes through the matching `rustfs_ecstore::api::*` group. +- Cluster control-plane read models stay owned by the crate-private `cluster` module and are published through `rustfs_ecstore::api::cluster`; pool-state, local-node storage, and peer-health projections are read-only. +- RustFS startup internals are crate-private: only `startup_entrypoint` is a public startup module of the `rustfs` library (`rustfs/src/lib.rs`), and items inside the other `startup_*` modules use crate visibility. +- The observability dependency baseline is [obs-ecstore-dependency-inventory.md](obs-ecstore-dependency-inventory.md); observability extraction updates it together with the guard. ## Loss-Prevention Coverage -Architecture migration checks must keep public contract re-exports and ECStore -compatibility coverage from silently drifting during cleanup PRs. +The guard pins specific public re-export lines (its `require_source_line` entries) so contract surfaces cannot silently disappear during cleanup. The canonical lists are the guard script and the owning files, not this page: -Required `rustfs-storage-api` public re-exports: +- `crates/storage-api/src/lib.rs`: admin, bucket, capability, error, multipart, observability, object, and topology contract re-exports; +- `crates/concurrency/src/lib.rs`: workload admission contract re-exports; +- `rustfs/src/lib.rs`: `pub mod startup_entrypoint;`. -- `pub use admin::{DiskSetSelector, StorageAdminApi};` -- `pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};` -- `pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};` -- `pub use error::{StorageErrorCode, StorageResult};` -- `pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};` -- `pub use observability::{MemorySamplingState, ObservabilitySnapshot, ObservabilitySnapshotProvider, PlatformSupport, UserspaceProfilingCapability};` -- `pub use object::{HTTPPreconditions, HTTPRangeError, HTTPRangeSpec, ObjectLockRetentionOptions};` -- `pub use object::{ExpirationOptions, TransitionedObject};` -- `pub use object::{HealOperations, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations};` -- `pub use object::{ListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr};` -- `pub use object::{ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState};` -- `pub use object::{VersionMarker, WalkOptions, WalkVersionsSortOrder};` -- `pub use topology::{DiskCapabilities, TopologyCapabilities, TopologyDisk, TopologyLabels, TopologyPool, TopologySet, TopologySnapshot, TopologySnapshotProvider};` +ECStore keeps compile-time coverage for `StorageAdminApi`, `HealOperations`, and the separate `NamespaceLocking` operation group (`crates/ecstore/tests/ecstore_contract_compat_test.rs`), and its internal consumers use the `rustfs-storage-api` lifecycle DTOs `ExpirationOptions` and `TransitionedObject` directly. -Required `rustfs-concurrency` public workload admission contract re-exports: +## Temporary Compatibility Code -- `pub use workload::{AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadAdmissionSnapshotProvider, WorkloadClass};` +Every temporary compatibility path carries a `RUSTFS_COMPAT_TODO()` source marker with a removal condition and a matching entry in [compat-cleanup-register.md](compat-cleanup-register.md); the guard enforces the match in both directions. Compatibility layers are deleted in their own cleanup change, never bundled with new migration logic. -ECStore must keep compile-time coverage for `StorageAdminApi`, `HealOperations`, -and the separate `NamespaceLocking` operation group. +## Config Model -The old `StorageAPI` aggregate facade must not reappear in production -`crates/ecstore/src` or `rustfs/src` code after the storage operation groups -have been made explicit. +The server-config model (`Config`, `KV`, `KVS`) and the global server-config snapshot accessors are owned by `rustfs_config::server_config`; ECStore keeps persistence, storage-class state, and startup wiring, and its public facades must not re-export those symbols. See [config-model-boundary-adr.md](config-model-boundary-adr.md). -Outer RustFS/IAM consumers must use `rustfs-storage-api` generic list response -contracts directly for `ListObjectsV2Info`, `ListObjectVersionsInfo`, and -`ObjectInfoOrErr`; ECStore keeps the concrete aliases only for internal -implementation and compatibility. +## Required Architecture Documents -Outer RustFS/scanner consumers must use `rustfs-storage-api` operation traits -directly for `ObjectIO`, `ObjectOperations`, `ListOperations`, -`MultipartOperations`, `HealOperations`, and `NamespaceLocking`; ECStore keeps -the concrete compatibility traits only for internal implementation and -downstream compatibility. -Outer consumers must not import ECStore directly outside compatibility -boundaries except for temporary trait imports needed for method resolution or -local test trait implementations. Non-trait ECStore surfaces must stay behind -local aliases, constants, or wrapper functions. - -Outer compatibility boundary modules must use `rustfs_ecstore::api` for ECStore -public facade surfaces such as layout, storage owner, admin, metrics, -notification, capacity, bucket/config helpers, disk/error contracts, global -state accessors, RPC constants/clients, reader helpers, tier helpers, and -rebalance status contracts. Any non-ECStore `storage_compat.rs` import from -`rustfs_ecstore` must route through the `rustfs_ecstore::api` facade. -The legacy ECStore root `endpoints` and `disks_layout` compatibility modules -must remain crate-private; public layout access goes through -`rustfs_ecstore::api::layout`. -Facade-covered ECStore root modules must remain crate-private after this -boundary is established; outer crates should use `rustfs_ecstore::api::*` -instead of legacy root module paths. This includes storage/layout surfaces as -well as remaining bitrot, erasure coding, object DTO/reader, event, list, and -batch processor root modules once their facade groups exist. -ECStore root `global` re-exports must also stay removed once consumers use -`rustfs_ecstore::api::global` or crate-internal `crate::global` paths. -RustFS root `storage_compat.rs` must expose bucket metadata and quota contracts -as explicit aliases only. Broad `metadata`, `metadata_sys`, and `quota` module -passthroughs are reserved to narrower app/admin/storage compatibility -boundaries that still need module-local owner cleanup. -Root runtime storage config initialization and disk endpoint contracts must also -stay explicit aliases. The root compatibility boundary must not restore `com`, -bare `init`, or grouped `endpoint::Endpoint` passthroughs. -RustFS root `storage_compat.rs` must not re-export ECStore API symbols directly; -remaining root runtime compatibility symbols must be local type aliases, -constants, traits, or wrapper functions so ownership stays visible at the -boundary. -RustFS admin `storage_compat.rs` must expose config IO and default -initialization through explicit aliases. The admin compatibility boundary must -not restore broad `com` or bare `init` passthroughs. -RustFS admin and app `storage_compat.rs` bucket-facing compatibility contracts -must stay explicitly whitelisted. They must not restore broad bucket module, -client object API, client transition API, or storage-class module passthroughs -once a local compatibility boundary has narrowed them to specific aliases. -RustFS storage `storage_compat.rs` must expose bucket metadata, object-lock, -policy, replication, tagging, versioning, object API, and test-only -storage-class config contracts through explicit aliases. The storage -compatibility boundary must not restore broad `metadata`, `metadata_sys`, -`object_lock`, `policy_sys`, `replication`, `tagging`, `utils`, `versioning`, -`versioning_sys`, `object_api_utils`, or `com` passthroughs. -RustFS storage owner `storage_compat.rs` must not re-export ECStore API symbols -directly except temporary trait imports needed for method resolution. Remaining -storage-owner compatibility symbols must be local constants, type aliases, or -wrapper functions so storage-owned global state and helper access stays visible -at the boundary. -RustFS app, admin, and storage outer `storage_compat.rs` object and error -facade aliases must stay anchored on storage-api associated object types and -local `StorageError` aliases. They must not reintroduce raw -`rustfs_ecstore::api::object::{ObjectInfo,ObjectOptions}` or -`rustfs_ecstore::api::error::{Error,Result}` references. -Outer compatibility function signatures must also use local aliases for ECStore -metadata, object-lock, lifecycle journal, monitor, and notification facade -types. The boundary may define the local alias, but call signatures must not -expose the raw ECStore facade path once narrowed. -The RustFS storage owner compatibility boundary must keep raw ECStore facade -paths centralized behind local `ecstore_*` module aliases rather than scattering -`rustfs_ecstore::api::...` references through its aliases and wrappers. -The RustFS app/admin storage compatibility boundaries must likewise route raw -ECStore facade access through their local `ecstore_*` module aliases instead of -scattering `rustfs_ecstore::api::...` paths through compatibility wrappers. -Peripheral consumer storage compatibility boundaries must follow the same -pattern. IAM, heal, scanner, notify, observability, Swift, S3 Select, test, and -fuzz storage compatibility modules keep raw ECStore facade access centralized -behind local `ecstore_*` module aliases. -RustFS root runtime and e2e storage compatibility boundaries must follow the -same pattern, keeping raw ECStore facade access centralized behind local -`ecstore_*` module aliases. -Outer bucket lifecycle, replication, versioning, object-lock, and -restore-request trait method access must stay behind local compatibility traits -or wrapper functions. Non-compat sources must not import those ECStore bucket -API traits directly after the wrapper boundary is established. Disk, RPC peer -client, and warm-backend method-resolution access must follow the same pattern: -non-compat sources use owner-local compatibility traits or test aliases instead -of importing ECStore traits directly. -Scanner, notify, observability, and e2e `storage_compat.rs` boundaries must -also stay narrow. Scanner must not restore grouped bucket compatibility exports -for target, lifecycle, metadata, replication, or versioning modules. Notify -must not restore broad `config`/`global` module imports. Observability must -consume data usage through a local DTO projection instead of re-exporting the -ECStore data-usage loader. The e2e harness must not restore grouped RPC -passthroughs. -Test and fuzz `storage_compat.rs` harnesses must also stay narrow. Heal and -scanner test harnesses must expose ECStore contracts through direct aliases or -local wrappers, and fuzz harnesses must wrap bucket utility entrypoints instead -of restoring grouped ECStore passthrough exports. -External ECStore API facade imports must stay inside local `storage_api` -boundary files after the external runtime, test, and fuzz consumers have been -narrowed. IAM, heal, scanner, notify, observability, Swift, S3 Select, e2e, and -fuzz code must not reintroduce direct `rustfs_ecstore::api::...` references -outside those boundary files. -The observability ECStore dependency baseline is tracked in -[`obs-ecstore-dependency-inventory.md`](obs-ecstore-dependency-inventory.md); -future observability extraction PRs must update that inventory with the guard. - -ECStore ClusterControlPlane read models must stay owned by the crate-private -`cluster` module. Public access goes through `rustfs_ecstore::api::cluster` so -outer crates cannot depend on ECStore root control-plane internals. -Pool-state, local-node storage, and peer-health status projections are part of -the same facade boundary and must remain read-only until a later controller -slice explicitly wires dynamic health or membership behavior. - -RustFS startup internals must stay crate-private after the startup owner split. -Only `startup_entrypoint` remains a public startup module for the binary -entrypoint; IAM bootstrap, optional runtime, and profiling startup shims must -not be re-exported as public library modules. Items inside crate-private -startup modules must also use crate visibility rather than bare public -visibility. - -ECStore internal consumers must use `rustfs-storage-api` lifecycle helper DTOs -directly for `ExpirationOptions` and `TransitionedObject`; ECStore keeps the -old lifecycle paths only as downstream compatibility re-exports. +The guard requires the documents and section headings listed in its `require_source_contains` entries (`scripts/check_architecture_migration_rules.sh`); the directory index is [README.md](README.md). diff --git a/docs/architecture/decommission-compatibility.md b/docs/architecture/decommission-compatibility.md index 3c243ac05..90e0fdc53 100644 --- a/docs/architecture/decommission-compatibility.md +++ b/docs/architecture/decommission-compatibility.md @@ -1,47 +1,31 @@ # Decommission Compatibility Scope -This note records the current RustFS decommission contract for admin/API -compatibility reviews. +**Use this when:** you change pool decommission or rebalance behavior, its admin API shape, the persisted `PoolMeta` decommission fields, or how tier free versions move between pools. +**Source of truth:** `crates/ecstore/src/core/pools.rs` (queue, recovery, cleanup predicates), `crates/ecstore/src/services/rebalance/worker.rs` (rebalance predicates), `rustfs/src/admin/handlers/pools.rs` plus the `pools/*` rows of `rustfs/src/admin/route_policy.rs` (admin surface), `crates/ecstore/src/data_movement/` and `crates/ecstore/src/set_disk/` (free-version movement). ## Current Contract -RustFS supports queued multi-pool decommission start requests on multi-pool -deployments. - -The admin handler accepts the request shape used by the MinIO-compatible admin -API, including comma-separated pool targets. An empty target list is rejected. -Single-pool deployments reject decommission because there is no destination pool. -On multi-pool deployments, one or more valid target pools are accepted as a -single queued operation. +RustFS supports queued multi-pool decommission start requests on multi-pool deployments. The admin handler accepts the MinIO-compatible request shape, including comma-separated pool targets. An empty target list is rejected; single-pool deployments reject decommission because there is no destination pool; on multi-pool deployments one or more valid target pools are accepted as a single queued operation. ### Request Semantics -`POST /v3/pools/decommission` with comma-separated pool targets is treated as a -queue submission: +`POST /v3/pools/decommission` with comma-separated pool targets is a queue submission: - validate all requested pool identifiers before mutating metadata; - reject duplicate target pools in the same request; - reject active or queued target pools; -- reject completed decommission targets because completion means the pool can be - removed from the deployment configuration; +- reject completed decommission targets, because completion means the pool can be removed from the deployment configuration; - allow failed or canceled targets to be retried; - persist queued metadata before starting workers; - start only the local-leader prefix of the queue on the receiving node. -The local-leader-prefix rule keeps the active worker on the leader for the pool -being moved while still allowing a request to contain later targets whose leaders -are different nodes. Later queued targets are recovered or promoted by the -leader that owns that target. +The local-leader-prefix rule keeps the active worker on the leader for the pool being moved while still allowing a request to contain later targets whose leaders are different nodes. Later queued targets are recovered or promoted by the leader that owns that target. -Admin start, cancel, and clear requests may arrive on any cluster node. When the -target pool first endpoint is remote, RustFS forwards the operation over the -authenticated internode RPC channel to that first endpoint. The receiving node -still enforces the local-leader rule before mutating decommission state. +Start, cancel (`POST /v3/pools/cancel`), and clear (`POST /v3/pools/clear`) requests may arrive on any cluster node. When the target pool's first endpoint is remote, RustFS forwards the operation over the authenticated internode RPC channel to that endpoint; the receiving node still enforces the local-leader rule before mutating decommission state. ### Persisted Metadata Shape -The queue is persisted in pool metadata and decoded with the rest of -`PoolMeta`. Each pool entry can distinguish: +The queue is persisted in pool metadata and decoded with the rest of `PoolMeta`. Each pool entry can distinguish: - `active`: at most one pool currently moving data; - `queued`: validated pools waiting for the active entry to finish; @@ -49,310 +33,112 @@ The queue is persisted in pool metadata and decoded with the rest of - `failed`: pools whose worker reached terminal failure; - `canceled`: pools canceled before or during execution. -Legacy metadata without queue fields decodes as a non-queued decommission entry, -preserving restart behavior for already deployed clusters. +Legacy metadata without queue fields decodes as a non-queued decommission entry, preserving restart behavior for already deployed clusters. ### Serial Scheduling And Recovery Only one queued entry may own a decommission worker at a time. Startup recovery: - loads pool metadata before rebalance recovery; -- resumes the first local non-terminal active/queued entry; -- skips a durably completed prefix and promotes the next queued entry only after - successful completion; -- treats failed or canceled terminal entries as an automatic-promotion barrier, - leaving later queued pools visible but stopped until an operator retries, - clears, or otherwise resolves the terminal entry; -- keeps queued pools out of active worker scheduling until promotion, while still - making their future state visible in admin status. +- computes the resumable entries with `resumable_decommission_queue_indices` (`crates/ecstore/src/core/pools.rs`): every pool that has decommission state and is not terminal (`complete`, `failed`, or `canceled`). Terminal predecessors are skipped, not treated as barriers, so a queued pool behind a failed or canceled attempt is still resumable (`test_resumable_decommission_queue_indices_skip_terminal_predecessors`); +- starts workers only for the local-leader prefix of those entries; later queued pools stay out of worker scheduling until promotion while their state remains visible in admin status. -Promotion is persisted before worker execution. If cancellation is already -requested immediately after promotion, RustFS persists a canceled terminal state -instead of leaving the promoted pool active without a worker. +Promotion is persisted before worker execution. If cancellation is already requested immediately after promotion, RustFS persists a canceled terminal state instead of leaving the promoted pool active without a worker. ### Cancel Semantics Cancel separates active and queued behavior: -- canceling the active entry requests worker cancellation and persists terminal - metadata; +- canceling the active entry requests worker cancellation and persists terminal metadata; - canceling a queued entry marks that entry canceled before it becomes active; -- failed or canceled terminal entries can be cleared explicitly when the operator - chooses to abandon the decommission attempt; -- peer reload failures during cancel must be surfaced in status and logs. +- failed or canceled terminal entries can be cleared explicitly (`POST /v3/pools/clear`) when the operator abandons the decommission attempt; +- peer reload failures during cancel are surfaced in status and logs. -Cancel requests can be accepted on non-leader nodes as remote cancel intent; the -leader observes the pending cancel and applies it to the active worker. +Cancel requests can be accepted on non-leader nodes as remote cancel intent; the leader observes the pending cancel and applies it to the active worker. ### Status Response Shape -`GET /v3/pools/list` and `GET /v3/pools/status?pool=...` expose per-pool -machine-readable decommission state. The `status` field can report `active`, -`running`, `queued`, `complete`, `failed`, or `canceled`. +`GET /v3/pools/list` and `GET /v3/pools/status?pool=...` expose per-pool machine-readable decommission state. The `status` field can report `active`, `running`, `queued`, `complete`, `failed`, or `canceled`. When decommission metadata is present, `decommissionInfo` includes: - queue and terminal flags: `queued`, `complete`, `failed`, `canceled`; -- progress counters: `objectsDecommissioned`, - `objectsDecommissionedFailed`, `bytesDecommissioned`, and - `bytesDecommissionedFailed`; +- progress counters: `objectsDecommissioned`, `objectsDecommissionedFailed`, `bytesDecommissioned`, and `bytesDecommissionedFailed`; - current location: `bucket`, `prefix`, and `object`; - queue/history lists: `queuedBuckets` and `decommissionedBuckets`; -- `waitingReason`, currently `queued` for queued entries and - `waiting_for_worker` when metadata exists but no worker has started. +- `waitingReason`: `queued` for queued entries and `waiting_for_worker` when metadata exists but no worker has started. -This makes queued pools and stalled metadata visible without requiring operators -to inspect pool metadata files directly. +This makes queued pools and stalled metadata visible without requiring operators to inspect pool metadata files directly. ## MinIO Divergence Decisions -This section records the current product decisions for behavior that is close to -MinIO but not always byte-for-byte identical. +Behavior that is close to MinIO but not byte-for-byte identical. Changing either decision requires an operator compatibility note and updated characterization tests. ### Empty Delete Markers -MinIO decommission documentation states that empty delete markers, meaning delete -markers with no successor object versions, are not transitioned to another pool. +MinIO decommission documentation states that empty delete markers (delete markers with no successor object versions) are not transitioned to another pool. RustFS follows that behavior for decommission when the bucket has no replication configuration: a lone remaining delete marker is cleanup-only metadata and is skipped. When replication is configured, RustFS keeps the delete marker eligible for movement so delete-marker replication and purge state are not lost. -RustFS follows that behavior for decommission when the bucket has no replication -configuration: a lone remaining delete marker is treated as cleanup-only metadata -and is skipped. When replication is configured, RustFS intentionally keeps the -delete marker eligible for movement so delete-marker replication and purge state -are not lost. - -RustFS rebalance uses the same predicate as decommission: skip only a lone delete -marker without replication. This is intentional even though MinIO's public -documentation calls out the decommission case more explicitly than the rebalance -case. - -Regression guards: - -- `should_skip_decommission_delete_marker_characterizes_empty_marker_without_replication` -- `should_skip_decommission_delete_marker_characterizes_replication_configured` -- `test_should_skip_rebalance_delete_marker_characterizes_empty_marker_without_replication` -- `test_should_skip_rebalance_delete_marker_characterizes_replication_configured` +Rebalance uses the same predicate as decommission (`should_skip_decommission_delete_marker` in `crates/ecstore/src/core/pools.rs`, `should_skip_rebalance_delete_marker` in `crates/ecstore/src/services/rebalance/worker.rs`), even though MinIO's public documentation calls out the decommission case more explicitly than the rebalance case. ### Lifecycle-Expired Versions During Cleanup -MinIO decommission ignores versions that are already expired by lifecycle rules. -RustFS follows that decommission behavior by allowing safely expired versions to -count toward source cleanup completion. - -RustFS rebalance is intentionally stricter. Expired versions do not prove that a -target pool received an equivalent version, so rebalance cleanup requires actual -rebalance completion for the source entry instead of treating lifecycle-expired -versions as moved. - -Regression guards: - -- `test_should_cleanup_decommission_source_entry_accepts_migrated_and_safely_expired_versions` -- `test_should_cleanup_decommission_source_entry_accepts_versions_only_safely_expired_by_lifecycle` -- `test_should_cleanup_rebalance_source_entry_rejects_versions_only_expired_by_lifecycle` - -No migration step is required for these decisions because this note documents the -current RustFS behavior. Changing either decision later requires an operator -compatibility note and updated characterization tests. +MinIO decommission ignores versions already expired by lifecycle rules. RustFS applies the same rule to decommission and rebalance: a source entry is cleanup-complete when moved versions plus safely expired versions equal the total version count (`should_cleanup_decommission_source_entry` in `crates/ecstore/src/core/pools.rs`, `should_cleanup_rebalance_source_entry` in `crates/ecstore/src/services/rebalance/worker.rs`). Versions retained by object lock or pending replication are not counted as safely expired by the callers, so an entry with such versions is retained. Both predicates accept an entry whose versions are all lifecycle-expired (`test_should_cleanup_decommission_source_entry_accepts_versions_only_safely_expired_by_lifecycle`, `test_should_cleanup_rebalance_source_entry_accepts_versions_only_expired_by_lifecycle`). ## Tier Free Versions During Decommission -A tier free version is an internal xl.meta record (`rustfs_filemeta::FREE_VERSION`, -flagged `XL_FLAG_FREE_VERSION`) shaped like a delete marker. It is created by -`MetaObject::init_free_version` when a version whose remote transition completed is -deleted locally: the visible version is removed and the record keeps the remote-tier -identity (tier, object name, version id, state, destination id) needed for an -idempotent remote delete. Free versions are not user-visible versions; `num_versions` -and all listing/GET paths exclude them. +A tier free version is an internal xl.meta record (`rustfs_filemeta::FREE_VERSION`, flagged `XL_FLAG_FREE_VERSION`) shaped like a delete marker. It is created by `MetaObject::init_free_version` when a version whose remote transition completed is deleted locally: the visible version is removed and the record keeps the remote-tier identity (tier, object name, version id, state, destination id) needed for an idempotent remote delete. Free versions are not user-visible versions; `num_versions` and all listing/GET paths exclude them. ### Lifecycle And Consumers -Creation: a local delete that removes a version whose transition status is -`complete` normally appends the record via `MetaObject::delete_version` → -`init_free_version`. User-facing single and batch deletes always retain that -historical owner when they actually remove a transitioned source; they do not -create a tier journal, probe a fleet capability, or issue a peer mutation RPC. -`TransitionVersionState::Unknown` and incomplete destination identities remain -on the same conservative free-version path. Delete-marker creation on an Enabled -bucket remains unchanged and does not schedule remote deletion. +Creation: a local delete that removes a version whose transition status is `complete` normally appends the record via `MetaObject::delete_version` → `init_free_version` (skipped only when `skip_tier_free_version` is set, as on data-movement copies). User-facing single and batch deletes always retain that historical owner when they actually remove a transitioned source; they do not create a tier journal, probe a fleet capability, or issue a peer mutation RPC. `TransitionVersionState::Unknown` and incomplete destination identities stay on the same conservative free-version path. Delete-marker creation on an Enabled bucket is unchanged and does not schedule remote deletion. -Recursive prefix/delete-all cannot preserve per-object markers across its -physical directory purge, so it requires a v6 recoverable journal for every -transitioned visible source plus a durable dispatch manifest for the complete -operation. It fails closed before mutation on legacy metadata or any existing -hidden tier free-version under the prefix. Its internal streaming walk -discovers logical keys, then exact-loads every key from its authoritative set in -every pool, including free versions; the S3 listing merge is never treated as a -complete physical-owner inventory. Tier-operation leases remain held from that -preflight through journal prepare and physical deletion. Once physical deletion -starts, any error is mutation-ambiguous: authorized/dispatched journals remain -for recovery to commit owners only after all physical sets prove both the source -and exact free-version identity absent; uncertain owners are retained. -If a retry discovers a later transitioned source after the manifest reached -`DispatchAuthorized`, it replays only the manifest's immutable predecessor set, -completes that operation, and leaves the newcomer for a successor dispatch. -Operators may retry after the legacy free-version worker has durably completed -remote and local cleanup. Journal-less internal deletes and older nodes retain -their established marker behavior. +Recursive prefix/delete-all cannot preserve per-object markers across its physical directory purge, so it requires a v6 recoverable journal for every transitioned visible source plus a durable dispatch manifest for the whole operation. It fails closed before mutation on legacy metadata or on any existing hidden tier free-version under the prefix. Its internal streaming walk discovers logical keys, then exact-loads every key from its authoritative set in every pool, including free versions; the S3 listing merge is never treated as a complete physical-owner inventory. Tier-operation leases stay held from that preflight through journal prepare and physical deletion. Once physical deletion starts, any error is mutation-ambiguous: authorized/dispatched journals remain for recovery to commit owners only after all physical sets prove both the source and the exact free-version identity absent; uncertain owners are retained. If a retry discovers a later transitioned source after the manifest reached `DispatchAuthorized`, it replays only the manifest's immutable predecessor set, completes that operation, and leaves the newcomer for a successor dispatch. Operators may retry after the legacy free-version worker has durably completed remote and local cleanup. Journal-less internal deletes and older nodes keep their established marker behavior. -Consumption while the record exists: the background recovery loop started by -`init_background_expiry` (spawned by `spawn_tier_free_version_recovery_once`, -enabled by default) scans disks for pending records and re-enqueues them; the -usage scanner does the same; the lifecycle worker then deletes the remote tier -object idempotently and only afterwards removes the local record. Heal walks -include free-version records in metadata healing. Transition planning, -replication, restore, GET, listings, and usage aggregation never depend on -them. +Consumption while the record exists: the background recovery loop started by `init_background_expiry` (spawned by `spawn_tier_free_version_recovery_once`, enabled by default) scans disks for pending records and re-enqueues them; the usage scanner does the same; the lifecycle worker then deletes the remote tier object idempotently and only afterwards removes the local record. Heal walks include free-version records in metadata healing. Transition planning, replication, restore, GET, listings, and usage aggregation never depend on them. ### Decommission Handling -The exact decommission inventory loader (`load_file_info_versions_exact` via -`get_all_file_info_versions`) keeps free-version records inline in `versions`. -The migration loop handles them before lifecycle expiry and delete-marker -shortcuts. It selects a target pool using the free-version-aware lookup, then -writes the original free record to every target disk with the normal metadata -write quorum. The free-version marker, local version id, transition identity, -transition state, and destination id are preserved at the FileInfo/metadata -boundary. +The exact decommission inventory loader (`load_file_info_versions_exact` via `get_all_file_info_versions`) keeps free-version records inline in `versions`. The migration loop handles them before lifecycle expiry and delete-marker shortcuts. It selects a target pool using the free-version-aware lookup, then writes the original free record to every target disk with the normal metadata write quorum. The free-version marker, local version id, transition identity, transition state, and destination id are preserved at the FileInfo/metadata boundary. -The source record is physically removed only after the target write quorum has -committed and the source cleanup preflight still matches the exact inventory. -If the lifecycle worker has already completed the remote delete and removed the -source record before decommission acquires the source lock, decommission records -that identity as already consumed and treats the missing source record as safe. -If target capacity, metadata validation, lock fencing, or quorum fails, the -source record remains and the entry records `state = "free_version_retained"` -with reason `tier_free_version_migration_failed`; the worker retries the -operation on a later pass. A target record with the same version id is accepted -only when its free-version identity matches; a conflicting ordinary version or -different free record is an overwrite error. This makes retries idempotent and -prevents a free record from replacing a user-visible version. +The source record is physically removed only after the target write quorum has committed and the source cleanup preflight still matches the exact inventory. If the lifecycle worker has already completed the remote delete and removed the source record before decommission acquires the source lock, decommission records that identity as already consumed and treats the missing source record as safe. If target capacity, metadata validation, lock fencing, or quorum fails, the source record remains and the entry records `state = "free_version_retained"` with reason `tier_free_version_migration_failed`; the worker retries the operation on a later pass. A target record with the same version id is accepted only when its free-version identity matches; a conflicting ordinary version or different free record is an overwrite error. This makes retries idempotent and prevents a free record from replacing a user-visible version. + +`TransitionVersionState::Unknown` records are migrated unchanged rather than discarded; the lifecycle worker retains them if remote identity validation cannot make a delete request. Only an authorized recursive prefix/delete-all v6 transaction may use a per-source journal as the sole retry source; ordinary single/batch deletes never take that path, and a journal discovered alongside an older or fallback free-version never authorizes dropping the xl.meta record. ### Remote-Tuple Publication Fence -Cross-pool capability v3 includes a commit-late publication contract for every -path that can copy an existing transition tuple to a new physical owner. This -capability version is independent of the tier-mutation RPC protocol version. -A mixed fleet whose minimum cross-pool capability is below v3 cannot authorize -journal-v6 remote deletion. +Cross-pool capability v3 adds a commit-late publication contract for every path that can copy an existing transition tuple to a new physical owner. This capability version is independent of the tier-mutation RPC protocol version; a mixed fleet whose minimum cross-pool capability is below v3 cannot authorize journal-v6 remote deletion. -Data movement captures a non-cloneable, process-local source capability before -copying, but it does not hold a namespace write lock or tier-operation lease -while reading a large body or uploading multipart parts. `NewMultipartUpload` -and `UploadPart` are staging only. Immediately before single-PUT rename, -Multipart Complete, or a pure-remote/free-version metadata quorum write, the -final consumer acquires the exact tier generation (when a remote tuple exists), -then fixed/source/target write domains in stable order. The fixed domain is used -only for a real remote-tuple decommission publisher; an ordinary local object -keeps the lighter source/target commit scope. +Data movement captures a non-cloneable, process-local source capability before copying, but it does not hold a namespace write lock or tier-operation lease while reading a large body or uploading multipart parts (`NewMultipartUpload` and `UploadPart` are staging only). Immediately before single-PUT rename, Multipart Complete, or a pure-remote/free-version metadata quorum write, the final consumer acquires the exact tier generation (when a remote tuple exists), then the fixed/source/target write domains in stable order. The fixed domain is used only for a real remote-tuple decommission publisher; an ordinary local object keeps the lighter source/target commit scope. -While that owned scope is held, the publisher re-reads the exact source pool and -compares version, data directory, modification time, ETag, checksums, transition -tuple, transition-version state, and destination identity. A missing or changed -source, changed/revoked tier generation, bucket incarnation change, or lost lock -fails before target rename. The scope remains owned through rename quorum and -the existing rename-tail guard handoff. Consequently, recovery-first ordering -cannot delete the remote object and then have a stale restored-transitioned -rebalance recreate its tuple; publisher-first ordering makes recovery wait and -rescan the newly committed owner. +While that owned scope is held, the publisher re-reads the exact source pool and compares version, data directory, modification time, ETag, checksums, transition tuple, transition-version state, and destination identity. A missing or changed source, a changed or revoked tier generation, a bucket incarnation change, or a lost lock fails before target rename. The scope stays owned through rename quorum and the rename-tail guard handoff, so recovery-first ordering cannot delete the remote object and then let a stale restored-transitioned rebalance recreate its tuple, and publisher-first ordering makes recovery wait and rescan the newly committed owner. -Full cross-key S3 Copy is not an ownership-sharing operation: it materializes -local data and strips transition, destination, transaction, and free-version -keys. Same-key metadata/version-only updates preserve the existing protected -state. Admin heal keeps the legacy `nolock` request field for wire compatibility -but ignores it as lock authority; final heal writes enter the normal locked -path. Restore similarly ignores ambient `ObjectOptions.no_lock`, acquires its -own commit-late PUT/Complete lock, validates the restore operation id, and keeps -an exact tier generation lease through the local commit. +Full cross-key S3 Copy is not an ownership-sharing operation: it materializes local data and strips transition, destination, transaction, and free-version keys. Same-key metadata/version-only updates preserve the protected state. Admin heal keeps the legacy `nolock` request field for wire compatibility but ignores it as lock authority; final heal writes enter the normal locked path. Restore likewise ignores ambient `ObjectOptions.no_lock`, acquires its own commit-late PUT/Complete lock, validates the restore operation id, and keeps an exact tier generation lease through the local commit. -### Reference-Audit Result +### Tier Mutation Protocol And Journal v6 Rollout -After migration, user-facing GET/list/transition/replication/restore paths still -exclude the record. Recovery, usage scanning, lifecycle tier cleanup, and heal -continue to see a legacy/fallback record when they request free versions, so an -unresolved remote delete remains actionable on the target pool. Only an -authorized recursive prefix/delete-all v6 transaction may instead use a -per-source journal as the sole retry source; ordinary single/batch deletes never -take that path. A journal discovered -alongside an older or fallback free-version does not authorize dropping the -record. In particular, `Unknown` transition state records are migrated unchanged -rather than discarded: the lifecycle worker retains them if remote identity -validation cannot make a delete request. +Tier edit/remove/clear reference proof uses the internal walk with `include_free_versions = true`, in addition to persisted journal and transition-transaction checks. Protocol v3 peer Prepare blocks new reference creators and drains existing tier-operation leases before this proof; protocol v4 preserves that state machine and adds a signed failure classification. Abort carries the canonical Prepare intent, so a peer can create an identity-bound `Aborted` tombstone even when Abort overtakes Prepare; a delayed matching Prepare then converges on `Aborted` instead of reinstalling the block, and a conflicting intent with the same mutation id fails closed. The tombstone stays durable until intent expiry plus the configured clock-skew allowance, including across reload and coordinator-record cleanup. After expiry, a missing-record replay of the original signed Prepare is rejected and cannot recreate a peer-only runtime fence. Abort checks an existing same-identity terminal record before consulting mutable current-config proof, and recovery reconstructs the original Prepared revision for Abort fanout. -Tier edit/remove/clear reference proof uses the internal walk with -`include_free_versions = true`, in addition to persisted journal and transition -transaction checks. Protocol v3 peer Prepare blocks new reference creators and -drains existing tier-operation leases before this proof; protocol v4 preserves -that state machine and adds a signed failure classification. Abort carries the -canonical Prepare intent, so a peer can create an identity-bound `Aborted` -tombstone even when Abort overtakes Prepare. A delayed matching Prepare then -converges on `Aborted` instead of reinstalling the block; a conflicting intent -with the same mutation id fails closed. The tombstone remains durable until the -intent expiry plus the configured clock-skew allowance, including across reload -and coordinator-record cleanup. After -expiry, a missing-record replay of the original signed Prepare is rejected and -cannot recreate a peer-only runtime fence. Abort checks an existing same-identity -terminal record before consulting mutable current-config proof, and recovery -reconstructs the original Prepared revision for Abort fanout. +A new server accepts both v3 and v4 requests and selects the matching canonical response proof. During a mixed rollout an older v3 server rejects a v4 request with an authenticated, byte-exact unsupported-version status before dispatch; the v4 coordinator treats only that exact rejection as definitely-not-installed, fails the admin mutation, and does not send the peer an incompatible Abort. There is deliberately no automatic v3 retry: `Unimplemented`, near-text, timeouts, missing or unknown failure classes, and other ambiguous outcomes still receive Abort and retain the coordinator retry record if Abort cannot be proven. Operators must pause and drain tier edit/remove/clear operations before starting a rolling upgrade, leave them disabled while any v3-only peer remains, and resume only after every topology member advertises the v4-capable release. Ordinary object I/O and free-version cleanup stay available; `xl.meta` is unchanged by a rejected mutation. -A new server accepts both v3 and v4 requests and selects the matching canonical -response proof. During a mixed rollout, an older v3 server rejects a v4 request -with an authenticated, byte-exact unsupported-version status before dispatch; -the v4 coordinator treats only that exact rejection as definitely not installed, -fails the admin mutation, and does not send the peer an incompatible Abort. -There is deliberately no automatic v3 retry. `Unimplemented`, near-text, -timeouts, missing/unknown failure classes, and other ambiguous outcomes still -receive Abort and retain the coordinator retry record if Abort cannot be proven. -Operators must pause and drain tier edit/remove/clear operations before starting -the rolling upgrade, leave them disabled while any v3-only peer remains, and -resume only after every topology member advertises the v4-capable release. -Ordinary object I/O and free-version cleanup remain available; xl.meta is -unchanged by the rejected mutation. +Sole-owner transactions use journal v6: v5-and-older readers reject and retain those records, so an old recovery worker cannot bypass the all-pool proof. Older nodes may keep creating fallback free-versions until the rollout is homogeneous. Do not downgrade every v6-aware recovery worker while any v6 record remains; drain the journal first or keep at least one v6-aware worker until cleanup converges. -Sole-owner transactions use journal v6: v5-and-older readers reject and retain -those records, so an old recovery worker cannot bypass the all-pool proof. Older -nodes may continue to create fallback free-versions until the rollout is -homogeneous. A deployment must not downgrade every v6-aware recovery worker -while any v6 record remains; drain the journal first or keep at least one v6-aware -worker until cleanup converges. +### Disposition Events -Each migrated record emits `state = "free_version_migrated"` with reason -`tier_free_version_migrated`. A record consumed before migration emits -`state = "free_version_consumed"` with reason -`tier_free_version_already_consumed`. Each failed record emits the retained state -and failure reason above. The entry also emits a disposition summary with -migrated, consumed, retained, and total counts. The final decommission sweep uses -the exact loader, counts free records still present, and emits one retained -record/reason for each unresolved free version before failing the sweep. This -makes successful migration, completed cleanup, and retained cleanup obligations -visible instead of silently omitting free records. +Free versions remain internal, so no S3-visible version or admin response field is added. The structured `decommission_entry` events are the operational status surface: -No new S3-visible version or admin response field is needed: free versions remain -internal and are never counted as user-visible versions. The structured -`decommission_entry` events are the operational status surface for the -free-version disposition; the existing decommission item/failed counters still -report the enclosing object migration result. +| Outcome | `state` | `reason` | +|---|---|---| +| Record migrated to the target pool | `free_version_migrated` | `tier_free_version_migrated` | +| Record consumed by the lifecycle worker before migration | `free_version_consumed` | `tier_free_version_already_consumed` | +| Migration failed, source retained for retry | `free_version_retained` | `tier_free_version_migration_failed` | -Regression guard: +The entry also emits a disposition summary with migrated, consumed, retained, and total counts. The final decommission sweep uses the exact loader, counts free records still present, and emits one retained record/reason per unresolved free version before failing the sweep. The existing decommission item/failed counters still report the enclosing object migration result. -- `decommission_tier_free_version_preserves_remote_identity` -- `decommission_tier_free_version_resume_requires_write_quorum` -- `decommission_tier_free_version_commit_rejects_lost_fence` -- `test_decommission_cleanup_preflight_accepts_migrated_free_version_consumed_from_source` -- `decommission_entry_skips_cleanup_only_marker_when_free_version_is_present` -- `decommission_entry_rejects_subquorum_free_version_conflict_and_retains_source` +## Regression Guards -## Regression Guard +Test names drift; locate the current guards instead of copying them: -The queued multi-pool contract is guarded by: - -- `test_contextualized_decommission_start_request_allows_multiple_target_pools` -- `test_decommission_start_local_leader_allows_remote_queued_pool` -- `test_local_decommission_queue_prefix_stops_at_remote_leader` -- `test_decommission_peer_target_returns_none_for_local_first_endpoint` -- `test_pool_meta_queued_decommission_is_not_suspended_until_promoted` -- `test_pool_meta_promoted_queued_decommission_can_be_canceled` -- `test_first_resumable_decommission_queue_indices_stops_at_failed_or_canceled_state` -- `test_first_resumable_decommission_queue_indices_allows_after_completed_prefix` -- `admin_pool_list_item_exposes_queued_decommission_state` - -These tests live in `crates/ecstore/src/core/pools.rs` and -`rustfs/src/app/admin_usecase.rs`. +```bash +rg -n 'fn [a-z_]*decommission[a-z_]*\(' crates/ecstore/src/core/pools.rs crates/ecstore/src/set_disk/mod.rs crates/ecstore/src/data_movement/mod.rs crates/ecstore/src/store/init.rs rustfs/src/admin/handlers/pools.rs rustfs/src/app/admin_usecase.rs +rg -n 'fn test_should_[a-z_]*rebalance[a-z_]*\(' crates/ecstore/src/services/rebalance/rebalance_unit_tests.rs +``` diff --git a/docs/architecture/ecstore-api-facade-inventory.md b/docs/architecture/ecstore-api-facade-inventory.md index 262ceac4b..7fbd79c52 100644 --- a/docs/architecture/ecstore-api-facade-inventory.md +++ b/docs/architecture/ecstore-api-facade-inventory.md @@ -1,76 +1,65 @@ # ECStore API Facade Inventory -This inventory records the current `rustfs_ecstore::api` compatibility surface -before any ECStore split PR removes or narrows re-exports. It is a planning and -guardrail document only. It must not be used as approval to move lifecycle, -replication, or `SetDisks` runtime behavior. +**Use this when:** you need something from `rustfs_ecstore` in another crate, you are narrowing a `rustfs_ecstore::api` facade group, or the architecture guard reports a facade bypass. +**Source of truth:** `crates/ecstore/src/api/mod.rs` (facade groups), the boundary files listed below, and the facade rules in `scripts/check_architecture_migration_rules.sh`. + +The broad `rustfs_ecstore::api` facade is a compatibility boundary, not an architecture target. It shrinks monotonically and only through guarded changes; it is never approval to move lifecycle, replication, or `SetDisks` runtime behavior. ## Facade Group Inventory -| Facade group | Current role | Shrink posture | +| Facade group | Role | Shrink posture | |---|---|---| | `storage`, `layout`, `error`, `runtime`, `cluster`, `rpc` | Compatibility spine for storage, topology, runtime handles, cluster control, and internode calls. | Keep until replacement contracts compile in downstream boundary files. | -| `bucket` | Domain facade consumed through owner-local `storage_api` boundaries. The public API keeps compatibility paths but exposes explicit submodules and symbol lists instead of whole bucket owner modules. | Keep explicit lists aligned with owner-boundary consumers; do not restore whole-module passthroughs. | -| `client`, `config`, `disk`, `tier` | Compatibility paths consumed through owner-local `storage_api` boundaries. The public API keeps existing path names but exposes explicit nested submodules and symbol lists instead of whole owner modules. | Keep explicit lists aligned with owner-boundary consumers; do not restore whole-module passthroughs. | -| `data_usage`, `capacity`, `notification`, `metrics`, `rebalance` | Domain and service facades still consumed through owner-local `storage_api` boundaries. | Narrow one group at a time after explicit aliases or wrappers exist. | -| `set_disk`, `object`, `rio`, `bitrot`, `erasure`, `compression`, `cache`, `store_list` | Low-level object IO, reader, erasure, cache, and migration helper compatibility. | Keep stable while `SetDisks` remains the shared state carrier. | -| `admin`, `event`, `global` | Admin, event hook, and legacy global compatibility. | Keep `global` limited to bootstrap writes and lifecycle controls; read-only runtime access must use runtime-source contracts. | +| `bucket` | Domain facade consumed through owner-local `storage_api` boundaries; explicit submodules and symbol lists, never whole bucket owner modules. | Keep lists aligned with boundary consumers; never restore whole-module passthroughs. | +| `config`, `disk`, `tier` | Compatibility paths with explicit nested submodules and symbol lists. | Same as `bucket`. | +| `data_usage`, `capacity`, `notification`, `metrics`, `rebalance` | Domain and service facades consumed through owner-local boundaries. | Narrow one group at a time after explicit aliases or wrappers exist. | +| `set_disk`, `object`, `object_api_utils`, `rio`, `bitrot`, `erasure`, `compression`, `cache`, `store_list` | Low-level object IO, reader, erasure, cache, and migration helper compatibility. | Keep stable while `SetDisks` remains the shared state carrier. | +| `admin`, `event`, `global` | Admin, event hook, and bootstrap-global compatibility. | `global` is limited to bootstrap writes and lifecycle controls; read-only runtime access goes through `runtime`. | + +The S3 client is no longer a facade group: it lives in `crates/s3-client` (`rustfs_s3_client`). Regenerate the group list with: + +```bash +rg -n '^pub mod ' crates/ecstore/src/api/mod.rs +``` ## External Consumer Boundaries -External `rustfs_ecstore::api` imports must stay in these local boundary files: +External `rustfs_ecstore::api` imports stay in these local boundary files: -| Boundary file | Current facade families | +| Boundary file | Facade families consumed | |---|---| -| `rustfs/src/storage/storage_api.rs` | Broad RustFS storage owner bridge for admin, explicit bucket facade submodules, capacity, client, compression, cluster, config, data usage, disk, error, event, global bootstrap controls, runtime-source getters, layout, metrics, notification, rebalance, rio, rpc, set disk, storage, and tier. Replication pool/stat handles are projected into RustFS-local wrapper types here. | -| `crates/scanner/src/storage_api.rs` | Scanner bridge for bucket lifecycle, replication, metadata, capacity, config, data usage, disk, error, runtime, set disk, storage, and tier. Replication queue config, admission, and heal object DTOs are projected into scanner-local types here. | -| `crates/obs/src/metrics/storage_api.rs` | Metrics bridge for bucket bandwidth, lifecycle, replication, quota, capacity, data usage, error, runtime, and storage. | -| `crates/iam/src/storage_api.rs` | IAM bridge for config, error, notification, runtime, and storage. | -| `crates/heal/src/heal/storage_api.rs` | Heal bridge for data usage, disk, error, runtime, and storage. | -| `crates/notify/src/storage_api.rs` | Notification bridge for config, runtime, and storage. | -| `crates/protocols/src/swift/storage_api.rs` | Swift bridge for bucket metadata, bucket metadata system, error, runtime, and storage. | -| `crates/s3select-api/src/storage_api.rs` | S3 Select bridge for error, runtime, set disk, and storage. | -| `crates/e2e_test/src/storage_api.rs` | E2E harness bridge for bucket targets, disk walking, and RPC helpers. | -| `crates/heal/tests/*/storage_api.rs`, `crates/scanner/tests/storage_api/mod.rs`, `fuzz/fuzz_targets/*_storage_api.rs` | Test and fuzz bridges for the same compatibility seams under test. | +| `rustfs/src/storage/storage_api.rs` | Broad storage-owner bridge: admin, bucket submodules, capacity, compression, cluster, config, data usage, disk, error, event, global bootstrap controls, runtime getters, layout, metrics, notification, rebalance, rio, rpc, set disk, storage, tier. Replication pool/stat handles are projected into RustFS-local wrapper types here. | +| `rustfs/src/storage_api.rs`, `rustfs/src/admin/storage_api.rs`, `rustfs/src/app/storage_api.rs` | Root, admin, and app owner boundaries: explicit aliases only, no `metadata`, `metadata_sys`, `quota`, `com`, or bare `init` module passthroughs; object and error aliases anchor on storage-api associated types and a local `StorageError`. | +| `crates/scanner/src/storage_api.rs` | Bucket lifecycle, replication, metadata, capacity, config, data usage, disk, error, runtime, set disk, storage, tier. Replication queue config, admission, and heal object DTOs are projected into scanner-local types. | +| `crates/obs/src/metrics/storage_api.rs` | Bucket bandwidth, lifecycle, replication, quota, capacity, data usage, error, runtime, storage; data usage is consumed as a local DTO projection. | +| `crates/iam/src/storage_api.rs` | Config, error, notification, runtime, storage. | +| `crates/heal/src/heal/storage_api.rs` | Data usage, disk, error, runtime, storage. | +| `crates/notify/src/storage_api.rs` | Config, runtime, storage; no broad `config` or `global` module imports. | +| `crates/protocols/src/swift/storage_api.rs` | Bucket metadata, bucket metadata system, error, runtime, storage. | +| `crates/s3select-api/src/storage_api.rs` | Error, runtime, set disk, storage. | +| `crates/e2e_test/src/storage_api.rs` | E2E harness bridge for bucket targets, disk walking, and RPC helpers; no grouped RPC passthroughs. | +| `crates/ecstore/tests/storage_api.rs`, `crates/heal/tests/storage_api.rs`, `crates/scanner/tests/storage_api/mod.rs`, `fuzz/fuzz_targets/*_storage_api.rs` | Test and fuzz bridges: direct aliases or local wrappers; fuzz harnesses wrap bucket utility entrypoints instead of grouped passthroughs. | +| `crates/test-utils/src/ecstore_test_compat.rs`, `crates/iam/tests/ecstore_test_compat/mod.rs`, `crates/protocols/tests/ecstore_test_compat/mod.rs` | Test-only compatibility harnesses that import the facade directly for fixture setup. | -New production imports outside these boundary files are migration drift. Add a -local boundary or storage-api contract first, then route consumers through it. +`crates/replication/src/storage_api.rs` shares the file name but is not an ECStore boundary: it owns the delete work DTOs of `rustfs-replication`, which imports neither `rustfs_ecstore` nor `rustfs-storage-api`. + +Regenerate the boundary list with: + +```bash +rg -l 'rustfs_ecstore::api' crates rustfs/src fuzz -g '*.rs' -g '!crates/ecstore/src/**' +``` + +New production imports outside these files are migration drift. Do not add direct `rustfs_ecstore::api` imports outside the boundary files; add a local boundary or a storage-api contract first, then route consumers through it. ## Split Dependency Inventory -| Candidate | ECStore dependencies that block a crate split | Required owner contracts before movement | -|---|---|---| -| Lifecycle | Object API, `ECStore`, `SetDisks`, runtime sources/globals, bucket metadata/versioning/object lock/replication, disk, config, notification, audit, and tier services. | `LifecycleObjectStore`, `LifecycleMetadataStore`, `LifecycleRuntime`, `LifecycleReplicationSink`, and `LifecycleAuditSink`. | -| Replication | Bucket target and metadata systems, bucket target client config, disk, object API, runtime sources, notification, and SetDisks lock timing. | `ReplicationStorage`, `ReplicationMetadataStore`, `ReplicationRuntime`, `ReplicationEventSink`, and `ReplicationLifecycleBridge`. | -| SetDisks | Shared disks, endpoints, format state, namespace locks, cache, and implementations for object IO, namespace locking, bucket, object, list, multipart, and heal operations. | Pure shard source, disk error, bitrot IO, namespace lock, metrics label, and file metadata contracts before any operation family moves. | - -`crates/ecstore/tests/ecstore_contract_compat_test.rs` keeps compile-time -coverage for `ECStore` and `SetDisks` storage-api trait compatibility before -any facade shrink or operation-family movement. +Lifecycle, replication, and `SetDisks` split blockers, extracted contracts, and guard rule names are tracked in [ecstore-module-split-plan.md](ecstore-module-split-plan.md) and the module inventories `crates/ecstore/src/bucket/lifecycle/README.md` and `crates/ecstore/src/bucket/replication/README.md`. `crates/ecstore/tests/ecstore_contract_compat_test.rs` keeps compile-time coverage for `ECStore` and `SetDisks` storage-api trait compatibility before any facade shrink or operation-family movement. ## Shrink Rules -1. Do not remove a facade item until its downstream boundary has compile-time - coverage or a documented replacement. -2. Do not add direct `rustfs_ecstore::api` imports outside the boundary files - listed above. -3. Do not split lifecycle or replication into crates while they depend on - ECStore runtime state, queues, notification, audit, scanner, or SetDisks - internals. -4. Do not replace `SetDisks` with multiple runtime structs in one PR. Move one - operation family only after contracts and focused tests exist. -5. Remove or narrow one facade group per PR so rollback preserves object IO, - quorum, lifecycle/replication queues, scanner repair, notification/audit - events, and metadata compatibility. -6. Keep `api::bucket`, `api::client`, `api::config`, `api::disk`, and - `api::tier` on explicit submodules and symbol lists; do not restore - `pub use crate::::{...}` whole-module passthroughs for those groups. - -## First PR Checklist - -- inventory the facade group and all external boundary consumers; -- add explicit aliases or wrappers before deleting any broad passthrough; -- run `./scripts/check_architecture_migration_rules.sh`; -- run focused compile or tests for the touched owner boundary; -- keep runtime behavior unchanged unless the PR is explicitly a code-bearing - follow-up with its own rollback plan. +1. Do not remove a facade item until its downstream boundary has compile-time coverage or a documented replacement. +2. Do not add direct `rustfs_ecstore::api` imports outside the boundary files listed above. +3. Do not split lifecycle or replication into crates while they depend on ECStore runtime state, queues, notification, audit, scanner, or `SetDisks` internals. +4. Do not replace `SetDisks` with multiple runtime structs in one change; move one operation family only after contracts and focused tests exist. +5. Remove or narrow one facade group per change so rollback preserves object IO, quorum, lifecycle/replication queues, scanner repair, notification/audit events, and metadata compatibility. +6. Keep `api::bucket`, `api::config`, `api::disk`, and `api::tier` on explicit submodules and symbol lists; do not restore `pub use crate::::{...}` whole-module passthroughs for those groups. diff --git a/docs/architecture/ecstore-config-consumer-inventory.md b/docs/architecture/ecstore-config-consumer-inventory.md deleted file mode 100644 index 4a0a91c8a..000000000 --- a/docs/architecture/ecstore-config-consumer-inventory.md +++ /dev/null @@ -1,199 +0,0 @@ -# ECStore Config Consumer Inventory - -This inventory is the Phase 0 baseline for moving -`rustfs_ecstore::config::{Config, KV, KVS}` safely. It records the current -definitions, persistence helpers, global accessors, and direct consumers before -any contract extraction, global-state migration, or crate split. - -Related issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) - -## Scope - -In scope: - -- `rustfs_ecstore::config::KV` -- `rustfs_ecstore::config::KVS` -- `rustfs_ecstore::config::Config` -- `rustfs_ecstore::config::DEFAULT_KVS` -- `rustfs_ecstore::config::{get_global_server_config, set_global_server_config}` -- `rustfs_ecstore::config::com::{read_config_without_migrate, save_server_config}` -- Consumers that persist, clone, inspect, mutate, or pass these types across - runtime boundaries. -- Selected adjacent users of `rustfs_ecstore::config::com::{read_config, - save_config, delete_config}` and related helper variants are listed separately - when they appear outside the core `Config`, `KV`, and `KVS` consumer map. - This is not a complete `com.rs` move inventory; any future `com.rs` move must - first inventory ECStore-internal persistence helper users too. - -Out of scope: - -- Unrelated `Config` types from `rustfs::config`, SDKs, TLS, SSH, KMS, OIDC - client libraries, or local module-specific config structs. -- Storage-class-only imports are not treated as `Config`, `KV`, or `KVS` - consumers unless they also use the server-config model. -- Pure route/action snapshot work already covered by - [`admin-route-action-snapshot.md`](admin-route-action-snapshot.md). - -## Current Shape - -Arrows show current source dependency or call direction: the left node imports -or calls the right node. - -```mermaid -flowchart TB - EC["crates/ecstore/src/config"] - Store["crates/ecstore/src/store/mod.rs"] - AppCtx["rustfs/src/app/context.rs"] - Server["rustfs/src/server/{event,audit}.rs"] - Admin["rustfs/src/admin"] - Notify["crates/notify"] - Audit["crates/audit"] - Targets["crates/targets"] - IAM["crates/iam/src/oidc.rs"] - Scanner["crates/scanner/src/{runtime_config,scanner}.rs"] - - Store --> EC - AppCtx --> EC - Server --> AppCtx - Admin --> EC - Notify --> EC - Audit --> EC - Targets --> EC - Notify --> Targets - Audit --> Targets - IAM --> EC - Scanner --> EC -``` - -The config model is currently both a persisted server-config representation and -the runtime carrier for notify, audit, target-plugin, scanner, and OIDC -settings. Any move must preserve that dual role until consumers are migrated -behind narrower contracts. - -## Core Model And Global State - -| Item | Current owner | Current role | Migration note | -|---|---|---|---| -| `KV` | `crates/ecstore/src/config/mod.rs` | Key/value entry with `hidden_if_empty` metadata and serde compatibility. | Preserve field names, aliases, defaults, and redaction behavior before any model move. | -| `KVS(Vec)` | `crates/ecstore/src/config/mod.rs` | Ordered key/value set used by server config, target factories, admin rendering, tests, and examples. | Preserve tuple shape and methods: `new`, `get`, `lookup`, `is_empty`, `keys`, `insert`, `extend`. | -| `Config(HashMap>)` | `crates/ecstore/src/config/mod.rs` | Server config map by subsystem and target. | Direct `.0` access is widespread; add wrappers only after preserving the current public shape. | -| `DEFAULT_KVS` | `crates/ecstore/src/config/mod.rs` | Registry for defaults across storage class, scanner, notify, audit, and OIDC. | Move defaults only after an explicit registration contract exists. | -| `GLOBAL_SERVER_CONFIG` | `crates/ecstore/src/config/mod.rs` | Process-wide mutable server config snapshot. | Migrate readers behind `AppContext` or a server-config provider before changing storage. | -| `ConfigSys::init` | `crates/ecstore/src/config/mod.rs` | Reads persisted config, looks up derived config, and stores the global snapshot. | Startup order must remain unchanged until the lifecycle contract owns this dependency. | -| `read_config_without_migrate` | `crates/ecstore/src/config/com.rs` | Loads persisted server config through ECStore-owned object I/O and storage-admin contracts. | Persistence stays in `ecstore` until pure model and persistence are separated. | -| `save_server_config` | `crates/ecstore/src/config/com.rs` | Persists the canonical server config object. | Preserve external object shape and config-history behavior. | -| `get_global_server_config` / `set_global_server_config` | `crates/ecstore/src/config/mod.rs` | Clone/read and replace the global server-config snapshot. | Do not remove until all runtime readers have an injected provider path. | - -## Consumer Map - -### ECStore Ownership, Persistence, And Defaults - -| Files | Current usage | -|---|---| -| `crates/ecstore/src/config/mod.rs` | Defines `KV`, `KVS`, `Config`, defaults, global snapshot, initialization, and tests. | -| `crates/ecstore/src/config/com.rs` | Encodes, decodes, reads, writes, creates, and normalizes server config objects through ECStore-local persistence helpers. | -| `crates/ecstore/src/config/{notify,audit,oidc,scanner,storageclass}.rs` | Register default `KVS` values and subsystem-specific parsing helpers. | -| `crates/ecstore/src/store/mod.rs` | Exposes store-level server-config accessors that delegate to the global config snapshot. | - -### App Context And Server Startup Consumers - -| Files | Current usage | -|---|---| -| `rustfs/src/app/context.rs` | Defines `ServerConfigInterface`, keeps an `AppContext` server-config handle, and still falls back to `get_global_server_config`. | -| `rustfs/src/server/event.rs` | Resolves server config through app context/global fallback before starting the notification runtime. | -| `rustfs/src/server/audit.rs` | Resolves server config through app context/global fallback before starting the audit runtime. | - -### Admin Control-Plane Readers And Writers - -| Files | Current usage | -|---|---| -| `rustfs/src/admin/handlers/config_admin.rs` | Reads active/persisted server config, validates against `DEFAULT_KVS`, mutates `KVS`, saves config history, saves server config, and updates the global snapshot. | -| `rustfs/src/admin/handlers/oidc.rs` | Reads and writes OIDC provider `KVS`, saves server config, and compares persisted config against the global snapshot for restart signaling. | -| `rustfs/src/admin/handlers/audit_runtime_config.rs` | Reads persisted config, applies audit runtime target changes, saves server config, and reloads audit runtime state. | -| `rustfs/src/admin/handlers/notify_runtime_access.rs` | Reads notification runtime config snapshots and passes `KVS` target changes into the notification system. | -| `rustfs/src/admin/handlers/{event,audit}.rs` | Lists and validates notification/audit targets from `Config`; tests build `KV` and `KVS` fixtures. | -| `rustfs/src/admin/handlers/plugins_instances.rs` | Maps target plugin `KVS` to response payloads and applies runtime target edits. | -| `rustfs/src/admin/handlers/target_descriptor.rs` | Converts descriptor payloads into `KVS` for target plugin instances. | -| `rustfs/src/admin/handlers/site_replication.rs` | Reads global server config for LDAP settings and parses LDAP `KVS` fixtures. | -| `rustfs/src/admin/service/config.rs` | Reads persisted server config, validates storage-class `KVS`, derives target state, and updates global config/storage-class state. | -| `rustfs/src/admin/router.rs` | Reads persisted/global server config for admin route behavior; route tests construct `Config`, `KV`, and `KVS`. | - -### Adjacent ECStore Config-Object Helper Users - -| Files | Current usage | -|---|---| -| `rustfs/src/admin/handlers/kms_dynamic.rs` | Uses generic `read_config` and `save_config` for dynamic KMS config objects. | -| `rustfs/src/site_replication/state.rs` | Uses generic `read_config`, `save_config`, and `delete_config` (via the root storage facade) for site-replication state objects. | -| `rustfs/src/admin/service/site_replication.rs` | Uses generic `read_config` and `save_config` for site-replication state normalization. | -| `rustfs/src/server/module_switch.rs` | Uses generic `read_config` and `save_config` for module-switch config objects. | -| `crates/iam/src/store/object.rs` | Uses generic `read_config_no_lock`, `read_config_with_metadata`, `save_config`, `save_config_with_opts`, and `delete_config` helper variants for IAM object-store persistence paths. | -| `crates/scanner/src/{scanner,data_usage_define}.rs` | Uses generic `read_config` and `save_config` for scanner metadata and cache persistence paths. | - -### Runtime Target, Notify, And Audit Crates - -| Files | Current usage | -|---|---| -| `crates/notify/src/{global,integration,services,registry}.rs` | Carries `Config` into notification runtime startup/reload and target creation. | -| `crates/notify/src/config_manager.rs` | Mutates `Config`, reads persisted server config with `read_config_without_migrate`, persists changes with `save_server_config`, and applies per-target `KVS` updates. | -| `crates/notify/src/factory.rs` | Builds notification target arguments from `KVS`. | -| `crates/notify/examples/{full_demo,full_demo_one}.rs` | Constructs `Config`, `KV`, and `KVS` directly for examples. | -| `crates/audit/src/{global,system,registry}.rs` | Carries `Config` into audit runtime startup/reload and target creation. | -| `crates/audit/src/factory.rs` | Builds audit target arguments from `KVS`. | -| `crates/audit/tests/*.rs` | Constructs `Config` and `KVS` directly for runtime and parsing tests. | -| `crates/audit/README.md` | Documents current direct `Config` usage. | -| `crates/targets/src/plugin.rs` | Creates plugin targets from `Config` and merged `KVS`. | -| `crates/targets/src/catalog/builtin.rs` | Declares builtin target descriptors and default `KVS` fields. | -| `crates/targets/src/config/{common,target_args,loader,instance}.rs` | Collects, normalizes, redacts, and materializes target configs from `Config` and `KVS`, including environment overrides. | - -### Identity, Scanner, Tests, And Fixtures - -| Files | Current usage | -|---|---| -| `crates/iam/src/oidc.rs` | Reads global server config and parses OIDC provider `KVS`. | -| `crates/scanner/src/{runtime_config,scanner}.rs` | Reads the global server-config snapshot and resolves scanner runtime config from `Config` and `KVS`. | -| `rustfs/src/admin` handler/router tests, `crates/audit/tests/*.rs`, and selected in-crate tests in `crates/{targets,scanner}/src` | Build direct tuple-struct fixtures; use them as candidate regression guards during a pure model move. | - -## Dependency Risk Classification - -| Risk | Why it matters | Guardrail | -|---|---|---| -| `Config` is both persistence model and runtime input | A move can accidentally change persisted JSON/object shape or runtime target behavior. | Separate pure model contract from persistence helpers before moving `com.rs`. | -| Direct `.0` map access is common | Replacing the tuple struct too early would create broad churn and likely behavior drift. | Preserve tuple shape in the first move, then add typed readers in later PRs. | -| `KVS` is the effective target config carrier | Notify, audit, and target factories consume `KVS` after file/env merge. | Keep `KVS` API stable until target descriptor and runtime crates are behind a shared contract. | -| `DEFAULT_KVS` registration is global | Defaults are initialized centrally and used by admin validation/rendering. | Add a registration contract before changing initialization order. | -| Global snapshot readers still exist | Server, admin, IAM, scanner, and site-replication paths can still read global config. | Migrate readers through `AppContext`/provider paths in small steps after the model contract is stable. | -| Persistence helpers depend on ECStore storage contracts | Moving them with the pure model would pull storage implementation dependencies upward. | Keep read/write helpers in `ecstore` until a storage-facing persistence contract is explicit. | - -## Recommended Migration Order - -1. Keep this inventory current while Phase 0 guardrails land. -2. Add a focused contract surface for `KV`, `KVS`, and `Config` without changing - serialization, tuple-struct shape, or method names. -3. Add compile-time or scripted checks for temporary compatibility markers and - config-model re-export coverage. -4. Move only the pure model and defaults registration surface after targeted - regression checks cover unchanged persisted object shape, target `KVS` merge - behavior, and representative admin config rendering paths. -5. Migrate global `Config` readers behind `ServerConfigInterface` or a narrower - provider in small PRs. -6. Move persistence helpers only after object-I/O and storage-admin dependencies - can stay below the model contract. -7. Evaluate crate split only after consumers no longer need old paths except - explicit `RUSTFS_COMPAT_TODO()` compatibility shims. - -## Do-Not-Change Contract - -The first migration steps must preserve: - -- `KV { key, value, hidden_if_empty }` serde behavior and redaction semantics. -- `KVS(Vec)` tuple shape and public methods. -- `Config(HashMap>)` tuple shape and public methods. -- `Config::set_defaults`, `Config::unmarshal`, `Config::marshal`, and - `Config::merge` behavior. -- `read_config_without_migrate` fallback/creation behavior for missing server - config objects. -- `save_server_config` external object shape and config-history compatibility. -- Existing notify, audit, scanner, OIDC, and target-plugin enable/disable - interpretation from `Config`/`KVS` inputs; business-rule changes stay out of - migration PRs. -- AppContext/global fallback behavior until all readers are explicitly migrated. diff --git a/docs/architecture/ecstore-layout-boundary.md b/docs/architecture/ecstore-layout-boundary.md index 1d616a4c1..afb0cc26d 100644 --- a/docs/architecture/ecstore-layout-boundary.md +++ b/docs/architecture/ecstore-layout-boundary.md @@ -1,61 +1,39 @@ # ECStore Layout Boundary -This document records the `E-001` and `E-SET-001` foundation slice for the -architecture migration. +**Use this when:** you touch endpoint expansion, `FormatV3`, pool/set layout, or move files between ECStore's internal directories. +**Source of truth:** `crates/ecstore/src/layout/` (static layout), `crates/ecstore/src/core/sets.rs` (`Sets`) and `crates/ecstore/src/set_disk/mod.rs` (`SetDisks`) for runtime orchestration, `crates/ecstore/src/api/mod.rs` (`pub mod layout`) for the public surface. -## Directory Skeleton +## Directory Ownership -The ECStore migration uses these internal ownership buckets before any pure -file moves: +Ownership buckets under `crates/ecstore/src` (a subset; list the rest with `ls crates/ecstore/src`): -- `api`: facade and compatibility re-export ownership. -- `core`: store facade, object, bucket, list, multipart, and heal paths. -- `layout`: static endpoint, disk, pool, and set layout descriptions. -- `disk`: local disk, format, health, and disk error ownership. -- `erasure`: erasure coding and bitrot ownership. -- `metadata`: bucket metadata, config object-store, and data-usage ownership. -- `cluster`: remote disk, peer, lock, membership, and health-control-plane - ownership. -- `services`: lifecycle, replication, tier, notification, rebalance, and - metrics service ownership. +| Directory | Owns | +|---|---| +| `api` | Facade and compatibility re-exports | +| `core` | Store facade, pools, sets, and object/bucket/list/multipart/heal orchestration | +| `layout` | Static endpoint, disk, pool, and set layout (`disks_layout`, `endpoint`, `endpoints`, `format`, `pool_space`, `set_heal`, `set_layout`) | +| `disk` | Local disk, format compatibility, health, disk errors | +| `erasure` | Erasure coding and bitrot | +| `metadata` | Bucket metadata, config object store, data usage | +| `cluster` | Remote disk, peer, lock, membership, health control plane | +| `services` | Lifecycle, replication, tier, notification, rebalance, metrics services | +| `set_disk`, `store`, `data_movement`, `data_usage`, `object_api`, `runtime` | Set-level operations, store init, pool data movement, usage accounting, object API helpers, runtime state owners | ## Static Set Layout -Static layout is derived from persisted `FormatV3` data and input endpoint -expansion. It may describe: +Static layout is derived from persisted `FormatV3` data (`crates/ecstore/src/layout/format.rs`) and endpoint expansion (`crates/ecstore/src/layout/disks_layout.rs`). It may describe the deployment id, set count and drives per set, disk UUID positions inside `format.erasure.sets`, the distribution algorithm, and endpoint grouping produced before runtime disk initialization. It must not own disk handles, lock clients, reconnect loops, repair state, or shutdown signaling. -- deployment id; -- set count and drives per set; -- disk UUID positions inside `format.erasure.sets`; -- distribution algorithm; -- endpoint grouping produced before runtime disk initialization. +## Visibility -Static layout must not own disk handles, lock clients, reconnect loops, repair -state, or shutdown signaling. - -## Format And Disk Layout Ownership - -`layout::format` owns persisted format structures and disk UUID position lookup. -`layout::disks_layout` owns command-line volume expansion into pool/set layout. - -Compatibility paths remain available through `disk::format` and `disks_layout` -until downstream callers are moved or compatibility coverage allows removal. +`layout::*` modules are `pub(crate)`; public access goes through `rustfs_ecstore::api::layout` (`DisksLayout`, `EndpointServerPools`, `Endpoints`, `PoolEndpoints`, `SetupType`). `disk::format` re-exports `layout::format` for crate-internal callers. Outer crates must not reach the root `endpoints` or `disks_layout` modules. ## Runtime Set Orchestration -Runtime orchestration remains owned by `Sets` and `SetDisks` until a later pure -move. It may describe: - -- flat disk index to `(set_index, disk_index)` mapping; -- per-set local disk replacement after distributed setup detection; -- per-set lock-client host deduplication; -- endpoint reconnect monitoring and runtime shutdown signaling; -- read/write/heal/list orchestration over initialized disks. +`Sets` and `SetDisks` own the flat disk index to `(set_index, disk_index)` mapping, per-set local disk replacement after distributed setup detection, per-set lock-client host deduplication, endpoint reconnect monitoring and runtime shutdown signaling, and read/write/heal/list orchestration over initialized disks. ## Preservation Rules - Object-to-set hashing and distribution algorithm selection must not change. - Format `sets` ordering and disk UUID position lookup must not change. - Local disk replacement and lock-client mapping stay runtime-only. -- Later file moves must keep old public paths or add explicit compatibility - coverage before deleting them. +- File moves keep old public paths or add explicit compatibility coverage before deleting them. diff --git a/docs/architecture/ecstore-module-split-plan.md b/docs/architecture/ecstore-module-split-plan.md index ef5f12362..eb26b73f4 100644 --- a/docs/architecture/ecstore-module-split-plan.md +++ b/docs/architecture/ecstore-module-split-plan.md @@ -1,353 +1,85 @@ # ECStore Module Split Plan -This plan records the remaining ECStore split work after the final audit -remediation pass. Runtime movement must still wait until each candidate -boundary has explicit contracts, compatibility coverage, dependency evidence, -and rollback steps. +**Use this when:** you add lifecycle or replication logic and need to know which crate it belongs in, you plan to move an operation family out of `SetDisks`, or the guard fails on one of the split rules named below. +**Source of truth:** `scripts/check_architecture_migration_rules.sh` (the rules), `crates/ecstore/src/bucket/lifecycle/README.md` and `crates/ecstore/src/bucket/replication/README.md` (module-level contract inventories, completion criteria, milestones), and [ecstore-api-facade-inventory.md](ecstore-api-facade-inventory.md) (facade groups and boundary files). ## Current Shape -| Area | Current owner | Size | Split status | -|---|---|---:|---| -| Bucket lifecycle | `crates/lifecycle/` + `crates/ecstore/src/bucket/lifecycle/` | core contracts + ECStore runtime | Core contract extracted | -| Bucket replication | `crates/ecstore/src/bucket/replication/` | 15,619 lines | Contracts extracted; runtime move pending | -| Set disks | `crates/ecstore/src/set_disk/` | state carrier plus operation modules | Keep in ECStore | -| Public ECStore facade | `crates/ecstore/src/api/mod.rs` | broad compatibility surface | Shrink only through guarded PRs | -| Embedded S3 client | `crates/s3-client/` (`rustfs-s3-client`) | ~8.4K lines | Extracted (rustfs/backlog#1842) | +| Area | Owner | Split status | +|---|---|---| +| Bucket lifecycle | `crates/lifecycle/` (`rustfs-lifecycle`, pure contracts) + `crates/ecstore/src/bucket/lifecycle/` (runtime) | Core contracts extracted; runtime stays in ECStore | +| Bucket replication | `crates/replication/` (`rustfs-replication`, contracts and wire formats) + `crates/ecstore/src/bucket/replication/` (worker runtime) | Contracts extracted; runtime move pending | +| Set disks | `crates/ecstore/src/set_disk/` | Shared state carrier plus operation modules; stays in ECStore | +| Public facade | `crates/ecstore/src/api/mod.rs` | Shrinks only through guarded changes | +| S3 client | `crates/s3-client/` (`rustfs-s3-client`) | Extracted | -Measured 2026-08-12: the whole crate is 265 files / ~288K lines (roughly half -is inline `#[cfg(test)]` code). The largest single files are `disk/local.rs` -(21,063 lines), `bucket/lifecycle/bucket_lifecycle_ops.rs` (11,961 lines), and -`set_disk/mod.rs` (11,151 lines). Reproduce with: +Measure size instead of trusting numbers in a document: ```bash find crates/ecstore/src -name '*.rs' | xargs wc -l | sort -rn | head find crates/ecstore/src/bucket/replication -name '*.rs' | xargs wc -l | tail -1 ``` -No split step has landed since the contract-extraction PRs of 2026-07-04, -while the `bucket/replication` runtime grew from 8,730 to 15,619 lines (+79%) -through feature work (e.g. SSE-C ciphertext passthrough replication #5898, -delete-marker purge retry/replay #5864). To keep the gap from widening: in -domains that already have a contract crate, new replication runtime logic that -does not need ECStore runtime state must land in `rustfs-replication`, not in -`crates/ecstore/src/bucket/replication/`. +Rule for new code: in a domain that already has a contract crate, new logic that does not need ECStore runtime state lands in that crate (`rustfs-lifecycle`, `rustfs-replication`), not under `crates/ecstore/src/bucket/`. -The file split inside `set_disk/` is already operation-oriented: read, write, -list, multipart, lock, heal, and replication code live in separate modules. -The remaining large surface is the shared `SetDisks` state and cross-cutting -contracts, not only file layout. - -## Completed: S3 Client Extraction (rustfs/backlog#1842) - -`crates/ecstore/src/client/` was a ~8.4K-line hand-written S3 HTTP client the engine uses to *consume* remote S3-compatible endpoints (ILM tier warm backends, transition targets). It was a legitimate engine capability misfiled inside the engine: it pulled `s3s`/`hyper` wire types into ecstore against ARCHITECTURE.md invariant 4, which distinguishes serving the S3 wire protocol (forbidden in ecstore) from consuming it (allowed, but in a dedicated crate). - -The extraction landed as: pure move of the 21 client modules to `crates/s3-client` (`rustfs-s3-client`) with a temporary re-export shim, then direct `rustfs_s3_client::` imports and shim deletion. The two server-side modules historically misfiled under `client/` stayed in ecstore and moved to their real homes: `object_api_utils.rs` under `object_api/`, `object_handlers_common.rs` under `bucket/lifecycle/` (behind the `replication_sink` boundary). The remaining serving-side `s3s` references in ecstore are ratcheted shrink-only by the `S3S_ECSTORE_FILES_BASELINE` counter in `scripts/check_s3s_footprint.sh`; per-module conversions to storage-level types (first: `bucket/object_lock/`) lower the baseline in the same change. +The S3 client extraction is complete: the former `client/` directory moved to `crates/s3-client`, its two server-side modules moved to `crates/ecstore/src/object_api/object_api_utils.rs` and `crates/ecstore/src/bucket/lifecycle/object_handlers_common.rs`, and the remaining serving-side `s3s` references in ECStore are ratcheted shrink-only by `S3S_ECSTORE_FILES_BASELINE` in `scripts/check_s3s_footprint.sh`. ## Non-Negotiable Rules -- Do not split crates in the same PR that moves runtime state or changes - startup behavior. -- Do not change object placement, quorum, reader semantics, lifecycle queues, - replication queues, notification dispatch, audit events, or scanner repair - behavior during inventory and contract PRs. -- Do not expose new direct ECStore internals to outer crates; use the existing - storage-api and owner-local facade boundaries. -- Keep `rustfs_ecstore::api` compatibility visible until each consumer path has - compile coverage and an explicit replacement. +- Do not split crates in the same change that moves runtime state or changes startup behavior. +- Do not change object placement, quorum, reader semantics, lifecycle queues, replication queues, notification dispatch, audit events, or scanner repair behavior during inventory and contract work. +- Do not expose new direct ECStore internals to outer crates; use storage-api and owner-local facade boundaries. +- Keep `rustfs_ecstore::api` compatibility visible until each consumer path has compile coverage and an explicit replacement. -## SetDisks Split Direction +## Guarded Split Rules -Do not replace `SetDisks` with several runtime structs in one change. The safe -path is: +Each rule is enforced by `scripts/check_architecture_migration_rules.sh`; the name is the vocabulary used in reviews and guard failures. -1. Keep `SetDisks` as the shared state carrier while operation modules continue - to own read/write/list/multipart/lock/heal/replication behavior. -2. Extract pure contracts first: shard source, disk error, bitrot IO, namespace - lock, metrics labels, and file metadata access. -3. Move one operation family only after its contracts are covered by focused - tests and the facade compatibility path is explicit. -4. Preserve the old `rustfs_ecstore::api::set_disk` surface until downstream - compatibility tests prove no caller depends on removed names. +| Rule | What the guard checks | +|---|---| +| `LifecycleCrateCoreIndependence` | `crates/lifecycle` (rule validation, filtering, event evaluation, transition/expiration options, tag decoding, object-lock metadata checks, expiry-time rounding) imports no ECStore internals, `rustfs-filemeta`, or `rustfs-utils`; ECStore owns the `ObjectInfo` adapter in `crates/ecstore/src/bucket/lifecycle/core.rs`. | +| `ReplicationCrateFileMetaIndependence` | Replication status, decision, MRF, resync, and target-reset wire contracts live in `crates/replication/src/filemeta.rs`; `rustfs-replication` neither imports nor depends on `rustfs-filemeta`. | +| `ReplicationCrateStorageApiIndependence` | Delete work DTOs live in `crates/replication/src/storage_api.rs`; ECStore converts storage-api delete DTOs at its replication storage boundary; `rustfs-replication` does not depend on `rustfs-storage-api`. | +| `ReplicationCrateUtilsIndependence` | HTTP metadata keys, S3 header labels, ETag trimming, and prefix matching used by replication wire contracts live in `crates/replication/src/http.rs`; `rustfs-replication` does not depend on `rustfs-utils`. | +| `EcstoreReplicationBoundaryImports` | ECStore-side `rustfs_replication` imports are confined to the `*_boundary.rs` modules under `crates/ecstore/src/bucket/replication/`; grouped queue, stats, resync, and object-decision symbols each have one owning boundary file. | +| `RuntimeReplicationFacadeConsumers` | Scanner, admin, storage-owner, and app code consume replication status/DTO/helper contracts through the `rustfs_ecstore` facade; the `rustfs` and `rustfs-scanner` crates do not depend on `rustfs-replication` directly. | +| `StorageApiReplicationContracts` | Owner-facing storage-api delete DTO replication state/status helpers stay in `crates/storage-api/src/replication.rs`; replication worker DTOs stay in `rustfs-replication`. | -The first executable SetDisks follow-up should be an inventory or guardrail PR, -not a runtime split PR. +## Lifecycle -## Lifecycle Candidate +`rustfs-lifecycle` owns the pure rule, event, evaluator, tag-filter, object-lock metadata check, and expiry-time contracts. ECStore keeps the object-store runtime, queues, tiering, audit/notification, metadata access (`crates/ecstore/src/bucket/lifecycle/metadata_boundary.rs`), and replication-delete scheduling adapters. -`rustfs-lifecycle` now owns the pure lifecycle rule, event, evaluator, tag -filtering, object-lock metadata check, and expiry-time contracts. ECStore keeps -the object-store runtime, queues, tiering, audit/notification, metadata, and -replication scheduling adapters. +Coupling that still blocks a runtime move: lifecycle workers read ECStore runtime sources (object store, expiry and transition state, tier config, deployment id, local node name); stale multipart cleanup depends on `SetDisks` internals and bucket metadata; expiry schedules replication deletes through the replication lifecycle bridge; the lifecycle runtime coordinates scanner metrics and notification/audit side effects. The contract list and the next step live in `crates/ecstore/src/bucket/lifecycle/README.md`. -Current coupling: +## Replication -- lifecycle workers and transition state read ECStore runtime sources for - object-store handles, expiry state, transition state, tier config, deployment - IDs, and local node names; -- stale multipart cleanup depends on `SetDisks` internals and bucket metadata - through the lifecycle metadata boundary; -- lifecycle expiry schedules bucket replication delete work through the - replication lifecycle bridge contract; -- lifecycle evaluation uses S3 DTOs and replication status contracts from the - independent `rustfs-lifecycle`/`rustfs-replication` crates, while ECStore maps - `ObjectInfo` into lifecycle object options at the compatibility boundary; -- lifecycle runtime still coordinates scanner metrics, notification/audit side - effects, metadata access, replication delete scheduling, and tier services. +`rustfs-replication` owns resync status contracts, the persisted resync status wire format, filemeta-derived wire contracts, delete work DTOs, and HTTP helper contracts. ECStore keeps the worker runtime, error mapping, MRF persistence, and global pool/stat initialization. -Current extracted contracts: +Boundary layout inside `crates/ecstore/src/bucket/replication/`: `*_boundary.rs` modules concentrate imports from `rustfs-replication`, storage-api, filemeta, config, target, error, lock, msgp, versioning, tagging, bandwidth, queue, stats, resync, and object-decision surfaces; `replication_*_bridge.rs` modules (lifecycle, scanner, object, migration, target-config) expose replication scheduling to other owners without leaking DTO construction; `replication_config_store.rs` exposes config persistence and storage-class labels. Modules inside the directory use relative self-imports, and the facade in `mod.rs` uses explicit symbol lists, never wildcard re-exports. -- `LifecycleCrateCoreIndependence`: lifecycle rule validation, filtering, - event evaluation, transition/expiration options, tag decoding, object-lock - metadata checks, and ILM expiry-time rounding live in `rustfs-lifecycle`. - `rustfs-lifecycle` must not import ECStore internals, file metadata, or - `rustfs-utils`; ECStore owns the `ObjectInfo` adapter in - `crates/ecstore/src/bucket/lifecycle/core.rs`. +Consumers outside ECStore: RustFS runtime code receives pool/stat handles through storage-owner wrapper types in `rustfs/src/storage/storage_api.rs`; scanner code receives scanner-local config/admission/heal DTOs from `crates/scanner/src/storage_api.rs`; observability reads replication metrics through obs-local snapshot DTOs in `crates/obs/src/metrics/storage_api.rs`; app object and multipart writes call object-replication bridge helpers instead of constructing replication work DTOs. -Required contracts before crate movement: +Completion criteria, the milestone order, and the per-dependency contract inventory live in `crates/ecstore/src/bucket/replication/README.md` (sections "Completion Criteria" and "Milestones"). Remaining work starts from moving resyncer pure decision logic. -- `LifecycleObjectStore`: object stat, delete, transition, restore, multipart - cleanup, and version-aware metadata operations needed by lifecycle workers. -- `LifecycleMetadataStore`: lifecycle, object-lock, replication, bucket - versioning, and stale multipart metadata lookups without importing ECStore - implementation modules. Current lifecycle config reads are concentrated in - `crates/ecstore/src/bucket/lifecycle/metadata_boundary.rs`. -- `LifecycleRuntime`: expiry state, transition state, tier config, deployment - ID, local node name, queue metrics, cancellation, and worker sizing. -- `LifecycleReplicationSink`: schedule lifecycle-originated replication deletes - without depending on the replication implementation module. -- `LifecycleAuditSink`: lifecycle audit and notification emission boundary. +## SetDisks -Next safe PR: +Do not replace `SetDisks` with several runtime structs in one change: -- move one runtime-facing dependency behind a trait or adapter owned by - `rustfs-lifecycle` without changing queue, transition, or delete behavior; -- keep ECStore compatibility shims until scanner and RustFS app consumers stop - depending on `rustfs_ecstore::api::bucket::lifecycle` paths; -- add focused tests for the moved contract and keep architecture guard coverage. +1. Keep `SetDisks` as the shared state carrier while operation modules own read/write/list/multipart/lock/heal/replication behavior. +2. Extract pure contracts first: shard source, disk error, bitrot IO, namespace lock, metrics labels, and file metadata access. +3. Move one operation family only after its contracts are covered by focused tests and the facade compatibility path is explicit. +4. Preserve the `rustfs_ecstore::api::set_disk` surface until downstream compatibility tests prove no caller depends on removed names. -The module-level inventory lives in -`crates/ecstore/src/bucket/lifecycle/README.md`. +## Facade Shrink -Focused verification for the first code-bearing lifecycle PR: - -- `cargo test -p rustfs-ecstore lifecycle --lib` -- `cargo check -p rustfs-ecstore --tests` -- `./scripts/check_architecture_migration_rules.sh` -- `git diff --check` - -## Replication Candidate - -`rustfs-replication` now owns the resync status contracts and persisted resync -status wire format. The remaining `bucket/replication` worker runtime is not -ready for a full standalone crate yet. - -The completion criteria and milestone sequence for this candidate (when the -split counts as done, the target end state, and the order of the remaining -moves) live in the module inventory: -`crates/ecstore/src/bucket/replication/README.md`, sections "Completion -Criteria" and "Milestones". The originally proposed first code-bearing step -(event sink / runtime contracts) has landed; remaining work starts from moving -resyncer pure decision logic. - -Current coupling: - -- replication workers depend on `ReplicationStorage`, ECStore object APIs and - owner storage-api contracts through the replication storage boundary, bucket - target clients, bucket metadata, file metadata replication state through the - filemeta boundary, config-derived storage class labels through the config store, scanner repair - classification, runtime replication pool/stat handles, bucket monitor and - bandwidth reader access through local boundaries, local node names, and - notification events; -- resync and delete replication paths call metadata paths through the metadata - boundary, while bucket target system access, target config types, and target - operation types are concentrated behind the replication target boundary; -- lifecycle delete paths schedule replication work through - `ReplicationLifecycleBridge`, while scanner heal paths schedule replication - work through `ReplicationScannerBridge`, and app/SetDisks object write/delete - paths use `ReplicationObjectBridge`; -- bucket metadata migration and bucket target removal checks use local - replication bridges instead of importing resyncer codec or config helper - internals; -- resync options, bucket/target resync status DTOs, status display labels, and - the persisted resync status wire format live in `crates/replication`, with - ECStore retaining only error mapping and MRF persistence locally; -- `ReplicationCrateFileMetaIndependence`: replication status, decision, MRF, - resync, and target-reset wire contracts are owned inside `rustfs-replication` - instead of importing `rustfs-filemeta`; -- `ReplicationCrateStorageApiIndependence`: delete work DTOs are owned inside - `rustfs-replication`; ECStore converts storage-api delete DTOs at the - replication storage boundary instead of `rustfs-replication` importing - `rustfs-storage-api`; -- `ReplicationCrateUtilsIndependence`: HTTP metadata keys, S3 header labels, - ETag trimming, and case-insensitive prefix matching used by replication wire - contracts are owned inside `rustfs-replication` instead of importing - `rustfs-utils`; -- direct ECStore replication imports from `rustfs-replication` are limited to - `*_boundary.rs` modules; -- storage-api delete replication status/state helpers use the local - `crates/storage-api/src/replication.rs` contract boundary; ECStore converts - those owner DTOs at the replication storage boundary before queueing work; -- admin replication extension target filtering and resync request construction - stay behind the admin storage boundary instead of exposing replication work - DTO construction to handlers; -- scanner, admin, storage-owner, and app storage replication status/DTO/helper - consumers import those contracts through the ECStore replication facade; -- app object and multipart writes call object-replication boundary helpers - instead of constructing replication work DTOs or choosing object replication - operation types at the use-case layer; -- RustFS runtime consumers receive replication pool/stat handles through - storage-owner wrapper types instead of carrying ECStore replication handles - through app, admin, startup, or workload-admission layers; -- global replication pool/stat initialization still lives with ECStore runtime - compatibility state; -- modules inside `bucket/replication` use local relative paths rather than the - ECStore owner path for replication self-imports; -- replication runtime source access uses storage/bandwidth boundary aliases for - ECStore object store and bucket monitor implementation types; -- the ECStore replication facade in `mod.rs` uses explicit compatibility - exports instead of wildcard re-exports from implementation modules. - -Required contracts before crate movement: - -- `ReplicationObjectIO`: object read/write primitives for config, MRF, resync - status, and multipart replication paths. ECStore object API reader/writer - types and storage-api object IO contracts are concentrated in - `crates/ecstore/src/bucket/replication/replication_storage_boundary.rs`. -- `ReplicationStorage`: keep the existing trait as the starting point, then - split object read/write/delete, walk, and metadata update responsibilities - only when call sites prove a narrower shape. ECStore object API, - storage-api contracts, and read option types are concentrated in - `crates/ecstore/src/bucket/replication/replication_storage_boundary.rs`. -- `ReplicationMetadataStore`: replication config, target reset headers, - MRF/resync state, and status persistence. Metadata sys access and replication - metadata path constants are exposed through the contract type in - `crates/ecstore/src/bucket/replication/replication_metadata_boundary.rs`. -- `ReplicationConfigStore`: replication config persistence and config-derived - labels used by target options. Config read/save helpers and storage class - labels are exposed through the contract type in - `crates/ecstore/src/bucket/replication/replication_config_store.rs`. -- `ReplicationFileMeta`: replication status, decisions, MRF entries, resync - decisions, and target reset helpers. ECStore concentrates filemeta-to- - replication compatibility conversions in - `crates/ecstore/src/bucket/replication/replication_filemeta_boundary.rs`, - while `FileInfo` remains in the storage boundary for storage trait bindings - and walk options. -- `ReplicationCrateFileMetaIndependence`: filemeta wire contracts consumed by - replication workers are owned in `crates/replication/src/filemeta.rs`, and - `rustfs-replication` must not import or depend on `rustfs-filemeta`. -- `ReplicationCrateStorageApiIndependence`: delete work DTOs consumed by - replication delete/queue/operation helpers are owned in - `crates/replication/src/storage_api.rs`, and `rustfs-replication` must not - import or depend on `rustfs-storage-api`. -- `ReplicationCrateUtilsIndependence`: replication-specific HTTP metadata, - header, ETag, and prefix helper contracts are owned in - `crates/replication/src/http.rs`, and `rustfs-replication` must not import or - depend on `rustfs-utils`. -- `EcstoreReplicationBoundaryImports`: ECStore-side imports from - `rustfs-replication` are concentrated in replication `*_boundary.rs` modules. -- `RuntimeReplicationFacadeConsumers`: scanner, admin, storage-owner, and app - storage replication status/DTO/helper consumers import through - `rustfs-ecstore`; runtime code under `rustfs/src` does not import - `rustfs-replication` directly, and the RustFS runtime/scanner crates do not - depend on it. -- `StorageApiReplicationContracts`: owner-facing storage-api delete DTO - replication state/status helpers remain concentrated in - `crates/storage-api/src/replication.rs`, while replication worker DTOs live in - `rustfs-replication`. -- `ReplicationErrorBoundary`: ECStore error/result contracts and - replication-specific error classifiers. `crate::error` imports are - concentrated in - `crates/ecstore/src/bucket/replication/replication_error_boundary.rs`. -- `ReplicationTargetStore`: bucket target listing, target client lookup, - target offline checks, target config types, and target operation option - types. Bucket target sys access, `BucketTargets`, and target operation types - are exposed through the contract type in - `crates/ecstore/src/bucket/replication/replication_target_boundary.rs`. -- `ReplicationRuntime`: pool, stats, worker admission, bucket monitor, local - node identity, cancellation, and queue sizing. Concrete ECStore object store - and bucket monitor types stay behind local storage/bandwidth boundaries. -- `ReplicationBandwidthLimiter`: target reader wrapping for replication - bandwidth accounting and throttling. -- `ReplicationVersioningStore`, `ReplicationLockTiming`, `ReplicationMsgpCodec`, - and `ReplicationTagFilter`: smaller state/codec/filter contracts that keep - bucket versioning, SetDisks lock timing, MessagePack helpers, and bucket - tagging helper access behind local replication boundary types. -- `ReplicationEventSink`: notification/audit events for skipped, failed, and - completed replication operations, including local event host selection. -- `ReplicationLifecycleBridge`: lifecycle-originated delete and version-purge - scheduling is exposed through the contract type in - `crates/ecstore/src/bucket/replication/replication_lifecycle_bridge.rs`. -- `ReplicationMigrationBridge`: persisted resync status decode/encode access - for bucket metadata migration is exposed through the contract type in - `crates/ecstore/src/bucket/replication/replication_migration_bridge.rs`. -- `ReplicationResyncContracts`: resync options, target/bucket resync status, - status labels, and persisted status encoding live in `crates/replication`. -- `ReplicationObjectBridge`: app and SetDisks object write/delete replication - decisions and scheduling are exposed through the contract type in - `crates/ecstore/src/bucket/replication/replication_object_bridge.rs`. -- `ObsReplicationStatsSnapshot`: observability reads replication bucket/site - metrics through obs-local snapshot DTOs in - `crates/obs/src/metrics/storage_api.rs` instead of carrying the ECStore - replication stats handle through collectors. -- `StorageReplicationPoolHandle` / `StorageReplicationStatsHandle`: RustFS app, admin, - startup, and workload-admission code use storage-owner wrapper types from - `rustfs/src/storage/storage_api.rs` for pool activity, resync, queue counts, - proxy stats, and site metrics snapshots. -- `ReplicationScannerBridge`: scanner-originated replication heal scheduling is - exposed through the contract type in - `crates/ecstore/src/bucket/replication/replication_scanner_bridge.rs`. - Scanner consumers receive scanner-local replication config/admission/heal - object DTOs from `crates/scanner/src/storage_api.rs` instead of constructing - or inspecting replication queue DTOs directly. -- `ReplicationTargetConfigBridge`: bucket target removal checks against - replication target rules are exposed through the contract type in - `crates/ecstore/src/bucket/replication/replication_target_config_bridge.rs`. -- `ReplicationFacade`: the current `rustfs_ecstore::api::bucket::replication` - compatibility surface is an explicit symbol list guarded against wildcard - re-exports while downstream owners migrate to narrower contracts. - -First safe PR: - -- add a replication extraction inventory section or module-level README; -- list current ECStore/runtime dependencies and the target contract owner for - each dependency; -- keep global pool/stat initialization and queue behavior unchanged. - -The module-level inventory lives in -`crates/ecstore/src/bucket/replication/README.md`. - -Focused verification for the first code-bearing replication PR: - -- `cargo test -p rustfs-ecstore replication --lib` -- `cargo check -p rustfs-ecstore --tests` -- `./scripts/check_architecture_migration_rules.sh` -- `git diff --check` - -## Facade Shrink Plan - -The broad `rustfs_ecstore::api` facade remains a compatibility boundary, not a -new architecture target. The current facade groups and external consumers are -recorded in -[`ecstore-api-facade-inventory.md`](ecstore-api-facade-inventory.md). -Shrinking it must be monotonic: - -1. Inventory every public facade group and consumer. -2. Add compile-time coverage before removing or narrowing a facade item. -3. Move outer consumers to storage-api or owner-local compatibility boundaries. -4. Remove one facade group per PR only after downstream compatibility tests pass. - -Do not delete facade groups only because the underlying module moved. Keep the -facade stable until the replacement path is visible and tested. +Facade groups, boundary files, and shrink rules are in [ecstore-api-facade-inventory.md](ecstore-api-facade-inventory.md). Shrinking is monotonic: inventory, add compile-time coverage, move consumers to storage-api or owner-local boundaries, then remove one group per change. Do not delete facade groups only because the underlying module moved. ## Ready-To-Split Checklist -A candidate split is ready for code movement only when all items below are true: +A candidate is ready for code movement only when all of these hold: -- dependency graph shows no cycle with ECStore, storage-api, runtime sources, or - owner-local compatibility modules; +- the dependency graph shows no cycle with ECStore, storage-api, runtime sources, or owner-local compatibility modules; - contract traits compile without importing ECStore implementation modules; - old facade names have compatibility tests or explicit deprecation coverage; - focused tests cover the changed owner path before any full gate is attempted; -- rollback preserves object IO, quorum, lifecycle/replication queues, scanner - repair, notification/audit events, and metadata compatibility. +- rollback preserves object IO, quorum, lifecycle/replication queues, scanner repair, notification/audit events, and metadata compatibility. diff --git a/docs/architecture/erasure-coding.md b/docs/architecture/erasure-coding.md index c5f1b4e11..921f845e7 100644 --- a/docs/architecture/erasure-coding.md +++ b/docs/architecture/erasure-coding.md @@ -1,10 +1,13 @@ # Erasure Coding — Normative Algorithm & On-Disk Compatibility Contract +**Use this when:** changing anything under `crates/ecstore/src/erasure/`, `crates/filemeta/`, `crates/ecstore/src/set_disk/`, storage-class or layout code, or any decode, quorum, or heal boundary; read §12 and §13 before editing. +**Source of truth:** this document is normative for the algorithm and the on-disk / on-wire compatibility contract; the cited symbols are where the code enforces each rule. + Status: normative. This document is the source of truth for how RustFS erasure-codes, stores, reads, reconstructs, and heals user data, and for the on-disk / on-wire compatibility contract that every future change must preserve. It governs the highest-risk code in the system: a regression here can silently corrupt or lose all user data, or make existing (and MinIO-migrated) objects permanently unreadable. -Erasure coding, quorum/heal, and metadata/on-disk formats are **High-risk** per [AGENTS.md](../../AGENTS.md) ("Risk tiers"). Any behavior-affecting change to code this document governs requires the full seven-role adversarial validation and, for anything touching decode or the on-disk format, a regression test against real on-disk and MinIO-migrated samples before merge. +Erasure coding, quorum/heal, and metadata/on-disk formats are **High-risk** per [AGENTS.md](../../AGENTS.md) ("Broad or High-Risk Changes"). Any behavior-affecting change to code this document governs requires adversarial review with the `adversarial-validation` skill and, for anything touching decode or the on-disk format, a regression test against real on-disk and MinIO-migrated samples before merge. -This document describes the baseline (`main`) algorithm. Where the baseline has a known defect that a specific change corrects, that is called out inline; the *invariant* stated is always the correct rule the code must converge to, never the defect. +This document describes the algorithm as implemented on `main`. The *invariant* stated is always the rule the code must satisfy; where the code enforces it, the enforcing symbol is cited. ## How to use this document @@ -34,7 +37,6 @@ This document describes the baseline (`main`) algorithm. Where the baseline has 11. Compatibility contract and decode tolerance 12. Invariants checklist (the frozen contract) 13. Change procedure and guardrails -14. References --- @@ -80,7 +82,7 @@ Two storage classes: `STANDARD` (SC) and `REDUCED_REDUNDANCY` (RRS) ([storagecla - **INVARIANT — parity bounds.** Parity must satisfy `parity ≤ N/2` for both classes, and `SC parity ≥ RRS parity` when both are non-zero ([storageclass.rs](../../crates/ecstore/src/config/storageclass.rs), `validate_parity` / `validate_parity_inner`). Enforcement nuance to be aware of: `validate_parity_inner` (the path a user-configured `EC:` storage class flows through) only applies the `parity ≤ N/2` check for `N > 2`, so degenerate small-set values (e.g. `EC:2` on `N = 2`, giving `data_blocks = 0`) are not caught there; the standalone `validate_parity` enforces the bound unconditionally but is applied only to the resolved default parity. A change that lets user-configured parity reach a write path must not assume the `≤ N/2` bound was enforced for `N ≤ 2`. Parity `0` is permitted (single-drive / capacity setups); there is no non-zero minimum. - **INVARIANT — per-pool validity.** Each pool's resolved parity must be valid for **that pool's own drive count**. A heterogeneous deployment (pools of different widths) must resolve parity per pool; applying one pool's parity to a narrower pool can drive `data_blocks = N − parity` to `0` and make encoding impossible. - - Baseline defect: `main` computes `common_parity_drives` from the **first** pool only and applies it to every pool ([store/init.rs](../../crates/ecstore/src/store/init.rs), `ec_drives_no_config` at [store/init_format.rs](../../crates/ecstore/src/store/init_format.rs)); this is issue #4801 (a smaller later pool panics with `TooFewDataShards`). The correct rule is per-pool resolution. + - Implemented by `resolve_write_layout` ([set_disk/mod.rs](../../crates/ecstore/src/set_disk/mod.rs)), which takes the pool index and resolves parity against that pool's own drive count; the default parity for a pool without explicit config comes from `ec_drives_no_config` ([store/init_format.rs](../../crates/ecstore/src/store/init_format.rs)). Per-write layout (the numbers that go into `xl.meta`), from the storage class or `default_parity_count`, with `opts.max_parity` forcing `N/2` for internal writes ([set_disk/ops/object.rs](../../crates/ecstore/src/set_disk/ops/object.rs)): @@ -219,7 +221,7 @@ Fields: `version_id`, `mod_time`, `signature: [u8;4]`, `version_type`, `flags: u ### 6.6 Inline data -Small objects store their payload inline after the container CRC ([filemeta_inline.rs](../../crates/filemeta/src/filemeta_inline.rs)): 1 version byte (`INLINE_DATA_VER = 1`) then a msgpack map of `version-key → bin`. **INVARIANT — the map key** is the version-id string, `"null"` (`NULL_VERSION_ID`) for the null/None version, else the lowercase hyphenated UUID. Presence is determined **on read** solely by the `meta_sys[inline-data]` body marker (`FileInfo::inline_data`); the read path gates inline extraction on that marker alone. The header `InlineData` flag is **written** (mirrored from the body on marshal) but is **not** consulted on read, and a disagreement is tolerated — MinIO may leave the header flag unset while inline data is present, so a reader must **not** require the flag and the marker to agree. The inline threshold is `should_inline` ([storageclass.rs](../../crates/ecstore/src/config/storageclass.rs)): inline if `shard_size ≤ inline_block/8` for versioned buckets, else `≤ inline_block`; `DEFAULT_INLINE_BLOCK = 128 KiB`. +Small objects store their payload inline after the container CRC ([filemeta_inline.rs](../../crates/filemeta/src/filemeta_inline.rs)): 1 version byte (`INLINE_DATA_VER = 1`) then a msgpack map of `version-key → bin`. **INVARIANT — the map key** is the version-id string, `"null"` (`NULL_VERSION_ID`, [fileinfo.rs](../../crates/filemeta/src/fileinfo.rs)) for the null/None version, else the lowercase hyphenated UUID. Presence is determined **on read** solely by the `meta_sys[inline-data]` body marker (`FileInfo::inline_data`); the read path gates inline extraction on that marker alone. The header `InlineData` flag is **written** (mirrored from the body on marshal) but is **not** consulted on read, and a disagreement is tolerated — MinIO may leave the header flag unset while inline data is present, so a reader must **not** require the flag and the marker to agree. The inline threshold is `should_inline` ([storageclass.rs](../../crates/ecstore/src/config/storageclass.rs)): inline if `shard_size ≤ inline_block/8` for versioned buckets, else `≤ inline_block`; `DEFAULT_INLINE_BLOCK = 128 KiB`. --- @@ -230,7 +232,7 @@ Small objects store their payload inline after the container CRC ([filemeta_inli - Encode-time gates: writable disks `< write_quorum` ⇒ `ErasureWriteQuorum`; committed shards `< write_quorum` after encode ⇒ error ([set_disk/ops/object.rs](../../crates/ecstore/src/set_disk/ops/object.rs)). - **INVARIANT — atomic commit with best-effort rollback.** Commit is `rename_data` (per-disk temp → final) fanned across all disks ([core/io_primitives.rs](../../crates/ecstore/src/set_disk/core/io_primitives.rs)). If write quorum is not met (`reduce_write_quorum_errs`), every successful disk is undone (`delete_version{undo_write:true}`) and the original quorum error is returned. **Baseline:** the rollback is **best-effort** — undo failures are counted and `warn!`-logged, never propagated or retried — so a write that both misses quorum *and* whose rollback partially fails can leave shards on some disks; that partial residue is reconciled later by heal/scanner, not by the commit path. The guarantee the commit path enforces is "never *reports* success below quorum", not "never leaves any bytes behind". - On success the newly committed dir is `fi.data_dir`. Separately, `reduce_common_data_dir` votes over each disk's **`old_data_dir`** (the *superseded* dir being dereferenced) and returns it when it reaches write_quorum, so the old dir can be reclaimed (`commit_rename_data_dir`) — it is a GC input, **not** the new `data_dir`. `classify_rename_convergence` classifies the commit (`PartialCommit` / `SignatureDivergent`), but **only the multipart-complete path consumes it** (`convergence.needs_heal()` → `send_heal_request`); the regular `put_object` path discards the convergence result and relies on the old-data-dir cleanup / `add_partial` heal enqueue instead. -- The write layout (per-pool parity, storage class, `max_parity`) is computed **inline** in the write path ([set_disk/ops/object.rs](../../crates/ecstore/src/set_disk/ops/object.rs)); on `main` there is **no** `WriteLayout` type or `resolve_write_layout` function — do not cite either as if it exists (§13's symbol-citation rule). A future refactor may centralize this; add the symbol to the spec only once it lands in code. +- The write layout (per-pool parity, storage class, `max_parity`) is resolved once by `resolve_write_layout` into a `WriteLayout` ([set_disk/mod.rs](../../crates/ecstore/src/set_disk/mod.rs)) and consumed by the object and multipart write paths ([set_disk/ops/object.rs](../../crates/ecstore/src/set_disk/ops/object.rs), [set_disk/ops/multipart.rs](../../crates/ecstore/src/set_disk/ops/multipart.rs)); resolution is per pool (§2.2). --- @@ -238,7 +240,7 @@ Small objects store their payload inline after the container CRC ([filemeta_inli - **INVARIANT — read quorum = `data_blocks`.** `object_quorum_from_meta` returns `(read_quorum = data_blocks, write_quorum)` ([set_disk/metadata.rs](../../crates/ecstore/src/set_disk/metadata.rs)); `parity_blocks = common_parity(...)` is the parity value held by the most disks that still reaches its own read quorum. When `default_parity_count == 0`, read = write = all shards. - Authoritative FileInfo selection — `find_file_info_in_quorum` ([set_disk/metadata.rs](../../crates/ecstore/src/set_disk/metadata.rs)) groups valid metas by a content-identity SHA-256 (`file_info_quorum_hash`) that hashes size/flags/mod_time/transition/version_id/data_dir/parts and, for real objects, data/parity/distribution — **excluding replication-status keys** so replication noise never splits quorum. A meta counts only if its mod_time equals the common mod_time (or etag matches when mod_time is absent). The winning hash must reach quorum, else `ErasureReadQuorum`. Latest-version reads may escalate to write_quorum to avoid resurrecting a partially-overwritten version. -- **INVARIANT — decode needs ≥ `data_blocks` shards.** The stripe reader requires `available_shards ≥ data_shards`; below that the read fails closed with a read-quorum error (never silent truncation) ([set_disk/read.rs](../../crates/ecstore/src/set_disk/read.rs), [set_disk/shard_source.rs](../../crates/ecstore/src/set_disk/shard_source.rs)). Before any `block_size` / `data_shards` division, `has_valid_dimensions()` must hold (`block_size > 0 && data_shards > 0`) or the read fails instead of dividing by zero ([erasure.rs](../../crates/ecstore/src/erasure/coding/erasure.rs)); note this guard runs *after* codec construction and fully covers only `block_size == 0` — a `data_blocks == 0` geometry panics earlier in the constructor (§13). +- **INVARIANT — decode needs ≥ `data_blocks` shards.** The stripe reader requires `available_shards ≥ data_shards`; below that the read fails closed with a read-quorum error (never silent truncation) ([set_disk/read.rs](../../crates/ecstore/src/set_disk/read.rs), [set_disk/shard_source.rs](../../crates/ecstore/src/set_disk/shard_source.rs)). Codec geometry is validated at construction: `Erasure::try_new` / `try_new_with_options` ([erasure.rs](../../crates/ecstore/src/erasure/coding/erasure.rs)) return `ErasureConstructionError` for `data_shards == 0`, `block_size == 0`, shard-count overflow, or an unsupported shard configuration, so a read never reaches a `block_size` / `data_shards` division with invalid geometry (§13). - If `available ≥ data_blocks` but some shards are missing, the read is served **and** a background read-repair heal is enqueued. - **INVARIANT — cross-stripe read verification.** When a data shard is missing and `available > data_blocks`, reconstruction regenerates parity and compares it to the surviving parity; a mismatch is `InvalidData "inconsistent read source shards"` (backlog#832), catching corruption that passed per-shard bitrot but disagrees across the stripe ([erasure.rs](../../crates/ecstore/src/erasure/coding/erasure.rs)). @@ -324,22 +326,12 @@ Decode tolerance ## 13. Change procedure and guardrails -- **Risk tier.** All of the above is High-risk ([AGENTS.md](../../AGENTS.md)). Any behavior-affecting change requires the full seven-role adversarial validation. +- **Risk tier.** All of the above is High-risk ([AGENTS.md](../../AGENTS.md) "Broad or High-Risk Changes"). Any behavior-affecting change requires adversarial review with the `adversarial-validation` skill. - **Keep this document in sync.** A change to any governed behavior, formula, format field, or invariant must update this spec in the same PR; renaming a cited symbol must update its reference here. The spec is normative and is the checklist the next change is reviewed against, so drift is a correctness defect. References are symbol-based (not line numbers) specifically so ordinary refactors do not invalidate them — but semantic changes still must. - **Adding an on-disk field** must be additive: new msgpack key or a `minor`/`meta_ver` bump with a read path for the old value; keep decoders skipping unknown keys; write both internal-key prefixes; never repurpose or reorder existing keys or header array positions. - **Never make a decode boundary stricter** than what §11 allows without (a) proving no legitimate older-RustFS or MinIO-migrated shape is rejected, and (b) a regression test against real on-disk and MinIO fixtures. New validation belongs at the trust boundary and must fail *open to a tolerant default*, not closed to `FileCorrupt`, for anything recoverable. (Concretely: rejecting a negative `actual_size`, or hard-failing a non-16-byte `transitioned-versionID`, breaks existing data — see §11.) -- **Codec construction (baseline gap).** The baseline exposes panicking `Erasure::new` / `new_with_options`: the codec's shard-count validation surfaces as an `.expect` panic when `data_shards == 0 && parity_shards > 0` (`ReedSolomon::new` ⇒ `TooFewDataShards`). `has_valid_dimensions()` (`block_size > 0 && data_shards > 0`) is a **`&self`** method, so it can only run *after* construction — the read path builds the codec from on-disk geometry first and checks the guard second ([erasure.rs](../../crates/ecstore/src/erasure/coding/erasure.rs), [set_disk/read.rs](../../crates/ecstore/src/set_disk/read.rs)). It therefore reliably catches only the `block_size == 0` case (block size is never passed to `ReedSolomon::new`, so construction succeeds and the guard rejects it before any division); a `data_blocks == 0` xl.meta with `parity > 0` **panics in the constructor before the guard can run**. The correct fix is a **fallible constructor** (returning `Result`, not `.expect`) on any path reachable from untrusted metadata; until then `has_valid_dimensions()` is a partial preflight, not a complete guard. +- **Codec construction is fallible.** Read, heal, multipart, and object write paths construct the codec through `Erasure::try_new` / `try_new_with_options` ([erasure.rs](../../crates/ecstore/src/erasure/coding/erasure.rs)) and surface `ErasureConstructionError` instead of panicking on geometry decoded from untrusted metadata (`data_shards == 0`, `block_size == 0`, shard-count overflow, unsupported shard counts). Do not reintroduce a panicking constructor on any path reachable from on-disk metadata; `has_valid_dimensions()` remains only a `&self` preflight for already-built codecs. - **Guardrail scripts** (part of `make pre-commit` / `make pre-pr`): - [check_architecture_migration_rules.sh](../../scripts/check_architecture_migration_rules.sh) keeps the erasure engine crate-private and under its owner module, and keeps erasure-cache / `GLOBAL_IS_ERASURE*` access behind ecstore helpers. - [check_doc_paths.sh](../../scripts/check_doc_paths.sh) validates that every repo path this document cites exists — keep citations to real paths. - **Tooling.** Inspect on-disk metadata with `dump_fileinfo` / `dump_versions` per [../operations/tier-ilm-debugging.md](../operations/tier-ilm-debugging.md) rather than guessing at bytes. - ---- - -## 14. References - -- Reed–Solomon codes; MDS property and GF(2⁸) byte-oriented coding — the standard basis for `rs-vandermonde` (Vandermonde generator matrix over GF(2⁸)). -- `rustfs-erasure-codec` (RustFS fork of `reed-solomon-erasure`, GF(2⁸)) and `reed-solomon-simd` (GF(2¹⁶)) — declared in the workspace `Cargo.toml`. -- HighwayHash-256 — the bitrot checksum family; π-derived default key. -- MinIO `xl.meta` v1.3 format lineage — RustFS is byte-compatible for read + one-way migration; see [minio-file-format-compat.md](minio-file-format-compat.md) for the fixture-proven matrix and scope. -- Related invariants: [placement-repair-invariants.md](placement-repair-invariants.md), [ecstore-layout-boundary.md](ecstore-layout-boundary.md), [decommission-compatibility.md](decommission-compatibility.md), [../operations/tier-ilm-debugging.md](../operations/tier-ilm-debugging.md), and [AGENTS.md](../../AGENTS.md) Cross-Cutting Domain Invariants. diff --git a/docs/architecture/global-state-crate-split-plan.md b/docs/architecture/global-state-crate-split-plan.md index b60ef7b63..59bca1ae0 100644 --- a/docs/architecture/global-state-crate-split-plan.md +++ b/docs/architecture/global-state-crate-split-plan.md @@ -1,202 +1,67 @@ # Global State And Crate Split Plan -This document records the late global-state cleanup plan after the AppContext -foundation, storage API contracts, ECStore layout, runtime lifecycle, and cluster -control-plane boundaries are stable. +**Use this when:** business logic needs runtime state (object store, endpoints, lock clients, lifecycle state, config) and you must pick the right boundary, or you are evaluating a new crate split out of ECStore. +**Source of truth:** `crates/ecstore/src/runtime/global.rs` and `crates/ecstore/src/runtime/sources.rs` (ECStore-owned state and its adapter), `rustfs/src/app/context.rs` and the `runtime_sources.rs` owner modules under `rustfs/src` (RustFS resolvers), and the `rustfs_ecstore::api::global` boundary list in `scripts/check_architecture_migration_rules.sh`. The static inventory is [global-state-inventory.md](global-state-inventory.md). -As of the Phase 7 closeout, runtime resolver fallbacks have been pushed out of -the root facade and into explicit owner-local boundaries. Future work should -therefore treat broad fallback removal as complete and use this document for the -remaining ECStore-owned bootstrap state and crate-split decisions. - -The issue #730 global-state baseline and runtime migration target inventory are -recorded in [`global-state-inventory.md`](global-state-inventory.md). +Broad resolver-fallback removal is complete: runtime resolver fallbacks live in explicit owner-local boundaries, not in the root facade. What remains is ECStore-owned bootstrap state and crate-split decisions. ## Remaining Global Owners -| Owner | Current role | Migration stance | +| Owner | Role | Stance | |---|---|---| | `rustfs/src/app/context.rs` | AppContext-first resolver facade. | Resolver helpers stay context-first and do not construct concrete no-AppContext defaults. | -| `rustfs/src/app/context/runtime_sources.rs` | Default adapters for KMS, IAM, object store, endpoints, config, metrics, and notification state used by AppContext construction. | This is an allowed adapter boundary, not a business logic owner. | -| `rustfs/src/*/runtime_sources.rs` | Root, admin, app, server, startup, and storage owner-local runtime-source boundaries. | Business modules use these boundaries instead of calling global state directly; owner facades own any remaining no-AppContext compatibility defaults. | -| `rustfs/src/*/storage_api.rs` | Root, admin, app, and storage owner-local storage contract/facade boundaries. | Storage helper and ECStore facade access remains visible at local owner boundaries. | -| `crates/*/storage_api.rs` | External crate-local storage facade boundaries for IAM, scanner, heal, notify, observability, Swift, and S3 Select. | External runtime crates consume ECStore runtime state through `rustfs_ecstore::api::runtime` instead of the direct global facade. | -| `crates/ecstore/src/runtime/global.rs` | ECStore bootstrap/runtime state owner. | Keep internal until ECStore has explicit owner handles for all remaining bootstrap state. | -| `crates/ecstore/src/runtime/sources.rs` | ECStore runtime-source adapter over global state. | Preferred ECStore-internal access path while shrinking direct `runtime::global` reads. | +| `rustfs/src/app/context/runtime_sources.rs` | Default adapters for KMS, IAM, object store, endpoints, config, metrics, and notification state used by AppContext construction. | Allowed adapter boundary, not a business-logic owner. | +| `rustfs/src/runtime_sources.rs`, `rustfs/src/admin/runtime_sources.rs`, `rustfs/src/app/runtime_sources.rs`, `rustfs/src/server/runtime_sources.rs`, `rustfs/src/storage/runtime_sources.rs` | Owner-local runtime-source boundaries. | Business modules use these instead of global state; owner facades decide when to apply no-AppContext compatibility defaults. | +| `rustfs/src/storage_api.rs`, `rustfs/src/admin/storage_api.rs`, `rustfs/src/app/storage_api.rs`, `rustfs/src/storage/storage_api.rs` | Owner-local storage contract/facade boundaries. | Storage helper and ECStore facade access stays visible at local owner boundaries. | +| `crates/*/storage_api.rs` | External crate-local storage facade boundaries (IAM, scanner, heal, notify, observability, Swift, S3 Select). | External runtime crates read ECStore runtime state through `rustfs_ecstore::api::runtime`, never the global facade. | +| `crates/ecstore/src/runtime/global.rs` | ECStore bootstrap/runtime state owner. | Internal until ECStore has explicit owner handles for all remaining bootstrap state. | +| `crates/ecstore/src/runtime/sources.rs` | ECStore runtime-source adapter over global state. | Preferred ECStore-internal access path while direct `runtime::global` reads shrink. | ## Runtime Source Boundaries -Runtime-source modules are the allowed compatibility layer between migrated -consumers and process-global state. They must keep these properties: +Runtime-source modules are the allowed compatibility layer between migrated consumers and process-global state. They keep these properties: - context-first lookup when an `AppContext` handle exists; -- explicit fallback to the existing global only where compatibility still - requires it; +- explicit fallback to the existing global only where compatibility still requires it, decided by the owner facade; - no hidden service construction in business logic; -- no startup, readiness, IAM, KMS, lock, notification, or storage behavior - change in inventory or guardrail PRs. +- the root `rustfs/src/runtime_sources.rs` is an entrypoint only: it composes no concrete fallback defaults (`unwrap_or`, `unwrap_or_else`, direct `init_global` or `new_global` calls); +- production callers outside runtime-source and `storage_api.rs` boundary modules do not import ECStore global state directly. -## Guarded Boundary List +### Guarded Boundary List -The architecture guard snapshots the files currently allowed to reference -`rustfs_ecstore::api::global` directly: +The guard pins the production files allowed to reference `rustfs_ecstore::api::global` directly: - `rustfs/src/storage/storage_api.rs` -That boundary now keeps only bootstrap writes and lifecycle controls in the -global facade. Read-only runtime getters must be exported through -`rustfs_ecstore::api::runtime` and consumed through the local storage facade. -New direct uses must either move behind an existing owner-local boundary or -update this plan and the guard in the same reviewed migration PR. +That boundary keeps only bootstrap writes and lifecycle controls (`set_global_endpoints`, `set_global_region`, `set_global_rustfs_port`, `set_object_store_resolver`, `shutdown_background_services`, `update_erasure_type`). Read-only runtime getters are exported through `rustfs_ecstore::api::runtime` and consumed through the local storage facade. A new direct use either moves behind an existing owner-local boundary or updates this plan and the guard in the same reviewed change. ## Fallback Removal Plan -1. Keep AppContext-first lookup as the stable resolver contract. -2. Keep concrete no-AppContext compatibility defaults only at owner-local - runtime-source facades that consume them. -3. Do not let business logic call `AppContext` or ECStore globals directly when - an owner-local runtime-source boundary exists. -4. Keep embedded startup and tests working before deleting any remaining owner - fallback. -5. Do not remove ECStore bootstrap globals until ownership handles exist for - local disks, endpoint pools, lock clients, notification state, tier config, - lifecycle state, and object-store publication. - -## GLOB-007 Closeout Boundary - -`GLOB-007` is complete when these invariants hold: - -- root `rustfs/src/runtime_sources.rs` is an AppContext/root facade entrypoint - and no longer composes concrete fallback defaults with `unwrap_or`, - `unwrap_or_else`, direct `init_global`, or direct `new_global` calls; -- private AppContext resolver helpers are context-first and do not hide fallback - closure parameters; -- admin, app, storage, server, startup, and config owner facades decide when to - apply no-AppContext compatibility defaults; -- production callers outside runtime-source and storage-api boundary modules do - not import ECStore global state directly; -- the architecture guard keeps the direct `rustfs_ecstore::api::global` - boundary list explicit. - -Allowed remaining fallbacks are owner compatibility decisions, not resolver -fallback families. They are kept so embedded startup, tests, and no-context -callers preserve the previous behavior while higher layers continue migrating -to explicit AppContext ownership. +1. AppContext-first lookup is the stable resolver contract. +2. Concrete no-AppContext compatibility defaults exist only at the owner-local runtime-source facades that consume them. +3. Business logic does not call `AppContext` or ECStore globals directly when an owner-local runtime-source boundary exists. +4. Embedded startup and tests keep working before any remaining owner fallback is deleted. +5. ECStore bootstrap globals stay until ownership handles exist for local disks, endpoint pools, lock clients, notification state, tier config, lifecycle state, and object-store publication. ## Crate Split Evaluation -`ecstore-erasure` and `storage-cluster` remain proposal-only until dependency -cycles and hot-path risks are proven safe. The Phase 7 evaluation is complete -for now: neither split is ready for code movement in this migration round. -The follow-up ECStore module split plan is recorded in -[`ecstore-module-split-plan.md`](ecstore-module-split-plan.md), including the -remaining `SetDisks`, lifecycle, replication, and facade-shrink boundaries. +`ecstore-erasure` and `storage-cluster` are proposal-only; neither is ready for code movement. Lifecycle and replication split status is tracked in [ecstore-module-split-plan.md](ecstore-module-split-plan.md). -### CRATE-001: `ecstore-erasure` +### `ecstore-erasure` -Current coupling: +Coupling: erasure decoding depends on disk errors, disk read timeouts, and set-disk shard sources; set-disk read/write/heal paths construct codecs in hot object I/O paths; bitrot readers/writers live in ECStore IO support and serve both erasure and set-disk code; `rustfs_ecstore::api::erasure` is still a public compatibility surface. -- erasure decoding depends on disk errors, disk read timeouts, and set-disk - shard sources; -- set-disk read/write/heal paths construct erasure codecs in hot object I/O - paths; -- bitrot readers/writers live in ECStore IO support and are used by both - erasure and set-disk code; -- public compatibility still exposes erasure symbols through - `rustfs_ecstore::api::erasure`. +Decision: do not split. The boundary becomes a candidate only after shard-source, disk-error, bitrot, and metrics contracts are explicit enough to avoid a dependency cycle back into ECStore, backed by encode/decode/reconstruction benchmarks and a rollback plan that keeps read/write quorum and old-version decode unchanged. -Decision: do not split in code yet. The erasure boundary is a candidate only -after the shard-source, disk-error, bitrot, and metrics contracts are explicit -enough to avoid a dependency cycle back into ECStore. +### `storage-cluster` -Required evidence before proposing the split: +Coupling: cluster RPC remote-disk code depends on disk stores, disk health tracking, set-disk buffer sizing, local disk scan guards, internode metrics, and runtime credential/signature sources; peer S3 and peer REST clients share bucket metadata, disk quorum reduction, endpoint layout, local disk initialization, and store helpers; control-plane snapshots are separate from data-plane RPC, but remote disk and peer clients still own data-movement side effects inside ECStore. -- `cargo tree -p rustfs-ecstore -e normal --depth 2` snapshot for dependency - impact; -- focused benchmarks for encode/decode, read reconstruction, bitrot verification, - and large-object streaming; -- contract sketch for shard sources, disk errors, bitrot IO, metrics, and file - metadata without importing ECStore implementation modules; -- compatibility plan for `rustfs_ecstore::api::erasure` and test harnesses; -- rollback plan that keeps object read/write quorum and old-version file decode - behavior unchanged. - -### CRATE-002: `storage-cluster` - -Current coupling: - -- cluster RPC remote disk code depends on disk stores, disk health tracking, - set-disk buffer sizing, local disk scan guards, internode metrics, and runtime - credential/signature sources; -- peer S3 and peer REST clients share bucket metadata, disk quorum reduction, - endpoint layout, local disk initialization, and store helpers; -- control-plane snapshots are separated from data-plane RPC, but remote disk and - peer clients still own data movement side effects inside ECStore. - -Decision: do not split in code yet. The storage-cluster boundary is a candidate -only after remote disk, peer health, lock/quorum, runtime metrics, and endpoint -layout contracts are explicit enough to stand below ECStore without circular -dependencies. - -Required evidence before proposing the split: - -- dependency graph showing no cycle with ECStore, `rustfs-storage-api`, runtime - source owners, or cluster control-plane owners; -- RPC contract sketch for remote disk, peer S3, peer REST, auth/signature, - internode metrics, and cancellation; -- compatibility plan for `rustfs_ecstore::api::cluster`, `api::rpc`, and test - fixtures that build local disks or endpoint pools; -- focused tests for remote disk error classification, peer health recovery, - per-pool quorum reduction, lock behavior, and data-stream request paths; -- rollback plan that preserves quorum, remote disk IO, lock, peer health, and - data movement behavior. - -### CRATE-003: `bucket-lifecycle` - -Decision: do not split in code yet. Lifecycle remains coupled to ECStore object -operations, bucket metadata, `SetDisks` stale multipart cleanup, tier config, -runtime lifecycle state, scanner metrics, notification/audit side effects, and -replication delete scheduling. - -Required evidence before proposing the split: - -- contract sketch for lifecycle object operations, metadata access, runtime - state, replication delete scheduling, and audit/notification sinks; -- dependency graph showing the candidate crate can avoid importing ECStore - implementation modules; -- focused tests for lifecycle evaluation, expiry, transition, stale multipart - cleanup, tier journal recovery, and lifecycle-originated replication deletes; -- compatibility plan for `rustfs_ecstore::api::bucket::lifecycle` consumers; -- rollback plan that preserves lifecycle queues, scanner repair accounting, - tier transitions, object deletion behavior, and notification/audit events. - -### CRATE-004: `bucket-replication` - -Decision: do not split in code yet. Replication remains coupled to ECStore -object APIs, bucket target clients, metadata systems, file metadata replication -state, ECStore-owned runtime replication pool/stat handles, bucket monitor -state, scanner repair classification, lifecycle-originated deletes, and -notification events. RustFS-facing runtime consumers should use storage-owner -wrapper handles while that state remains in ECStore. - -Required evidence before proposing the split: - -- contract sketch for replication storage operations, metadata/target access, - runtime pool and stats, event sinks, and lifecycle/heal bridges; -- dependency graph showing the candidate crate can avoid importing ECStore - implementation modules; -- focused tests for object replication, delete replication, resync state, heal - repair queueing, target error handling, and queue admission; -- compatibility plan for `rustfs_ecstore::api::bucket::replication` consumers; -- rollback plan that preserves replication queues, MRF/resync state, target - client behavior, scanner repair, and event emission. +Decision: do not split. The boundary becomes a candidate only after remote disk, peer health, lock/quorum, runtime metrics, and endpoint layout contracts can stand below ECStore without cycles, with compatibility plans for `rustfs_ecstore::api::cluster` and `api::rpc` and focused tests for remote disk error classification, peer health recovery, per-pool quorum reduction, lock behavior, and data-stream request paths. ## Preservation Rules -- Do not reintroduce AppContext resolver fallback families in broad cleanup PRs. -- Do not introduce direct global reads in admin, app, server, storage, scanner, - heal, IAM, notify, observability, Swift, or S3 Select business logic. -- Do not split crates in the same PR that moves runtime state. -- Do not change startup order, readiness, KMS fatal boundaries, IAM recovery, - lock quorum, object placement, reader behavior, or notification/audit - lifecycle while shrinking global state. +- Do not reintroduce AppContext resolver fallback families in broad cleanups. +- Do not introduce direct global reads in admin, app, server, storage, scanner, heal, IAM, notify, observability, Swift, or S3 Select business logic. +- Do not split crates in the same change that moves runtime state. +- Do not change startup order, readiness, KMS fatal boundaries, IAM recovery, lock quorum, object placement, reader behavior, or notification/audit lifecycle while shrinking global state. diff --git a/docs/architecture/global-state-inventory.md b/docs/architecture/global-state-inventory.md index 66726639d..345d7444b 100644 --- a/docs/architecture/global-state-inventory.md +++ b/docs/architecture/global-state-inventory.md @@ -1,146 +1,64 @@ # Global State Inventory -This inventory records the issue #730 baseline for global runtime state after -the AppContext foundation and owner-local runtime-source boundaries were added. -It is intentionally documentation-only: it classifies migration targets without -changing startup, readiness, object IO, lifecycle, replication, or notification -behavior. - -## Counting Baseline - -The audit uses the current workspace Rust sources and keeps broad static -caches separate from runtime migration targets. - -| Scope | Count | Command | -|---|---:|---| -| Rust source files | 1,252 | `rg --files -g '*.rs'` | -| `OnceLock` references | 221 lines | `rg -n --glob '*.rs' 'OnceLock'` | -| `GLOBAL_*` references | 273 lines | `rg -n --glob '*.rs' '\bGLOBAL_[A-Za-z0-9_]*\b'` | -| `static NAME:` definitions | 621 lines | `rg -n --glob '*.rs' '^\s*(pub(\([^)]*\))?\s+)?static(\s+mut)?\s+[A-Za-z_][A-Za-z0-9_]*\s*:'` | -| `lazy_static!` `static ref` definitions | 58 lines | `rg -n --glob '*.rs' '^\s*(pub\s+)?static\s+ref\s+[A-Za-z_][A-Za-z0-9_]*\s*:'` | -| `static mut` definitions | 0 lines | `rg -n --glob '*.rs' '^\s*(pub(\([^)]*\))?\s+)?static\s+mut\s+'` | +**Use this when:** you meet a `GLOBAL_*` static or an `OnceLock` and need to know whether it is a runtime ownership handle (reach it through a boundary), an owner-local static (leave it inside its module), or process-global by design. +**Source of truth:** `crates/ecstore/src/api/mod.rs` (the `pub mod runtime` and `pub mod global` re-export lists), `crates/ecstore/src/runtime/global.rs`, `crates/ecstore/src/runtime/sources.rs`, and the statics themselves. Boundary rules are in [global-state-crate-split-plan.md](global-state-crate-split-plan.md). ## Global State Classification | Category | Rule | Representative owners | |---|---|---| -| Process-global | Process identity, metrics registries, lock manager, audit guard, TLS material, or other state that is intentionally one per process. | `crates/credentials`, `crates/common`, `crates/io-metrics`, `crates/lock`, `crates/obs`, `crates/tls-runtime` | -| Runtime migration target | Mutable runtime state that describes the active object store, endpoints, local disks, lifecycle, replication, notification, config, or background controllers. | `crates/ecstore/src/runtime/global.rs`, `crates/ecstore/src/runtime/sources.rs`, `rustfs/src/app/context/*` | -| Owner-local compatibility | Existing compatibility adapters that are allowed to read globals while callers migrate to AppContext-first or owner-local runtime-source APIs. | `rustfs/src/*/runtime_sources.rs`, `rustfs/src/*/storage_api.rs`, `crates/*/storage_api.rs` | -| Test or fixture state | Static setup used by tests to amortize expensive ECStore setup or isolate compatibility harness state. | `rustfs/src/app/*_test.rs`, `crates/scanner/tests/*`, `crates/ecstore/src/**/tests` | -| Cache or constant | Regexes, metrics descriptors, defaults, KVS registrations, headers, path constants, and small process caches that are not runtime ownership handles. | `crates/config`, `crates/obs/src/metrics`, `crates/utils`, `rustfs/src/server/readiness.rs` | -| Legacy naming or review-needed | Old MinIO-port naming, stale comments, or names that need owner confirmation before code movement. | `GLOBAL_OBJECT_API` | +| Process-global | Process identity, metrics registries, lock manager, audit guard, TLS material, or other state intentionally one per process. | `GLOBAL_LOCK_MANAGER` (`crates/lock`), `GLOBAL_CONN_MAP` (`crates/common`), `GLOBAL_RUSTFS_RPC_SECRET` (`crates/credentials`), `AUDIT_SYSTEM` (`crates/audit`), `crates/io-metrics`, `crates/obs`, `crates/tls-runtime` | +| Runtime migration target | Mutable runtime state describing the active object store, endpoints, local disks, lifecycle, replication, notification, config, or background controllers. | `crates/ecstore/src/runtime/global.rs`, `crates/ecstore/src/runtime/sources.rs`, `rustfs/src/app/context/` | +| Owner-local compatibility | Adapters allowed to read globals while callers migrate to AppContext-first or owner-local runtime-source APIs. | `rustfs/src/*/runtime_sources.rs`, `rustfs/src/*/storage_api.rs`, `crates/*/storage_api.rs` | +| Owner-local static | A static private to one module and reached only through that module's functions: caches, single-run guards, admission locks, module toggles. | The RustFS inventory below | +| Test or fixture state | Static setup that amortizes expensive ECStore setup or isolates harness state. | `rustfs/src/app/*_test.rs`, `crates/scanner/tests/`, `crates/test-utils/src/ecstore_test_compat.rs` | +| Cache or constant | Regexes, metrics descriptors, defaults, KVS registrations, headers, path constants. | `crates/config`, `crates/obs/src/metrics`, `crates/utils` | ## Runtime Migration Inventory -These are the issue #730 targets that should remain visible until an owner -migration PR removes or replaces each item. +Runtime ownership handles that exist today. Reads go through `rustfs_ecstore::api::runtime`, bootstrap writes go through `rustfs_ecstore::api::global`, and RustFS code reaches both only from `rustfs/src/storage/storage_api.rs` and the AppContext resolvers. -| State | Current boundary | Category | Migration stance | -|---|---|---|---| -| `APP_CONTEXT_SINGLETON` | `rustfs/src/app/context/global.rs` | Owner-local compatibility | Keep as the context-first facade while no-context startup and embedded callers still exist. | -| `GLOBAL_OBJECT_API`, `GLOBAL_OBJECT_STORE_RESOLVER` | `crates/ecstore/src/runtime/global.rs`, `rustfs/src/app/context/global.rs`, and storage compatibility APIs | Runtime migration target | Do not migrate first; it is tied to storage startup, IAM-after-storage AppContext publication, and data-plane resolver compatibility. The object-store resolver is now published from the AppContext owner path, no longer re-exported from the RustFS storage root, and RustFS AppContext tests no longer use the old `new_object_layer_fn` fallback chain. RustFS storage root no longer re-exports ECStore runtime/global facade symbols; callers must use storage/app/admin facades. | -| `GLOBAL_ENDPOINTS`, `GLOBAL_IS_ERASURE`, `GLOBAL_IS_DIST_ERASURE`, `GLOBAL_IS_ERASURE_SD`, `GLOBAL_ROOT_DISK_THRESHOLD` | `crates/ecstore/src/runtime/global.rs` and `crates/ecstore/src/runtime/sources.rs` | Runtime migration target | Endpoint and setup-type reads now flow through ECStore `api::runtime` helpers at the RustFS storage facade boundary; root-disk-threshold access stays behind ECStore runtime helpers. Move endpoint ownership only after readiness and quorum behavior have explicit coverage. | -| `GLOBAL_LOCAL_DISK_MAP`, `GLOBAL_LOCAL_DISK_ID_MAP`, `GLOBAL_LOCAL_DISK_SET_DRIVES` | `crates/ecstore/src/runtime/global.rs` and `crates/ecstore/src/runtime/sources.rs` | Runtime migration target | Local disk map, disk-id cache, and set-drive access now stay behind ECStore runtime-source helpers instead of direct global access; preserve disk lookup, remote/local classification, and test reset hooks in later ownership changes. | -| `GLOBAL_EXPIRY_STATE`, `GLOBAL_TRANSITION_STATE`, `GLOBAL_LIFECYCLE_SYS` | `crates/ecstore/src/bucket/lifecycle/*`, `crates/ecstore/src/runtime/global.rs`, and `crates/ecstore/src/runtime/sources.rs` | Runtime migration target | Lifecycle state globals now stay behind ECStore lifecycle owner helpers and ECStore runtime-source helpers; RustFS AppContext has expiry/transition state interfaces and resolver coverage, and daily tier stats derive from the transition-state handle instead of a separate context boundary; scanner expiry-state access still uses the ECStore runtime `expiry_state_handle` boundary until scanner gets an injected provider. | -| `GLOBAL_REPLICATION_POOL`, `GLOBAL_REPLICATION_STATS`, `GLOBAL_BUCKET_MONITOR` | `crates/ecstore/src/bucket/replication/*`, `crates/ecstore/src/runtime/global.rs` | Runtime migration target | Replication pool/stat access now stays behind replication owner and ECStore runtime-source helpers; bucket-monitor reads now flow through ECStore `api::runtime` at the RustFS storage facade boundary while AppContext/runtime-source resolvers remain the caller boundary. | -| `GLOBAL_TIER_CONFIG_MGR`, `GLOBAL_STORAGE_CLASS`, `GLOBAL_CONFIG_SYS`, `GLOBAL_SERVER_CONFIG` | `crates/ecstore/src/config`, `crates/config`, `rustfs/src/app/context/runtime_sources.rs` | Runtime migration target | Tier config manager reads and reloads now use the ECStore runtime-source helper; move remaining config state through config/runtime-source owners only, without combining storage-class behavior or persistence changes. | -| `GLOBAL_EVENT_NOTIFIER`, `GLOBAL_NOTIFICATION_SYS` | `crates/ecstore/src/runtime/global.rs`, `crates/ecstore/src/runtime/sources.rs`, and `crates/ecstore/src/services/*` | Runtime migration target | `GLOBAL_EVENT_NOTIFIER` and `GLOBAL_NOTIFICATION_SYS` access now stay behind ECStore runtime-source and notification owner helpers; move remaining notification ownership only through notify/runtime-source boundaries. | -| `EVENT_DISPATCH_HOOK` | `crates/ecstore/src/services/event_notification.rs`, RustFS server event bridge, and storage compatibility APIs | Runtime migration target / owner helper | Direct hook storage stays inside the ECStore event-notification owner; RustFS registers the bridge through the storage compatibility facade until event dispatch ownership moves behind an injected notification sink. | -| `GLOBAL_BUCKET_METADATA_SYS` | `crates/ecstore/src/bucket/metadata_sys.rs`, `crates/ecstore/src/runtime/sources.rs`, and RustFS storage compatibility APIs | Runtime migration target | Bucket metadata system direct access now stays inside the ECStore metadata owner; callers use metadata owner helpers or storage/runtime-source compatibility functions until metadata ownership moves behind an injected runtime context. | -| `GLOBAL_BOOT_TIME`, `GLOBAL_BACKGROUND_SERVICES_CANCEL_TOKEN`, `GLOBAL_DEPLOYMENT_ID`, `GLOBAL_REGION`, `GLOBAL_RUSTFS_PORT`, `GLOBAL_LOCAL_NODE_NAME_FALLBACK`, `GLOBAL_LOCAL_NODE_NAME_HEX_FALLBACK` | `crates/ecstore/src/runtime/global.rs`, `crates/ecstore/src/runtime/sources.rs` | Runtime migration target | Boot time, background service cancellation token reads, ECStore local-node-name fallback reads, and deployment ID/region/port reads now stay behind the ECStore runtime-source API; scalar writes remain behind bootstrap owner helpers until ownership handles replace them. | -| `WORKLOAD_ADMISSION_SNAPSHOT_PROVIDER` | `crates/ecstore/src/runtime/sources.rs`, RustFS startup background setup, and storage compatibility APIs | Runtime migration target / owner helper | Startup publishes the workload provider through the storage compatibility facade, and ECStore data movement reads it only through the runtime-source helper until workload admission ownership moves into an explicit runtime context. | -| `GLOBAL_LOCAL_LOCK_CLIENT`, `GLOBAL_LOCK_CLIENTS`, `GLOBAL_LOCK_MANAGER` | `crates/ecstore/src/runtime/global.rs`, `crates/lock` | Runtime migration target / process-global split | ECStore lock client reads now flow through ECStore `api::runtime` helpers at the RustFS storage facade boundary; preserve lock quorum and lock client selection while keeping the process-level lock manager separate from endpoint-specific clients. | -| `GLOBAL_CONN_MAP`, `GLOBAL_LOCAL_NODE_NAME`, `GLOBAL_RUSTFS_HOST`, `GLOBAL_RUSTFS_ADDR`, `GLOBAL_ROOT_CERT`, `GLOBAL_MTLS_IDENTITY`, `GLOBAL_OUTBOUND_TLS_GENERATION` | `crates/common`, `crates/tls-runtime`, `crates/ecstore/src/runtime/sources.rs` | Runtime migration target / process-global split | Internode connection cache, common local node name, RustFS host/address reads, and outbound TLS material reads are now owned behind `rustfs_common` helpers; migrate the remaining transport and TLS state only after internode transport and outbound TLS ownership are explicit, without changing cached channel reuse or TLS reload semantics. | -| `GLOBAL_RUSTFS_RPC_SECRET` | `crates/credentials`, `crates/ecstore/src/runtime/sources.rs` | Runtime migration target / process-global split | RPC auth token writes now stay behind the `rustfs_credentials` helper boundary; migrate only if runtime secret ownership changes, preserving lazy environment and credential-derived token semantics. | -| `GLOBAL_HEAL_MANAGER`, `GLOBAL_HEAL_CHANNEL_PROCESSOR`, `GLOBAL_AHM_SERVICES_CANCEL_TOKEN` | `crates/heal/src/lib.rs` | Runtime migration target / process-global split | Direct access now stays inside the heal owner; callers use heal helper functions until heal runtime ownership moves behind explicit owner handles. | -| `AUDIT_SYSTEM` | `crates/audit/src/global.rs` | Runtime migration target / process-global split | Direct global access now stays inside the audit owner; callers use audit helper functions until audit lifecycle ownership moves behind AppContext or a runtime-source boundary. | -| `GLOBAL_PROCESSORS` | `crates/ecstore/src/services/batch_processor.rs`, `crates/ecstore/src/runtime/sources.rs` | Runtime migration target / owner helper | Direct static access now stays inside the ECStore batch processor owner; callers use `get_global_processors` or the ECStore runtime-source helper until processor ownership moves into an injected runtime context. | -| `INTERNODE_DATA_TRANSPORT` | `crates/ecstore/src/cluster/rpc/internode_data_transport.rs` | Runtime migration target / owner helper | Direct static access now stays inside the ECStore internode transport owner; callers use `build_internode_data_transport_from_env` until backend selection moves into an injected runtime context. | -| `GLOBAL_KMS_SERVICE_MANAGER` | `crates/kms/src/service_manager.rs`, RustFS KMS runtime sources | Runtime migration target / owner helper | Direct static access now stays inside the `rustfs_kms` service manager owner; RustFS callers use KMS helpers or AppContext/runtime-source handles until KMS ownership fully moves into runtime context. | -| `GLOBAL_CAPACITY_MANAGER` | `crates/object-capacity/src/capacity_manager.rs`, RustFS capacity service | Runtime migration target / owner helper | Direct static access now stays inside the object-capacity owner; callers use `get_capacity_manager` or isolated manager factories until capacity ownership moves into an injected runtime context. | -| `GLOBAL_BUCKET_TARGET_SYS` | `crates/ecstore/src/bucket/bucket_target_sys.rs`, admin/app/scanner/replication target paths | Runtime migration target / owner helper | Direct static access now stays inside the ECStore bucket target owner; callers still use `BucketTargetSys::get()` until bucket target ownership moves behind a runtime-source or replication target boundary. | -| `USAGE_MEMORY_CACHE`, `USAGE_CACHE_UPDATING` | `crates/ecstore/src/data_usage/mod.rs` | Runtime migration target / owner-local cache | Data-usage memory overlay and singleflight state stay private to the ECStore data-usage owner; callers use data-usage functions until scanner/data-usage ownership moves behind an injected runtime context. | +| Handle (`rustfs_ecstore::api::runtime`) | Backing state | Stance | +|---|---|---| +| `object_store_handle` | `GLOBAL_OBJECT_API`, `GLOBAL_OBJECT_STORE_RESOLVER` (`crates/ecstore/src/runtime/global.rs`); the resolver is published from the AppContext owner path | Do not migrate first: tied to storage startup, IAM-after-storage AppContext publication, and data-plane resolver compatibility. | +| `endpoint_pools`, `setup_is_erasure`, `setup_is_dist_erasure`, `setup_is_erasure_sd`, `first_cluster_node_is_local` | `GLOBAL_ENDPOINTS` and setup-type state (`crates/ecstore/src/runtime/global.rs`) | Move endpoint ownership only after readiness and quorum behavior have explicit coverage. | +| `local_disk_map_read` | Local disk map and set-drive state (`crates/ecstore/src/runtime/sources.rs`) | Preserve disk lookup, remote/local classification, and test reset hooks. | +| `expiry_state_handle`, `transition_state_handle` | Lifecycle expiry and transition state, `GLOBAL_LIFECYCLE_SYS` (`crates/ecstore/src/runtime/global.rs`) | Lifecycle owner helpers and the AppContext `ExpiryStateInterface` (`rustfs/src/app/context/interfaces.rs`) are the caller boundary; the scanner still reads `expiry_state_handle` until it gets an injected provider. | +| `global_tier_config_mgr` | Tier config manager | Reads and reloads stay behind this helper. | +| `bucket_monitor` | Replication bandwidth monitor | Replication pool/stat handles are projected into RustFS wrapper types at the storage boundary. | +| `global_lock_client`, `global_lock_clients` | `GLOBAL_LOCAL_LOCK_CLIENT`, `GLOBAL_LOCK_CLIENTS` (`crates/ecstore/src/runtime/global.rs`) | Preserve lock quorum and client selection; the process-level `GLOBAL_LOCK_MANAGER` stays separate. | +| `boot_time`, `deployment_id`, `region`, `rustfs_port` | `GLOBAL_BOOT_TIME`, deployment id, region, and port state (`crates/ecstore/src/runtime/global.rs`) | Scalar writes remain behind the `api::global` setters (`set_global_endpoints`, `set_global_region`, `set_global_rustfs_port`, `set_object_store_resolver`, `shutdown_background_services`, `update_erasure_type`). | -## Owner-Local Cache Inventory +Owner-helper handles outside the runtime-source list stay inside their owner and are reached through owner functions: `GLOBAL_EVENT_NOTIFIER` (`crates/ecstore/src/runtime/global.rs`); `GLOBAL_NOTIFICATION_SYS`, `EVENT_DISPATCH_HOOK`, `GLOBAL_PROCESSORS`, `INTERNODE_DATA_TRANSPORT`, `GLOBAL_BUCKET_TARGET_SYS`, `GLOBAL_CONFIG_SYS`, `GLOBAL_STORAGE_CLASS`, `WORKLOAD_ADMISSION_SNAPSHOT_PROVIDER` (ECStore owner modules); `GLOBAL_SERVER_CONFIG` (`crates/config/src/server_config.rs`); `GLOBAL_HEAL_RUNTIME`, `GLOBAL_AHM_SERVICES_CANCEL_TOKEN` (`crates/heal/src/lib.rs`); `GLOBAL_KMS_SERVICE_MANAGER` (`crates/kms/src/service_manager.rs`); `GLOBAL_CAPACITY_MANAGER` (`crates/object-capacity/src/capacity_manager.rs`); `APP_CONTEXT_SINGLETON` (`rustfs/src/app/context/global.rs`). -These owner-local caches and static guards are part of the broad issue #730 -`OnceLock` audit, but they are not runtime ownership handles. They stay private -to the defining owner module; callers must use the existing owner APIs instead -of reaching across module boundaries. +Regenerate: -| State | Owner boundary | Category | Migration stance | -|---|---|---|---| -| `READ_REPAIR_HEAL_CACHE` | `crates/ecstore/src/set_disk/read.rs` | Cache or constant / owner-local cache | Read-repair heal suppression stays local to set-disk read handling. | -| `DISK_COMPRESSION_CONFIG` | `crates/ecstore/src/io_support/compress.rs` | Cache or constant / owner-local cache | Disk compression environment parsing stays local to IO support compression helpers. | -| `CACHED_MAX_INFLIGHT_BYTES`, `CACHED_BATCH_BLOCKS`, `CACHED_BYTESMUT_INGEST` | `crates/ecstore/src/erasure/coding/encode.rs` | Cache or constant / owner-local cache | Erasure encode tuning caches stay local to the coding owner. | -| `CACHED_PUT_LARGE_BATCH_MIN_SIZE_BYTES`, `CACHED_MULTIPART_PUT_LARGE_BATCH_MIN_SIZE_BYTES`, `OBJECT_LOCK_DIAG_ENABLED` | `crates/ecstore/src/set_disk/mod.rs` | Cache or constant / owner-local cache | Set-disk batching and diagnostics caches stay local to the set-disk owner. | -| `DRIVE_TIMEOUT_PROFILE_CACHE`, `DRIVE_TIMEOUT_HEALTH_POLICY_CACHE` | `crates/ecstore/src/disk/disk_store.rs` | Cache or constant / owner-local cache | Drive timeout environment caches stay local to the disk-store owner. | -| `TIER_FREE_VERSION_RECOVERY_STARTED`, `TIER_DELETE_JOURNAL_RECOVERY_STARTED` | `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs` | Cache or constant / owner-local static guard | Lifecycle recovery single-run guards stay local to lifecycle operations. | -| `REMOTE_DELETE_INFLIGHT`, `REMOTE_DELETE_LIMITER`, `REMOTE_DELETE_BREAKER`, `REMOTE_TIER_DELETE_TEST_HOOK` | `crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs` | Cache or constant / owner-local static guard | Remote tier delete concurrency, breaker, and test hook state stay local to the tier sweeper owner. | -| `ACTIVE_REGISTRY`, `BackendCapacity` | `crates/kms/src/policy.rs` | Process-global owner-local admission capacity registry | KMS policy generations share only active semaphore capacity by backend identity; each generation owns fresh bounded queues and circuit breakers. Callers access this state only through `RetryPolicy`. | +```bash +rg -n -A4 'pub use crate::runtime::(sources|global)::' crates/ecstore/src/api/mod.rs +rg -n --glob '*.rs' 'static (ref )?GLOBAL_[A-Z_]+' crates rustfs/src +``` ## RustFS Owner-Local Static Inventory -These RustFS-side lazy, atomic, and `OnceLock` statics are also part of the -issue #730 process-static audit. They are private implementation details for -their owner modules, not shared runtime ownership handles. This section excludes -allocator statics, public contract/error references, route handler constants, -and `APP_CONTEXT_SINGLETON`, which is classified in the runtime migration -inventory. Generic function-local names such as `CACHE`, `LOCK`, `INIT`, and -`ENABLED` are documented by owner row instead of name-regex guarded. +RustFS-side statics that matter architecturally because other modules are tempted to reach them. They stay private to their owner module; callers use the owner's functions. -| State | Owner boundary | Category | Migration stance | -|---|---|---|---| -| `KEYSTONE_AUTH`, `KEYSTONE_MAPPER`, `KEYSTONE_CONFIG` | `rustfs/src/auth_keystone.rs` | Process-global owner-local state | Keystone authentication provider, identity mapper, and config stay private to the Keystone auth owner. | -| `LICENSE_STATE`, `LICENSE_VERIFIER` | `rustfs/src/license.rs` | Process-global owner-local state | License state and verifier selection stay private to the license owner; callers use license helper functions. | -| `CPU_CONT_GUARD`, `PROFILING_CANCEL_TOKEN` | `rustfs/src/profiling.rs` | Process-global owner-local guard | CPU profiling guard and cancellation state stay private to the profiling owner. | -| `MEMORY_SYSTEM` | `rustfs/src/memory_observability.rs` | Process-global owner-local cache | Memory sampling keeps the `sysinfo::System` cache private to the memory observability owner. | -| `DISPLAY_CONFIG_SNAPSHOT`, `GLOBAL_CONFIG_SNAPSHOT` | `rustfs/src/config/snapshot.rs` | Process-global owner-local state | Config snapshots stay private to the config snapshot owner. | -| `BUFFER_CONFIG_SINGLETON`, `BUFFER_PROFILE_ENABLED` | `rustfs/src/config/workload_profiles.rs` | Process-global owner-local state | Workload buffer profile configuration stays private to workload profile helpers. | -| `LEGACY_CREDENTIAL_WARNED_KEYS` | `rustfs/src/config/config_struct.rs` | Process-global owner-local cache | Legacy credential warning de-duplication stays private to config parsing. | -| `CONSOLE_CONFIG` | `rustfs/src/admin/console.rs` | Process-global owner-local state | Console bootstrap config stays private to the admin console owner. | -| `ACTIVE_HTTP_REQUESTS` | `rustfs/src/server/http.rs` | Process-global owner-local counter | HTTP request inflight accounting stays private to the HTTP server owner. | -| Function-local `CACHE` and `LOCK` statics | `rustfs/src/server/readiness.rs` | Cache or constant / owner-local cache | Readiness and cluster-health caches stay function-local to readiness probes. | -| `USE_STARSHARD_CACHE`, `BUCKET_CACHE_SMALL`, `BUCKET_CACHE_LARGE` | `rustfs/src/storage/ecfs_extend.rs` | Cache or constant / owner-local cache | Bucket validation cache backend selection and cache storage stay private to the ECFS extension owner. | -| `GLOBAL_SSE_DEK_PROVIDER`, `SSE_TEST_LOCK` | `rustfs/src/storage/sse.rs` | Owner-local cache / test state | SSE DEK provider cache and test serialization lock stay private to the SSE owner. | -| `AUTH_FS` | `rustfs/src/storage/access.rs` | Cache or constant / owner-local cache | Authorization tag-condition lookup keeps its filesystem helper private to the access owner. | -| `DEADLOCK_DETECTOR` | `rustfs/src/storage/deadlock_detector.rs` | Process-global owner-local state | Deadlock detector lifecycle state stays private to the storage deadlock detector owner. | -| `CONCURRENCY_MANAGER`, `ACTIVE_GET_REQUESTS`, `ACTIVE_PUT_REQUESTS` | `rustfs/src/storage/concurrency/*` | Process-global owner-local scheduler state | Storage concurrency manager and request counters remain inside the storage concurrency owner boundary. | -| `GET_OBJECT_BUFFER_THRESHOLD_WARNED`, `GET_READER_STREAM_BUFFER_SIZE_OVERRIDE`, function-local `ENABLED`, `OBJECT_SEEK_SUPPORT_THRESHOLD`, `OBJECT_SEEK_SUPPORT_CONCURRENCY_THRESHOLDS` | `rustfs/src/app/object/get.rs` | Cache or constant / owner-local cache | Object GET/seek tuning caches and warning guards stay private to object usecase helpers. | -| `SUPPORTED_HEADERS` | `rustfs/src/storage/options.rs` | Cache or constant / owner-local constant | Supported-header lookup state stays private to storage option parsing. | -| `AUDIT_TARGET_SPECS`, `NOTIFICATION_TARGET_SPECS` | `rustfs/src/admin/handlers/audit.rs`, `rustfs/src/admin/handlers/event.rs`, `rustfs/src/admin/handlers/plugins_instances.rs` | Cache or constant / owner-local constant | Admin target descriptor tables stay private to their handler owners. | -| `SITE_REPLICATION_PEER_CLIENT` | `rustfs/src/site_replication/transport.rs` | Process-global owner-local cache | Site-replication peer client cache stays private to the site-replication transport module. The state RMW transaction holds no process-local mutex — see `rustfs/src/site_replication/state_lock.rs`. | -| `AUDIT_MODULE_ENABLED`, `NOTIFY_MODULE_ENABLED`, `PERSISTED_NOTIFY_MODULE_ENABLED`, `PERSISTED_AUDIT_MODULE_ENABLED`, `PERSISTED_MODULE_SWITCH_CONFIGURED` | `rustfs/src/server/audit.rs`, `rustfs/src/server/event.rs`, `rustfs/src/server/module_switch.rs` | Process-global owner-local toggles | Audit/notify module snapshots stay private to the server module switch owners. | -| `DELETE_TAIL_TOTAL`, `DELETE_CLEANUP_TOTAL`, `DELETE_REPLICATION_TOTAL`, `DELETE_NOTIFY_TOTAL` | `rustfs/src/delete_tail_activity.rs` | Process-global owner-local counters | Delete-tail activity counters stay private behind delete-tail activity helpers. | -| `EMBEDDED_SERVER_STARTED` | `rustfs/src/startup_lifecycle.rs` | Process-global owner-local guard | Embedded startup single-start protection stays private to startup lifecycle. | -| `TEST_OUTBOUND_TLS_GENERATION` | `rustfs/src/site_replication/mod.rs` | Test or fixture state | Outbound TLS generation test hook state stays private to site-replication transport tests. | -| `TEST_REMAINING_FAILURES` | `rustfs/src/startup_iam.rs` | Test or fixture state | IAM startup retry injection state stays private to debug/test startup code. | -| `CAPACITY_DIRTY_SCOPE_ENV`, `CAPACITY_DIRTY_SCOPE_INIT`, `GLOBAL_ENV`, function-local `INIT` | `rustfs/src/app/*_test.rs` | Test or fixture state | App integration test fixture state stays private to the owning test modules. | +| Static | Owner | Stance | +|---|---|---| +| `KEYSTONE_AUTH`, `KEYSTONE_MAPPER`, `KEYSTONE_CONFIG` | `rustfs/src/auth_keystone.rs` | Keystone provider, mapper, and config stay private to the Keystone owner. | +| `DEADLOCK_DETECTOR` | `rustfs/src/storage/deadlock_detector.rs` | Detector lifecycle stays private to the storage deadlock detector. | +| `CONCURRENCY_MANAGER` | `rustfs/src/storage/concurrency/manager.rs` | Storage concurrency scheduler state stays inside the concurrency owner. | +| `GLOBAL_KMS_DEK_PROVIDER`, `GLOBAL_SSE_DEK_PROVIDER` | `rustfs/src/storage/sse.rs` | DEK provider caches stay private to the SSE owner. | +| `ECSTORE_EVENT_DISPATCH_HOOK` | `rustfs/src/server/event.rs` | Event bridge registration goes through the storage facade. | +| `AUDIT_MODULE_ENABLED`, `NOTIFY_MODULE_ENABLED` | `rustfs/src/module_switches.rs` | Module toggles are read through module-switch helpers; `MODULE_SWITCH_RMW_LOCK` (`rustfs/src/server/module_switch.rs`) serializes persisted updates. | +| `RUNTIME_CONFIG_RELOAD_MUTEX` | `rustfs/src/admin/service/config.rs` | Serializes dynamic config reload fanout. | +| `EMBEDDED_RUNTIME_OWNERS` | `rustfs/src/startup_shutdown.rs` | Embedded runtime owner handles used for shutdown ordering. | +| `SERVICE_FROZEN` | `rustfs/src/admin/handlers/system.rs` | Service freeze flag stays behind the system admin handler. | +| `RECONCILER` | `rustfs/src/site_replication_reconcile.rs` | Site-replication reconciler singleton. | +| `CONSOLE_CONFIG` | `rustfs/src/admin/console.rs` | Console bootstrap config. | +| `LICENSE_STATE`, `LICENSE_VERIFIER` | `rustfs/src/license.rs` | License state and verifier stay behind license helpers. | -## First Code-Bearing Candidate +Regenerate the full list (long, mostly caches and test hooks): -`GLOBAL_EXPIRY_STATE` is the safest first runtime migration candidate: - -- AppContext already exposes `ExpiryStateInterface` and resolver coverage in - `rustfs/src/app/context.rs`. -- ECStore access is already concentrated in - `crates/ecstore/src/runtime/sources.rs`. -- The main external readers can be moved through storage/observability facades - before changing lifecycle queue ownership. - -Do not migrate `GLOBAL_OBJECT_API` first. It is coupled to storage startup, -object-store resolver publication, IAM-after-storage AppContext initialization, -and broad data-plane compatibility. - -## Verification - -Inventory and guardrail PRs should run: - -- `bash -n scripts/check_architecture_migration_rules.sh` -- `./scripts/check_architecture_migration_rules.sh` -- `cargo fmt --all --check` -- `git diff --check` - -Code-bearing migration PRs must add focused tests for the owner being moved -before running broader gates. +```bash +rg -n '^\s*(pub(\(crate\))? )?static [A-Z_]+' rustfs/src +``` diff --git a/docs/architecture/heal-concurrency-model.md b/docs/architecture/heal-concurrency-model.md new file mode 100644 index 000000000..a71dcca37 --- /dev/null +++ b/docs/architecture/heal-concurrency-model.md @@ -0,0 +1,77 @@ +# Heal concurrency model + +**Use this when:** changing heal, PUT/multipart commit, delete, lifecycle expiry, or data-movement code that touches the same `(bucket, object)` commit surface; or evaluating whether RustFS needs a persistent per-object healing marker like MinIO's `x-minio-healing`. +**Source of truth:** `crates/ecstore/src/set_disk/ops/heal.rs` (`heal_object_with_explicit_version_regen`, `HealObjectLockKind`, `HEAL_RENAME_INCOMPLETE`), `crates/ecstore/src/set_disk/ops/object.rs` (PUT/DELETE lock sections, `reconcile_old_data_cleanup_receipts`), `crates/ecstore/src/set_disk/core/io_primitives.rs` (`commit_rename_data_dir`, `report_old_data_dir_cleanup`, `reclaim_orphan_data_dirs`), `crates/filemeta/src/fileinfo.rs` (`FileInfo::set_healing`), `crates/heal/src/heal/manager/queue.rs` (dedup keys). + +## Model + +Heal and every foreground or background write path serialize on the same object-level namespace write lock (a quorum lock RPC in distributed mode, the in-process lock manager on a single node; granularity is the object, the version component is always `None`), and heal holds its guard across the whole rename commit. MinIO's `x-minio-healing` marker is an out-of-lock defence against version-cleanup logic inside `RenameData` interleaving with a heal commit; RustFS's commit model has no such interleaving, so no persistent marker exists (`x-minio-healing` does not occur in `crates/` or `rustfs/`) and none is needed. Three layers replace it: + +| Layer | Mechanism | Owner | +| --- | --- | --- | +| In-lock mutual exclusion | Heal and all write-path commit points take the `(bucket, object)` namespace write lock. | `acquire_heal_object_lock` in `crates/ecstore/src/set_disk/ops/heal.rs`; lock sections in `ops/object.rs` and `ops/multipart.rs` | +| Commit-model isolation | `rename_data` contains no version cleanup that could interleave with heal. Physical deletion of a replaced old `data_dir` runs after the object lock is released (the commit tail) and only for unshared directories already superseded by the new commit. | `commit_rename_data_dir` in `crates/ecstore/src/set_disk/core/io_primitives.rs` | +| Transient healing flag | `FileInfo::set_healing` sets the internal `SUFFIX_HEALING` key on the in-memory `FileInfo` of a heal commit; `rename_data` reads it through `is_healing` to clear a stale non-empty target `data_dir` before the rename (in-place repair reuses the `data_dir`, and `rename(2)` cannot replace a non-empty directory). The key is never persisted (`is_skip_meta_key` in `crates/filemeta/src/filemeta.rs`). A non-heal commit that meets a non-empty target fails explicitly; tests lock both directions. | `crates/filemeta/src/fileinfo.rs`, `crates/ecstore/src/disk/local.rs` | + +## Heal lock scope + +`heal_object` delegates to `heal_object_with_explicit_version_regen`, which takes the namespace write lock at entry unless `opts.no_lock` is set and binds the guard to the function scope. The guard covers the quorum metadata read, EC reconstruction, per-disk rename commit, tmp cleanup, the `HEAL_RENAME_INCOMPLETE` partial-commit return, and orphan `data_dir` reclamation (`reclaim_orphan_data_dirs`). + +Read-repair heals (`opts.read_repair`) hold a shared lock (`HealObjectLockKind::Read`) during reconstruction so readers keep flowing, then `acquire_revalidated_read_repair_commit_lock` takes the write lock and re-reads a commit fingerprint; a changed fingerprint aborts the commit (`read_repair_commit_stale`). + +## Lock-intersection matrix + +| # | Concurrent path | Lock held by that path | Outcome | Where | +| --- | --- | --- | --- | --- | +| 1 | PUT commit | object write lock; `rename_data` inside it | serialized | `ops/object.rs` put commit | +| 2 | PUT old `data_dir` tail cleanup | none (runs after the lock is dropped) | unlocked, semantically safe ([commit tail](#commit-tail-cleanup)) | `commit_rename_data_dir` in `core/io_primitives.rs` | +| 3 | DELETE object or version | object write lock; `delete_version` inside it | serialized | `ops/object.rs` `delete_object` | +| 4 | Batch DELETE | per-object write locks (batch lock RPC in distributed mode) | serialized | `ops/object.rs` `delete_objects` | +| 5 | CompleteMultipartUpload | object write lock plus upload-path lock; rename inside | serialized | `ops/multipart.rs` | +| 6 | CompleteMultipart tail cleanup | none (after lock drop) | unlocked, semantically safe ([commit tail](#commit-tail-cleanup)) | `ops/multipart.rs` | +| 7 | AbortMultipartUpload | upload-path lock in the multipart bucket only | disjoint resources: abort never touches the object `data_dir` or `xl.meta` | `ops/multipart.rs` | +| 8 | ILM expiry including DeleteAllVersions | `delete_prefix_object=true` keeps the object lock; `FreeVersionTask` locks explicitly; noncurrent batches use batch locks | serialized | `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs` | +| 9 | Pure prefix delete | `delete_prefix` without `delete_prefix_object` takes no child-object lock | unlocked; no production caller ([prefix delete](#pure-prefix-delete)) | `ops/object.rs` lock condition in `delete_object` | +| 10 | Orphan `data_dir` reclamation | none inside the function; its only production caller runs inside the heal lock | serialized within heal | `reclaim_orphan_data_dirs` in `core/io_primitives.rs` | +| 11 | Old-cleanup receipt reconciliation | none inside the function; caller runs inside the heal lock and an epoch fence rejects stale receipts | serialized | `reconcile_old_data_cleanup_receipts` in `ops/object.rs` | +| 12 | Replication | data plane writes to the remote over HTTP; local metadata write-back takes the object lock | serialized or disjoint | `crates/ecstore/src/bucket/replication/replication_resyncer.rs` | +| 13 | Data movement, rebalance, decommission source cleanup | explicit object lock plus version-unchanged recheck; `no_lock` only reuses an already-held guard | serialized | `crates/ecstore/src/data_movement/mod.rs` | +| 14 | CopyObject | destination object lock through the PUT chain | serialized | `ops/object.rs` `copy_object` | +| 15 | Another heal task (different `HealType`, or `force_start`) | dedup keys are per `HealType` and `force_start` skips dedup, so tasks may coexist | serialized on the namespace write lock | `make_dedup_key_for_type` in `crates/heal/src/heal/manager/queue.rs` | +| 16 | Admin heal with `nolock=true` | caller bypasses the lock | unlocked by operator choice ([no_lock](#no_lock-and-force_start)) | `rustfs/src/admin/handlers/heal.rs` | +| 17 | Stale multipart cleanup | upload-path lock in the multipart bucket | disjoint resources | `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs` | + +## Residual windows + +### Commit tail cleanup + +Rows 2 and 6. After a write path commits and releases the object lock, it best-effort deletes the replaced old `data_dir`; the code deliberately does not block the next operation on this. The deletion can race a concurrent heal reading or rebuilding that same old `data_dir`, but the race is semantically safe: + +- The target is an unshared `data_dir` already replaced by the new commit. Heal's canonical metadata comes from quorum arbitration (ETag, mod time), and quorum already points at the new version, so heal cannot resurrect the replaced version as canonical. +- The worst outcome is one transient failure or no-op for the heal round on the old version; the next round converges. Cleanup residue is reported and re-queued for heal via `report_old_data_dir_cleanup`. +- Long heals such as drive replacement request explicit versions and read quorum metadata inside the lock, so the tail does not affect them. + +### Pure prefix delete + +Row 9. `delete_prefix && !delete_prefix_object` takes no child-object locks (an object namespace lock cannot protect a recursive prefix delete), so a heal running during the prefix delete could theoretically rebuild a version from stale quorum metadata. Every production `delete_prefix: true` call site also sets `delete_prefix_object: true` (and therefore takes the object lock); the remaining `delete_prefix`-only call sites are in test modules. A future caller that needs a pure prefix delete must prove isolation from heal and scanner at the call site (for example a bucket-level scan fence). + +### `no_lock` and `force_start` + +Row 16. Admin heal requests pass the client's `nolock` parameter through (`rustfs/src/admin/handlers/heal.rs`), matching the MinIO madmin option. Setting it is an explicit operator choice that accepts races with concurrent writes; it is documented, not restricted. + +Heal-side invariants that hold regardless of the caller: + +- Dedup keys are disjoint across `HealType` (object, metadata, MRF, EC decode, prefix), and admin `force_start` skips dedup. Several heal tasks for one object can therefore exist at once, but every production entry calls `heal_object` with `no_lock=false`, so their execution bodies serialize on the namespace write lock. +- Read-repair's local TTL reservation dedups only its own source and does not block heals from other sources; the namespace lock is the backstop. +- The healing flag is never persisted, so there is no reverse risk of a leftover marker making a later commit yield incorrectly. + +## Regression tests + +Both live in the test module of `crates/ecstore/src/set_disk/ops/heal.rs`: + +| Test | Invariant | +| --- | --- | +| `heal_racing_version_delete_never_resurrects_the_deleted_version` | With a doomed version's shards corrupted, a versioned DELETE and a deep heal contend on the same lock; the deleted version is not resurrected and the surviving version is intact. | +| `heal_racing_unversioned_overwrites_preserves_the_last_commit` | Unversioned overwrite commits (exercising the commit-tail old `data_dir` deletion) race a deep-heal loop; the final current version is exactly the last commit (ETag-level equality). | + +Related: the atomic-commit and best-effort-rollback invariants for the write path are in [erasure-coding.md](erasure-coding.md). diff --git a/docs/architecture/kms-bulk-rekey-contract.md b/docs/architecture/kms-bulk-rekey-contract.md index 267e5f9fb..5fcfd7c71 100644 --- a/docs/architecture/kms-bulk-rekey-contract.md +++ b/docs/architecture/kms-bulk-rekey-contract.md @@ -1,184 +1,143 @@ # KMS Bulk Rekey Job Contract -This document defines the contract for the object-side bulk rekey job: a long-running administrative job that re-wraps stored data-key envelopes under the current key-encryption key (KEK) without rewriting object bodies. A first execution engine has shipped: the sweep in `rustfs/src/kms_rekey.rs`, driven by the admin endpoints in `rustfs/src/admin/handlers/kms_rekey.rs`. The contract remains the acceptance bar; where the shipped v1 sweep deliberately narrows it, the [Implementation Status](#implementation-status-v1-sweep) section records the deviation so the document and the tree cannot drift apart silently. +**Use this when:** changing the bulk envelope re-wrap sweep (`rustfs/src/kms_rekey.rs`), its admin endpoints (`rustfs/src/admin/handlers/kms_rekey.rs`), the re-wrap primitive, or anything that decides which objects a rekey may touch. +**Source of truth:** `rustfs/src/kms_rekey.rs`, `rustfs/src/admin/handlers/kms_rekey.rs`, `rewrap_object_encryption_metadata` in `rustfs/src/storage/sse.rs`, `KmsManager::rewrap_data_key` / `KmsManager::describe_data_key_wrapping` in `crates/kms/src/manager.rs`, `put_object_metadata` in `crates/ecstore/src/set_disk/ops/object.rs`. -It tracks [`rustfs/backlog#1642`](https://github.com/rustfs/backlog/issues/1642), which lands the `bulk migrate/rekey` line of [`rustfs/backlog#1562`](https://github.com/rustfs/backlog/issues/1562). +The bulk rekey job re-wraps stored data-key envelopes under the current key-encryption key (KEK) without rewriting object bodies. This document is the acceptance bar; where the shipped v1 sweep deliberately narrows it, [Implementation Status](#implementation-status-v1-sweep) records the deviation. ## Scope -- Applies to: the job lifecycle, ownership, idempotency, failure semantics, exclusion rules, and completion evidence for bulk envelope re-wrap. -- Out of scope, and deliberately so: the cryptographic definition of a single-object re-wrap (owned by the re-wrap primitive), master key material migration between KMS backends, a pause state, multi-node parallel execution, and destruction of superseded key versions. - -### Why master key material migration is not this job - -Vault Transit, AWS KMS, and HSM backends are designed so that key material cannot be exported. There is no path that moves a Local master key into Transit, and the reverse direction would export production key material from an HSM onto local disk, which is a security regression. The one case that is both possible and useful, Local to Local, is already served by the KMS backup and restore bundle in `crates/kms/src/backup/local_export.rs` and `crates/kms/src/backup/local_restore.rs`. Nothing in this contract creates a second, weaker copy of that capability. +- Applies to: job lifecycle, ownership, idempotency, failure semantics, exclusion rules, and completion evidence for bulk envelope re-wrap. +- Out of scope: the cryptographic definition of a single-object re-wrap (owned by the primitive), master key material migration between backends, a pause state, multi-node parallel execution, destruction of superseded key versions. +- Master key material migration is not this job: Vault Transit, AWS KMS, and HSM backends do not export key material, and the one useful case (Local to Local) is already served by `crates/kms/src/backup/local_export.rs` and `crates/kms/src/backup/local_restore.rs`. ## Implementation Status (v1 Sweep) -The shipped sweep (`rustfs/src/kms_rekey.rs`, admin surface `POST /rustfs/admin/v3/kms/keys/rekey` plus `/status` and `/cancel`, all gated on the cluster-scoped `kms:Rekey` action) implements the contract with these deliberate narrowings: +The shipped sweep (`POST /rustfs/admin/v3/kms/keys/rekey` plus `/status` and `/cancel`, gated on the cluster-scoped `kms:Rekey` action) narrows the contract as follows: -- **One sweep per process, not scope-scoped admission.** A single in-memory slot serializes sweeps cluster-wide on the node that received the request; a second start request is refused with the running job id. This is narrower than the scope-scoped ownership below — two disjoint-scope jobs cannot run concurrently — which is the safe direction: concurrent sweeps would double every KMS round-trip for zero extra coverage. The persisted CAS job record, lease, and crash-recovered ownership described under [Skeleton, Ownership, And Admission](#skeleton-ownership-and-admission) are not implemented; job state and counters are process-local and reset on restart. Correctness does not depend on them: the envelope itself is the resume state. -- **Cursor-free convergence.** No checkpoint exists at all. The contract already declared the cursor a performance optimization; v1 takes that to its limit — recovery from a crash, cancel, or partial failure is re-running the sweep, and every already-current envelope costs one describe-shaped KMS call and no write. -- **Backend gate at start.** The start endpoint refuses with `501` when the configured backend does not advertise `BackendCapabilities::rewrap`. Vault KV2 and Vault Transit pass; Local, Static, and AWS are refused. This is the "refused at admission" behavior the contract requires for AWS, and it is also what disarms the Local blocker below: a sweep can only run where superseded key versions demonstrably remain decryptable. -- **Collapsed exclusion counting.** Plaintext objects, SSE-C objects, and MinIO-sealed envelopes are counted together as `not_applicable` rather than per-class; delete markers and directory entries are skipped without counting. Per-class exclusion counts remain future work. -- **No dry run.** The dry-run report model below is not implemented; the closest present capability is reading `/status` counters from a completed sweep. -- **Admission posture.** The sweep processes exactly one object at a time — each iteration awaits a KMS round-trip and, on rewrap, one metadata write — so its foreground contention is bounded by strict serialization, the KMS policy layer's shared concurrency cap, and the storage layer's own namespace locks and quorum rules. It does not integrate with a workload-admission mechanism, because [workload-admission-contracts.md](workload-admission-contracts.md) currently defines an observation-only snapshot surface repo-wide, with no runtime admission API for any background job to join. When such a mechanism exists, this job joins it alongside the scanner, heal, and decommission; until then, the requirement is bounded contention, which serialization provides. +| Contract item | v1 behavior | +|---|---| +| Ownership / admission | One in-memory slot per process serializes sweeps; a second start request is refused with the running job id. No persisted CAS job record, lease, or crash-recovered ownership; counters are process-local and reset on restart. | +| Resume cursor | None. Recovery from crash, cancel, or partial failure is re-running the sweep; every already-current envelope costs one describe-shaped KMS call and no write. | +| Backend gate | Start refuses with `501` when the backend does not advertise `BackendCapabilities::rewrap` (`crates/kms/src/backends/mod.rs`). Vault KV2 and Vault Transit pass; Local, Static, and AWS are refused. | +| Exclusion counting | Plaintext, SSE-C, and MinIO-sealed envelopes are counted together as `not_applicable`; delete markers and directory entries are skipped without counting. | +| Dry run | Not implemented; the closest capability is `/status` counters from a completed sweep. | +| Admission posture | Exactly one object at a time (one KMS round-trip, then at most one metadata write). No workload-admission integration: [workload-admission-contracts.md](workload-admission-contracts.md) defines an observation-only snapshot surface with no runtime admission API for a background job to join. | -What v1 keeps exactly as contracted: work units are `(bucket, object, versionId)` with `latest_only: false`; `mod_time` is never set on the rewrap write; object-lock retention is inherited from `put_object_metadata`; the rewrap replaces every stored envelope copy by value match across the RustFS-internal and MinIO-compatible slots, and treats "no replaceable copy found" as an error rather than a silent success — the stale-branch hazard rule from [Metadata Write Contract](#metadata-write-contract); failures are counted and logged per object and never abort the sweep; cancellation is cooperative and terminal. +Kept exactly as contracted: work units are `(bucket, object, versionId)` with `latest_only: false`; `mod_time` is never set on the rewrap write; object-lock retention is inherited from `put_object_metadata`; every stored envelope copy is replaced by value match across the RustFS-internal and MinIO-compatible slots, and "no replaceable copy found" is an error, not a silent success; failures are counted and logged per object and never abort the sweep; cancellation is cooperative and terminal. ## Terms | Term | Meaning | |---|---| -| Envelope | The sealed data key (DEK) stored on an object version's metadata, together with the identifiers needed to unseal it. | -| Re-wrap primitive | A single-object operation that unseals one envelope and re-seals it under the target KEK, changing metadata only. Implemented as `rewrap_object_encryption_metadata` in `rustfs/src/storage/sse.rs`, over `KmsManager::rewrap_data_key`. | -| Rekey job | The scan-and-drive layer defined by this document, which applies the re-wrap primitive across a scope. | +| Envelope | The sealed data key (DEK) stored on an object version's metadata, with the identifiers needed to unseal it. | +| Re-wrap primitive | Single-object operation that unseals one envelope and re-seals it under the target KEK, changing metadata only: `rewrap_object_encryption_metadata` over `KmsManager::rewrap_data_key`. | +| Rekey job | The scan-and-drive layer defined here, applying the primitive across a scope. | | Work unit | One `(bucket, object, versionId)` triple. Never `(bucket, object)`: each version carries its own envelope. | -| Scope | The bucket and prefix selector that bounds one job, and the unit of admission exclusion. | -| Target state | The envelope state the job is driving toward: sealed under the intended key id at the current KEK version. | +| Scope | The bucket and prefix selector that bounds one job; the unit of admission exclusion. | +| Target state | Envelope sealed under the intended key id at the current KEK version. | -## What the Job Does And Does Not Do +## What The Job Does And Does Not Do -The job re-wraps envelopes. It never rewrites object bodies. Erasure-coded shards, part layout, ETag, and storage usage must be unchanged across a rekey; only encryption metadata keys may differ. Metadata-only rewrite is supported by the storage layer: `put_object_metadata` is declared on `ObjectStore` in `crates/ecstore/src/store/mod.rs`, dispatched in `crates/ecstore/src/core/sets.rs`, and implemented in `crates/ecstore/src/set_disk/ops/object.rs`, where it takes a namespace write lock, selects the version named by `opts.version_id`, and merges `opts.eval_metadata` into the existing `FileInfo` metadata under read and write quorum. - -**The job never destroys a superseded key version.** This is the hardest constraint in this contract, and every other guarantee rests on it. A job that fails halfway leaves some objects wrapped under the new KEK version and some under the old one. That state is fully serviceable — reads and writes both succeed — precisely and only because the old version can still decrypt. Destroying old versions from inside the job would convert a resumable operational action into irreversible data loss on partial failure. Destruction stays a separate, human-initiated operation gated on usage evidence. - -A job must therefore refuse to start when the target key's retention policy would allow the superseded version to leave the retention window while the job runs. +- Re-wraps envelopes only. Erasure-coded shards, part layout, ETag, and storage usage are unchanged; only encryption metadata keys may differ. The metadata-only write is `put_object_metadata` (declared on `ObjectStore` in `crates/ecstore/src/store/mod.rs`, dispatched in `crates/ecstore/src/core/sets.rs`, implemented in `crates/ecstore/src/set_disk/ops/object.rs`). +- **Never destroys a superseded key version.** A half-finished job leaves some envelopes under the new KEK version and some under the old; that state is serviceable only because the old version still decrypts. Destruction stays a separate, human-initiated operation gated on usage evidence. +- Must refuse to start when the target key's retention policy would let the superseded version leave the retention window while the job runs. ## Idempotency Model -Re-running the job must be safe and must converge. The intended source of idempotency is the object metadata itself: the envelope's own state is the target state, so a re-run reads what is already correct and skips it. No separate idempotency table is required, and the job identity is only a `job_id: Uuid` for reporting and ownership, following the ILM manual transition job record in `crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs`. +Idempotency comes from object metadata itself: the envelope's state is the target state, so a re-run reads what is already correct and skips it. No idempotency table; the job identity is a `job_id: Uuid` for reporting and ownership, following `ManualTransitionJobRecord` in `crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs`. -Two consequences follow, and both are contract requirements: - -- **The resume cursor is a performance optimization, not a correctness dependency.** Losing a checkpoint may cause a rescan and a higher skip count, never a wrong result. This is what makes crash recovery cheap: checkpoints may be throttled rather than written per object, following the `PersistThrottle` policy in `crates/heal/src/heal/resume.rs`, which flushes after a bounded number of buffered mutations or a bounded interval, whichever comes first. That module states the same reasoning for heal: because the operation is idempotent, a crash re-does at most one throttle window. -- **The job is at-least-once with target-state idempotency, never exactly-once.** No design may introduce exactly-once machinery for work units. +- **The resume cursor is a performance optimization, not a correctness dependency.** Losing a checkpoint may cause a rescan and a higher skip count, never a wrong result. Checkpoints may therefore be throttled (`PersistThrottle` in `crates/heal/src/heal/resume.rs`). +- **At-least-once with target-state idempotency, never exactly-once.** No design may introduce exactly-once machinery for work units. ### Reading the wrapping KEK version -The self-evidencing property above holds only when the wrapping KEK version is observable. It is, for every backend that actually rotates, but not from a dedicated metadata field and not by the same mechanism on each backend. - -There is no key-version metadata key: object metadata carries the key **id** (`x-rustfs-encryption-key-id` in `rustfs/src/storage/sse.rs`, defaulting to `default`) and the sealed blob under `x-rustfs-encryption-key`, and nothing else names a version. `DecryptResponse` in `crates/kms/src/types.rs` does not report one either, though `EncryptResponse` does. - -The version is nonetheless recoverable, because the sealed blob is structured. `x-rustfs-encryption-key` stores the base64 of the backend ciphertext, and for every backend that builds one that ciphertext is the JSON of `DataKeyEnvelope` (`crates/kms/src/encryption/dek.rs`). Reading it needs no new metadata: base64-decode the value, then parse the JSON. The read path in `rustfs/src/storage/sse.rs` already does exactly this discrimination, calling `is_data_key_envelope` on the decoded blob to pick a provider, so this is an established in-tree pattern rather than a new capability. - -Where the version sits inside that structure is backend-specific: +There is no key-version metadata key. Object metadata carries the key id (`x-rustfs-encryption-key-id`) and the sealed blob under `x-rustfs-encryption-key`; `DecryptResponse` in `crates/kms/src/types.rs` does not report a version either. The version is recoverable because the sealed blob is structured: for every backend that builds one, the ciphertext is the JSON of `DataKeyEnvelope` (`crates/kms/src/encryption/dek.rs`), and the read path already discriminates on it via `is_data_key_envelope` in `rustfs/src/storage/sse.rs`. | Backend | Rotates | Where the wrapping version lives | Recoverable by a scan | |---|---|---|---| -| Vault KV2 (`crates/kms/src/backends/vault.rs`) | Yes | `DataKeyEnvelope::master_key_version`, populated from the key record's version | Yes, from the envelope JSON | -| Vault Transit (`crates/kms/src/backends/vault_transit.rs`) | Yes | The `vault:vN:` prefix of the ciphertext held in the envelope's `encrypted_key`; the envelope's own version field is deliberately `None` because Transit ciphertext self-describes | Yes, by parsing that prefix | -| Local (`crates/kms/src/backends/local.rs`) | No — rotation is rejected | Nowhere; the version field is hardcoded `None` because a key has exactly one material | Moot while rotation is rejected | -| Static (`crates/kms/src/backends/static_kms.rs`) | No — single fixed key | Nowhere; hardcoded `None` | Moot | -| AWS (`crates/kms/src/backends/aws.rs`) | AWS-managed | Inside the opaque `CiphertextBlob`; no `DataKeyEnvelope` is built at all | **No** | +| Vault KV2 (`crates/kms/src/backends/vault.rs`) | Yes | `DataKeyEnvelope::master_key_version` | Yes, from the envelope JSON | +| Vault Transit (`crates/kms/src/backends/vault_transit.rs`) | Yes | `vault:vN:` prefix of the ciphertext in `encrypted_key`; the envelope's version field is deliberately `None` | Yes, by parsing that prefix | +| Local (`crates/kms/src/backends/local.rs`) | No, rotation is rejected | Nowhere; hardcoded `None` | Moot while rotation is rejected | +| Static (`crates/kms/src/backends/static_kms.rs`) | No | Nowhere; hardcoded `None` | Moot | +| AWS (`crates/kms/src/backends/aws.rs`) | AWS-managed | Inside the opaque `CiphertextBlob`; no `DataKeyEnvelope` | **No** | -Two traps follow, and both are contract rules. +Contract rules that follow: -**`None` does not mean one thing.** On Vault KV2 it means a pre-versioning envelope, and `resolve_envelope_master_key_version` resolves it to the key's recorded baseline version, or to the current version for a key that was never rotated — never implicitly to whatever is current now. On Transit it is permanent and expected, and the version must be read from the ciphertext prefix instead. On Local and Static it is unconditional. A scan that reads `None` as a single condition will misclassify three different situations, so version extraction must be dispatched by backend, never inferred from the field alone. - -**Local's `None` is coupled to the blocker below.** The Local backend omits the version specifically because rotation is rejected there. When [`rustfs/backlog#1565`](https://github.com/rustfs/backlog/issues/1565) gives Local a rotation history, that construction must begin recording the wrapping version in the same change, or Local silently becomes a second unreadable backend and loses idempotent skip along with it. This coupling is not obvious from either issue and must not be discovered later. - -The requirement this places on the re-wrap primitive is therefore narrower than "record a version", most of which the tree already satisfies: - -- The primitive must expose the wrapping version through **one backend-dispatched accessor** — satisfied by `KmsManager::describe_data_key_wrapping`, which dispatches per backend so callers never reimplement envelope-field or ciphertext-prefix parsing, which would also put KMS format knowledge on the wrong side of the crate boundary. -- The primitive must report **"already at target state" as an outcome distinct from "re-wrapped"**, so the job counts a skip instead of inferring one. -- For AWS, neither is achievable by inspection, and the contract must say so rather than pretend otherwise (see below). - -### The cost of recognizing the target state - -Skipping already-current objects is achievable, and it is not free. Every scanned work unit costs a base64 decode plus a JSON parse of its envelope, and on Transit an additional prefix parse. That is CPU and allocation per object version, not extra I/O: the metadata is already being read by the scan, and no KMS round trip is involved. Envelopes are small, so the cost is bounded per object, but at bulk scale it is the dominant cost of a dry run and of the skip check in a re-run, and it belongs in the rate and admission budget rather than being treated as free. - -This cost buys three things, all of which the contract requires and none of which are available without it: a re-run that skips completed work and performs zero metadata writes, a dry run that reports which KEK versions are actually in scope, and the per-object half of completion evidence. - -**AWS is the exception, and it is a scoping exception rather than a cost.** Its ciphertext is opaque to RustFS, so no inspection can tell a current envelope from a stale one. A rekey scope on an AWS-backed key therefore cannot skip, cannot report version composition in a dry run, and cannot self-evidence completion; a re-run would re-wrap every object again. AWS also rotates backing key material transparently on decrypt, so the operational need that motivates this job is weaker there to begin with. Until there is a reason to do otherwise, AWS-backed keys are out of scope for bulk rekey, and a job must refuse such a scope at admission rather than start one whose re-runs silently rewrite everything. +- **`None` does not mean one thing.** KV2: pre-versioning envelope, resolved by `resolve_envelope_master_key_version` to the key's recorded baseline, never implicitly to "current". Transit: permanent and expected; read the ciphertext prefix. Local/Static: unconditional. Version extraction must be dispatched by backend, never inferred from the field alone. +- **Local's `None` is coupled to the Local blocker.** If Local gains rotation history (`rustfs/backlog#1565`), envelope version recording must land in the same change, or Local becomes a second unreadable backend. +- The primitive exposes the wrapping version through **one backend-dispatched accessor** (`KmsManager::describe_data_key_wrapping`) and reports **"already at target state" as an outcome distinct from "re-wrapped"**. +- **AWS is a scoping exception.** Its ciphertext is opaque, so no scan can skip, report version composition, or self-evidence completion; a re-run would rewrap everything. AWS-backed keys are out of scope and must be refused at admission. +- Skip detection costs a base64 decode plus JSON parse (plus a prefix parse on Transit) per work unit: CPU, not I/O, and part of the rate budget rather than free. ## Failure Semantics -A partially complete rekey is a valid, serviceable state, not a damaged one. It requires no emergency handling, no fail-closed startup guard, and no rollback. This is the sharpest difference from KMS backup restore, whose intermediate state genuinely is unserviceable and which therefore fails closed on startup when its commit marker is present. - -The precondition is that superseded key versions remain decryptable. Where that precondition does not hold, the whole model collapses (see Blockers). - -Cancellation is cooperative and terminal. A canceled job reaches a terminal state with already-processed objects left in the target state; restarting on the same scope skips them. - -## Pause Is Not Provided - -The originating requirement asked for pause, resume, and idempotent retry. This contract provides cancel, cursor restart, and rate control instead, and does not provide a pause state. - -Seven long-running job frameworks exist in the tree — ILM manual transition, heal resume (`crates/heal/src/heal/resume.rs`), tier mutation intent (`crates/ecstore/src/services/tier/tier_mutation_intent.rs`), decommission and rebalance (`crates/ecstore/src/core/pools.rs`), the scanner (`crates/scanner/src/scanner.rs`), and KMS backup restore. None of them has a pause state; each has cancel or stop only. That consistency is a design position, not an oversight. A paused job has to answer what it still holds: whether its lease is renewed, whether it keeps its scope admission slot, and how long it may stay paused before it is abandoned. Each answer adds state and a failure mode. - -The two things pause is actually asked for are that the job must not overwhelm the data path, and that stopping it must not throw away progress. Rate and admission control delivers the first; cancel plus cursor restart delivers the second. Both are existing patterns. +- A partially complete rekey is a valid, serviceable state: no emergency handling, no fail-closed startup guard, no rollback. This is the sharpest difference from KMS backup restore, whose intermediate state is unserviceable and fails closed on startup. +- Precondition: superseded key versions remain decryptable (see Blockers). +- Cancellation is cooperative and terminal; restarting on the same scope skips already-processed objects. +- Pause is not provided. None of the tree's long-running job frameworks (ILM manual transition, heal resume, tier mutation intent, decommission/rebalance, scanner, KMS restore) has a pause state; rate control plus cancel-and-restart deliver what pause is asked for without lease/slot/abandonment state. ## Objects That Cannot Be Rekeyed -These must be enumerated during the scan and excluded with a counted reason. Encountering one is never a job failure, and the execution phase must not touch them. +Enumerated during the scan and excluded with a counted reason; never a job failure; the execution phase must not touch them. | Class | Disposition | Reason | |---|---|---| -| SSE-C objects | Exclude and count | The server never holds the customer key, so it can neither unseal nor re-seal the envelope. | -| Objects transitioned to a remote tier | Exclude and count | The body lives remotely; the relationship between local metadata and the remote object's encryption needs its own analysis first. See [tier-ilm-debugging.md](../operations/tier-ilm-debugging.md). | -| In-progress multipart uploads | Exclude and count | Each part carries its own envelope and an incomplete upload is not a stable work unit. `crates/kms/src/key_impact.rs` already models this as a distinct reference scope. | -| Unencrypted objects | Exclude and count | No envelope to re-wrap. | -| Objects under object-lock retention | Governed by the storage layer, see below | | +| SSE-C objects | Exclude and count | The server never holds the customer key. | +| Objects transitioned to a remote tier | Exclude and count | Body lives remotely; see [tier-ilm-debugging.md](../operations/tier-ilm-debugging.md). | +| In-progress multipart uploads | Exclude and count | Each part carries its own envelope; `crates/kms/src/key_impact.rs` models this as a distinct reference scope. | +| Unencrypted objects | Exclude and count | No envelope. | +| Objects under object-lock retention | Governed by the storage layer (see Metadata Write Contract) | | -Replication destinations are unresolved: whether an envelope metadata rewrite must propagate to a replica depends on [`rustfs/backlog#1619`](https://github.com/rustfs/backlog/issues/1619). Until that closes, this contract does not authorize propagation. +Replication destinations are unresolved: propagation depends on `rustfs/backlog#1619`. Until it closes, a rewrap never propagates to a replica and each site runs its own sweep. ## Metadata Write Contract -Three properties of `put_object_metadata` constrain the re-wrap write, all confirmed in `crates/ecstore/src/set_disk/ops/object.rs`. +Three properties of `put_object_metadata` (`crates/ecstore/src/set_disk/ops/object.rs`) constrain the re-wrap write: -**The merge is additive; it cannot remove keys.** `opts.eval_metadata` entries are inserted into the existing metadata map. There is no removal path. Overwriting a key that keeps its name is therefore safe, but a re-wrap that changes *which* metadata keys describe the envelope leaves the old keys behind permanently. - -That is a structural hazard, not a theoretical one. `rustfs/src/storage/sse.rs` selects its decrypt branch on the mere presence of the MinIO-compatible seal-algorithm header: `parse_minio_managed_sealed_key` returns a sealed key whenever that header is present with the expected value, and the caller then takes the MinIO branch in preference to the RustFS-native one. A re-wrap that writes a RustFS-native envelope onto an object carrying MinIO-compatible headers, without clearing them, steers subsequent reads down the stale branch. Any re-wrap that changes envelope shape must neutralize the superseded keys in the same write, and cannot rely on deletion to do it. - -**Object-lock retention is enforced before the merge.** `check_object_lock_retention_update`, defined in `crates/ecstore/src/set_disk/mod.rs`, runs before `eval_metadata` is applied. Rekey inherits that decision rather than restating it: whatever that check permits for a metadata update, rekey permits; whatever it refuses, rekey counts as an exclusion. Rekey must not acquire a bypass. - -**`mod_time` is preserved unless the caller sets it.** The implementation assigns `fi.mod_time` only when `opts.mod_time` is `Some`. The re-wrap path must leave it unset, so that a rekey does not perturb lifecycle rule evaluation — an age-based expiry or transition rule reading a refreshed `mod_time` across a whole bucket would be a cross-feature regression. +- **The merge is additive; it cannot remove keys.** Overwriting a key that keeps its name is safe; a re-wrap that changes *which* keys describe the envelope leaves the old keys behind. This is a live hazard: `parse_minio_managed_sealed_key` in `rustfs/src/storage/sse.rs` selects the MinIO decrypt branch on the mere presence of the MinIO seal-algorithm header, so a RustFS-native envelope written onto MinIO-compatible headers without neutralizing them steers reads down the stale branch. Any envelope-shape change must neutralize superseded keys in the same write. +- **Object-lock retention is enforced before the merge.** `check_object_lock_retention_update` (`crates/ecstore/src/set_disk/mod.rs`) runs first; rekey inherits its decision and must not acquire a bypass. +- **`mod_time` is preserved unless the caller sets it.** The re-wrap path leaves it unset so age-based lifecycle rules are not perturbed. ## Skeleton, Ownership, And Admission -The ILM manual transition job is the structural template. `ManualTransitionJobRecord` in `crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs` already carries `job_id`, `scope_key`, `owner_id`, `lease_id` with an expiry, a state machine including an explicit `Unknown` state for a corrupt journal, `cancel_requested`, a report, and a queue snapshot. Records are persisted under dedicated metadata-bucket prefixes with a schema string and checksum, and mutated with S3 conditional writes (`if_match` for updates, `if_none_match` for creates) so that ownership transitions are compare-and-swap rather than last-write-wins. Crash recovery, cooperative cancel via `request_manual_transition_job_cancel`, and capability advertisement through `ManualTransitionJobCapabilities` in `rustfs/src/admin/handlers/system.rs` all follow from that shape. - -Ownership is scope-scoped, not cluster-scoped. Two jobs on disjoint scopes may run concurrently; two jobs on the same scope must be refused by admission. The scanner's leader lock with epoch fencing in `crates/scanner/src/scanner.rs` is the wrong granularity here because it enforces exactly one worker per cluster; it stays a reference for fencing technique only. - -The first implementation is single-node: one owner plus a lease plus recovery is sufficient for correctness. Multi-node parallel execution is a throughput optimization and is out of scope until correctness and its acceptance evidence are both in place. - -Because the job runs online, it must not contend its way into the foreground data path. The v1 posture — strict serialization plus the KMS policy layer's shared cap and the storage layer's own locks — and the reason no workload-admission mechanism is joined yet are recorded under [Implementation Status](#implementation-status-v1-sweep); when a runtime admission mechanism exists per [workload-admission-contracts.md](workload-admission-contracts.md), this job joins it alongside the scanner, heal, and decommission. - -## What Is Taken From KMS Backup, And What Is Not - -Four things transfer: - -- The durable file commit protocol in `crates/kms/src/backends/local.rs` — write, fsync the file, publish by rename or hard link, fsync the parent directory — together with its injectable `CommitStep` failpoints. -- The write-receipt ownership proof in `crates/kms/src/backup/vault_restore.rs`. Its distinction is the reusable idea: the list of intended targets proves nothing about ownership, and only a receipt recording the version a write actually landed at may authorize touching that record later; everything else is reported as never-written or not-at-written-version. Bulk rekey faces the identical problem when a concurrent writer modifies an object between the job's read and its write-back. Such an object must be counted as a conflict and skipped, never overwritten. -- The sequence guard `VaultRestoreSequence` in the same module: a small, domain-free state machine that makes phase order structural. Rekey's phases are scan, plan, apply, verify. -- The three-part dry-run report model in `crates/kms/src/backup/dry_run.rs` — blockers, conflicts, and external mismatches, with a permission predicate that requires all three to be empty — and its zero-write contract: the report is pure data with no handles and no drop-time side effects. - -The lifecycle model does not transfer, and must not be adapted. Backup restore is synchronous, one-shot, single-node, requires an empty target, has no progress surface, and requires the KMS service to be out of `Running` state; its admin layer says as much in `rustfs/src/admin/handlers/kms_backup.rs`. Its commit marker enumerates every file up front, which does not scale to object counts. Its publish primitive is no-clobber, whereas rekey rewrites existing state by definition. Forcing rekey into that four-phase protocol produces an all-or-nothing transaction over the whole scope, which is not operable at this scale. - -The job also does not belong in the KMS crate. `crates/kms/Cargo.toml` does not depend on `rustfs-ecstore` and must not: the job body is object scanning and metadata rewriting, which is ecstore and admin territory. The KMS crate supplies the re-wrap primitive only. +- Structural template: `ManualTransitionJobRecord` (`job_id`, `scope_key`, `owner_id`, `lease_id` with expiry, state machine with explicit `Unknown`, `cancel_requested`, report, queue snapshot), persisted with S3 conditional writes so ownership transitions are compare-and-swap; capability advertisement via `ManualTransitionJobCapabilities` in `rustfs/src/admin/handlers/system.rs`. +- Ownership is scope-scoped: disjoint scopes may run concurrently; same-scope jobs are refused by admission. The scanner leader lock in `crates/scanner/src/scanner.rs` is the wrong granularity (one worker per cluster) and is a fencing reference only. +- First implementation is single-node; multi-node parallelism is a throughput optimization deferred until correctness evidence exists. +- Taken from KMS backup: the durable file commit protocol in `crates/kms/src/backends/local.rs` (`CommitStep` failpoints); the write-receipt ownership proof and `VaultRestoreSequence` phase guard in `crates/kms/src/backup/vault_restore.rs` (a concurrent writer between read and write-back is a conflict-and-skip, never an overwrite); the three-part zero-write dry-run report model in `crates/kms/src/backup/dry_run.rs`. Not taken: the synchronous, empty-target, all-or-nothing restore lifecycle. The job does not belong in `crates/kms` (which must not depend on `rustfs-ecstore`); KMS supplies the primitive only. ## API Surface -This section originally required reusing the MinIO-compatible batch-job endpoints in `rustfs/src/admin/handlers/batch_job.rs` and forbade a second REST surface. The shipped v1 superseded that rule: the sweep landed on RustFS-specific endpoints (`/v3/kms/keys/rekey`, `/status`, `/cancel`), reviewed and merged with the engine. The batch-job surface parses MinIO's full job-definition format, whose semantics (per-job flags, retries, notifications) the v1 sweep does not implement — and accepting a job definition whose semantics cannot be executed is exactly what this section forbids. - -The rule that survives is about live semantics, not endpoint shape: **one operation must never have two live semantics.** Today there is one live surface (the RustFS endpoints) and one refusing stub — `KNOWN_JOB_TYPES` in `batch_job.rs` still lists `keyrotate`, and `start-job` still returns a deliberate `NotImplemented`, unknown types get `InvalidRequest`, `list-jobs` returns an empty list, and status, describe, and cancel return a no-such-job error. That `NotImplemented` remains an external promise: the batch-job `keyrotate` type must keep refusing until it either proxies to this same engine with full batch-job semantics or is removed. It must never report success while it executes nothing, and it must never grow a second, divergent rekey implementation. +- Live surface: the RustFS endpoints above. The MinIO-compatible batch-job surface (`rustfs/src/admin/handlers/batch_job.rs`) still lists `keyrotate` in `KNOWN_JOB_TYPES` and returns a deliberate `NotImplemented` from `start-job`. +- Rule: **one operation must never have two live semantics.** The batch-job `keyrotate` type must keep refusing until it proxies to this engine with full batch-job semantics or is removed; it must never report success while executing nothing. ## Completion Evidence -A job that reports success has not proven anything until no object in the scope still references the superseded key version. That evidence surface is the key usage inventory, whose typed foundation already exists in `crates/kms/src/key_impact.rs`. That module is deliberately built so a report can never claim a key is unused: it has no `in_use`, no `unreferenced`, and no `safe_to_delete` field, and instead reports which sources were consulted and how completely they could be read. It lists object envelopes and in-progress multipart uploads among its reference scopes and currently marks both as not scanned. - -Rekey must inherit that discipline. An empty result means nothing was found in the sources that were scanned, never that nothing references the key. A report that cannot state its own coverage is not completion evidence, and must not be used to authorize destroying anything. +Completion is proven only when no object in scope still references the superseded key version. The evidence surface is the key usage inventory in `crates/kms/src/key_impact.rs`, which deliberately has no `in_use` / `unreferenced` / `safe_to_delete` field and instead reports which sources were consulted and how completely. Rekey inherits that discipline: an empty result means nothing was found in the sources scanned, never that nothing references the key. ## Blockers -**Resolved by capability gating — Local rotation history.** [`rustfs/backlog#1565`](https://github.com/rustfs/backlog/issues/1565) (no rotation history in the Local backend) was a hard blocker while a sweep could run against Local: without retained superseded versions, a rekey interrupted halfway would leave every unprocessed object permanently unreadable after rotation, falsifying the partial-completion guarantee this contract is built on. The shipped resolution is not rotation history but scope: the Local backend is positioned as non-production, rotation stays rejected there, and the sweep's start endpoint refuses any backend that does not advertise `BackendCapabilities::rewrap` — so a sweep can only run where the retained-versions invariant holds by construction (Vault KV2 and Vault Transit). If Local ever gains rotation, the coupling recorded under [Reading the wrapping KEK version](#reading-the-wrapping-kek-version) still applies: rotation history and envelope version recording must land in the same change before Local may advertise `rewrap`. - -**Resolved — the execution chain is complete.** The envelope-level primitive (`KmsManager::rewrap_data_key`, `KmsManager::describe_data_key_wrapping` in `crates/kms/src/manager.rs`), the object-level adapter (`rewrap_object_encryption_metadata` in `rustfs/src/storage/sse.rs`, which reads a version's envelope, reconstructs its encryption context, re-wraps, and returns the metadata overrides), and the sweep that drives the adapter and persists through `put_object_metadata` (`rustfs/src/kms_rekey.rs`) all exist. - -**Affects acceptance, not start — still open.** Key usage inventory coverage over object envelopes: `crates/kms/src/key_impact.rs` still reports `ObjectEnvelopes` and `InProgressMultipartUploads` as not scanned, so a completed sweep's counters are evidence from that run only, not inventory-grade completion proof. KMS key list pagination, which a job enumerating keys would hit. And [`rustfs/backlog#1619`](https://github.com/rustfs/backlog/issues/1619), which decides replica propagation — until it closes, a rewrap never propagates to a replica site and each site runs its own sweep. +| Item | Status | +|---|---| +| Local rotation history (`rustfs/backlog#1565`) | Resolved by capability gating: Local stays non-production, rotation stays rejected, and the start endpoint refuses any backend without `BackendCapabilities::rewrap`. If Local ever gains rotation, envelope version recording must land in the same change. | +| Execution chain | Resolved: primitive (`rewrap_data_key`, `describe_data_key_wrapping`), object adapter (`rewrap_object_encryption_metadata`), sweep (`rustfs/src/kms_rekey.rs`). | +| Key usage inventory coverage | Open: `key_impact.rs` still reports `ObjectEnvelopes` and `InProgressMultipartUploads` as not scanned, so sweep counters are evidence from that run only. | +| KMS key list pagination | Open; a job enumerating keys would hit it. | +| Replica propagation (`rustfs/backlog#1619`) | Open; no propagation until it closes. | ## Verification Expectations -This list is the acceptance bar for the full contract, not a claim about what the v1 sweep has already demonstrated: the dry-run and checkpoint items await the features themselves (a cursor-free sweep satisfies the checkpoint-deletion clause vacuously), and per-class exclusion counting is narrowed as recorded under [Implementation Status](#implementation-status-v1-sweep). +Acceptance bar for the full contract (dry-run and checkpoint items await those features; a cursor-free sweep satisfies the checkpoint clause vacuously): -Implementation work under this contract must be able to demonstrate, at minimum: that dry run performs zero storage writes; that non-rekeyable objects are excluded and counted rather than failing the job; that an immediate second run skips every object and writes no metadata, on both a KV2-backed and a Transit-backed scope, since the two recover the wrapping version by different mechanisms; that a scope on an AWS-backed key is refused at admission rather than accepted as a job whose re-runs rewrite everything; that an envelope with no recorded version is classified by backend rather than by the bare `None`; that deleting the checkpoint changes only the skip count, not the outcome; that a killed and recovered job reaches a terminal state while every object remains readable throughout; that a concurrent writer causes a conflict-and-skip rather than an overwrite; that ETag, part layout, and storage usage are unchanged at the `xl.meta` level; that each version of a multi-version object is processed independently with its `versionId` intact; that superseded key versions still exist and still decrypt afterward; and that success, skip, exclusion, conflict, and failure counts sum to the number of work units scanned. +1. Dry run performs zero storage writes. +2. Non-rekeyable objects are excluded and counted rather than failing the job. +3. An immediate second run skips every object and writes no metadata, on both a KV2-backed and a Transit-backed scope. +4. A scope on an AWS-backed key is refused at admission. +5. An envelope with no recorded version is classified by backend, not by the bare `None`. +6. Deleting the checkpoint changes only the skip count, not the outcome. +7. A killed and recovered job reaches a terminal state while every object stays readable throughout. +8. A concurrent writer causes conflict-and-skip, not an overwrite. +9. ETag, part layout, and storage usage are unchanged at the `xl.meta` level. +10. Each version of a multi-version object is processed independently with its `versionId` intact. +11. Superseded key versions still exist and still decrypt afterward. +12. Success, skip, exclusion, conflict, and failure counts sum to the number of work units scanned. diff --git a/docs/architecture/minio-file-format-compat.md b/docs/architecture/minio-file-format-compat.md index 86ae39fa4..9b7675624 100644 --- a/docs/architecture/minio-file-format-compat.md +++ b/docs/architecture/minio-file-format-compat.md @@ -1,416 +1,148 @@ -# MinIO File-Format Interoperability — Gap Analysis & Phased Plan +# MinIO On-Disk Format Interoperability -Assesses how closely the RustFS on-disk format matches MinIO's, so that a -MinIO drive set can be read (and eventually served) by RustFS and vice versa. -This is a **plan and analysis document**. It changes no storage code. Every -claim below cites the code that backs it. +**Use this when:** deciding whether a MinIO drive set, bucket-metadata blob, or SSE object can be read or imported by a given RustFS build, or before touching any constant or codec listed under Version Anchors. +**Source of truth:** `crates/filemeta/src/filemeta.rs`, `crates/filemeta/src/filemeta/codec.rs`, `crates/ecstore/src/bucket/metadata.rs`, `crates/ecstore/src/bucket/migration.rs`, `rustfs/src/storage/sse.rs`, `rustfs/Cargo.toml` `[features]`, `.github/workflows/ci.yml`, `.github/workflows/minio-interop.yml`. -Scope: the two on-disk artifacts that matter for interop are the per-object -`xl.meta` (object metadata + inline data) and the per-bucket `.metadata.bin` -(bucket configuration blob). IAM/config layout is noted where it affects -bucket-metadata migration. +This is an interop contract, not a plan. Migration is one-way (MinIO to RustFS). Erasure-coding internals are owned by [erasure-coding.md](erasure-coding.md); this document owns the interop claim, the fixture evidence, and the out-of-scope list. -Refs rustfs/backlog#580. +## Scope Matrix By Build Variant -## Executive Summary +Build variants are the `rustfs` crate features in `rustfs/Cargo.toml`: `default`, `full`, and `rio-v2` (which enables `rustfs-ecstore/rio-v2` and pulls in `crates/rio-v2`). `rio-v2` is absent from both `default` and `full`. -- **`xl.meta`**: RustFS writes `XL_META_VERSION = 3` and reads meta_ver ≤ 3, - including legacy meta_ver 2 objects with legacy checksums. Magic `XL2 `, - erasure algorithm `rs-vandermonde` (Reed-Solomon), and HighwayHash256 bitrot - all match MinIO. `xl.meta` interop is the **strong** part of the story. -- **`.metadata.bin`**: RustFS uses the same filename, the same 4-byte - `format|version` header, the same MessagePack blob layout, and the same - per-config field encodings (XML/JSON) as MinIO's `bucketMetadata`. The - divergence is a small set of RustFS-only fields (table-bucket support, - bucket-targets meta) — not a format mismatch. -- **Migration**: RustFS already ships a one-way importer that reads a legacy - meta bucket and rewrites bucket-metadata + IAM config into the RustFS meta - bucket (`crates/ecstore/src/bucket/migration.rs`). -- **Server-side encryption**: not covered by the above. Objects MinIO wrote with SSE-S3, SSE-KMS, or SSE-C are **not readable by RustFS** in any shipped build. See [Part C](#part-c--server-side-encryption-sse) before planning a migration that includes encrypted objects. +| MinIO artifact | `default` / `full` build | `rio-v2` build | Notes | +|---|:--:|:--:|---| +| Unencrypted `xl.meta` (meta_ver 1-3, inline, multipart, versioned, delete marker) | Read | Read | Part A. Normalized to meta_ver 3 on rewrite. | +| Transitioned (tiered) `xl.meta` | Not fixture-proven | Not fixture-proven | Out of scope; see erasure-coding.md for the tolerant `transitioned-versionID` read rule. | +| `.metadata.bin` bucket config | Read and imported | Read and imported | Part B. Importer reads a `.minio.sys` layout end to end. | +| IAM config under `config/iam/` | Imported | Imported | `try_migrate_iam_config`; legacy field aliases normalized. | +| SSE-S3 / SSE-KMS objects, MinIO builtin static KMS | Fail closed, diagnosed | Read | Part C. Requires the shared master key. | +| SSE-C objects | Fail closed, diagnosed | Read | Part C. Customer key supplied per request. | +| Any SSE object, MinIO backed by KES / KMS plugin / MinKMS | Fail closed | Fail closed | Not planned; the DEK is sealed by the KES service. | +| RustFS-written drive set read by a live MinIO binary | Unsupported | Unsupported | Set-level divergence: MinIO looks for `.minio.sys`, RustFS writes `.rustfs.sys`. | +| RustFS-written SSE objects read by MinIO | Unsupported | Unsupported | Part C, reverse direction. | -For unencrypted objects the remaining work is verification breadth and closing -per-config parsing gaps, not a format rewrite. Encrypted objects are a separate, -unsolved axis (rustfs/backlog#1638). +## Version Anchors ---- +These constants are compatibility anchors. Bumping any of them requires a read-compat path for the prior value and a migration story, exactly as the meta_ver 2 to 3 read path provides. Values live in code; do not copy them elsewhere. + +| Anchor | Symbol | File | Rule | +|---|---|---|---| +| `xl.meta` magic | `XL_FILE_HEADER` | `crates/filemeta/src/filemeta.rs` | Must equal MinIO's XL2 magic. | +| Container major / minor | `XL_FILE_VERSION_MAJOR`, `XL_FILE_VERSION_MINOR` | `crates/filemeta/src/filemeta.rs` | `check_xl2_v1` (`crates/filemeta/src/filemeta/codec.rs`) rejects `major > XL_FILE_VERSION_MAJOR`. | +| Header version | `XL_HEADER_VERSION` | `crates/filemeta/src/filemeta.rs` | `decode_xl_headers` rejects `header_ver > XL_HEADER_VERSION`. | +| Metadata version | `XL_META_VERSION` | `crates/filemeta/src/filemeta.rs` | Written by `FileMeta::new`; `decode_xl_headers` rejects `meta_ver > XL_META_VERSION` (accept-older, reject-newer). | +| Bucket metadata header | `BUCKET_METADATA_FORMAT`, `BUCKET_METADATA_VERSION` | `crates/ecstore/src/bucket/metadata.rs` | Checked by `check_header`; both match MinIO's `bucketMetadataFormat` / `bucketMetadataVersion`. | +| Erasure algorithm string | `ERASURE_ALGORITHM` | `crates/ecstore/src/object_api/mod.rs` | `rs-vandermonde`; enum `ErasureAlgo` in `crates/filemeta/src/fileinfo.rs`. | +| Meta bucket names | `RUSTFS_META_BUCKET`, `MIGRATING_META_BUCKET`, `BUCKET_META_PREFIX` | `crates/ecstore/src/disk/mod.rs` | `.rustfs.sys` is the live meta bucket; `.minio.sys` is the importer source. | ## Part A — `xl.meta` Object Format -### Version support - -| Aspect | Value | Evidence | +| Aspect | Contract | Where | |---|---|---| -| Write version (`meta_ver`) | 3 | `crates/filemeta/src/filemeta.rs:54` (`XL_META_VERSION = 3`), written in `FileMeta::new` at `crates/filemeta/src/filemeta.rs:121` | -| Read versions accepted | ≤ 3 (1, 2, 3) | Decode rejects only `meta_ver > XL_META_VERSION` — see `crates/filemeta/src/filemeta/codec.rs` (`decode_xl_headers`); `load_or_convert` doc at `crates/filemeta/src/filemeta.rs:864` | -| Legacy meta_ver 2 read | Supported (with legacy checksum) | Regression fixtures `test_issue_2265_legacy_meta_v2_object_compatibility` / `test_issue_2288_legacy_xlmeta_compatibility` at `crates/filemeta/src/filemeta.rs:1130`, `:1152`; `uses_legacy_checksum` asserted at `:1174` | +| Version probe | `read_format_versions` returns `(major, minor, header_ver, meta_ver)` without a full parse | `crates/filemeta/src/filemeta/codec.rs` | +| Read compatibility | Accepts meta_ver 1-3 including legacy meta_ver 2 with legacy checksums (`uses_legacy_checksum`); `load_or_convert` normalizes on rewrite | `crates/filemeta/src/filemeta.rs`, `crates/ecstore/src/set_disk/read.rs` | +| Container layout | 8-byte header, bin-length-prefixed msgpack header block, CRC trailer, optional inline data (MinIO XL2 v1 shape) | `crates/filemeta/src/filemeta/codec.rs` | +| Erasure coding | Reed-Solomon Vandermonde, `rs-vandermonde` identifier, codec crate `rustfs-erasure-codec` | `Cargo.toml`, [erasure-coding.md](erasure-coding.md) | +| Bitrot | `HighwayHash256S` default; `HighwayHash256SLegacy` (fixed key) for older shards | `crates/ecstore/src/io_support/bitrot.rs`, `crates/ecstore/tests/legacy_bitrot_read_test.rs` | +| Inline data | Inline block after the CRC trailer; `null` / version-id keying via `data_key_for_version`; `physical_data_dir` accounting | `crates/filemeta/src/filemeta.rs`, `crates/filemeta/src/filemeta/inline_data.rs` | -RustFS is a **read-forward-compatible** consumer of MinIO's `xl.meta`: it can -parse older MinIO objects and normalizes them to meta_ver 3 on rewrite. It does -not write MinIO's older versions. - -### Container header - -| Field | RustFS value | Evidence | -|---|---|---| -| Magic | `XL2 ` (`[b'X', b'L', b'2', b' ']`) | `crates/filemeta/src/filemeta.rs:46` | -| File version major / minor | 1 / 3 | `crates/filemeta/src/filemeta.rs:51-52` | -| Header version | 3 | `crates/filemeta/src/filemeta.rs:53` | -| Magic + version check (decode entry) | `check_xl2_v1` validates magic and rejects `major > 1` | `crates/filemeta/src/filemeta/codec.rs:45-61` | -| Version-only probe (no full parse) | `read_format_versions` returns `(major, minor, header_ver, meta_ver)` | `crates/filemeta/src/filemeta/codec.rs:30-43` | - -The layout after the 8-byte header is `bin-length-prefixed msgpack header block` -followed by a CRC trailer and optional inline data — matching MinIO's XL2 v1 -container. - -### Erasure coding - -| Aspect | Value | Evidence | -|---|---|---| -| Algorithm enum | `ErasureAlgo::ReedSolomon = 1` | `crates/filemeta/src/fileinfo.rs:83-106` | -| Algorithm string | `rs-vandermonde` | `crates/filemeta/src/fileinfo.rs:31` (`ERASURE_ALGORITHM`); also `crates/ecstore/src/object_api/mod.rs:52` | -| Codec crate | `rustfs-erasure-codec` (Reed-Solomon, SIMD) | `Cargo.toml:277` | - -Same Reed-Solomon Vandermonde scheme and identifier string as MinIO. - -### Bitrot / shard integrity - -| Aspect | Value | Evidence | -|---|---|---| -| Default hash | `HashAlgorithm::HighwayHash256S` | Bitrot read/write paths in `crates/ecstore/src/io_support/bitrot.rs` (e.g. `:564`, `:767`) | -| Legacy variant | `HighwayHash256SLegacy` (fixed key) for old objects | referenced from `rustfs_utils::HashAlgorithm` (imported at `crates/ecstore/src/io_support/bitrot.rs:26`) | -| HighwayHash crate | `highway` 1.3.0 | `Cargo.toml:252` | -| Legacy bitrot read coverage | dedicated test | `crates/ecstore/tests/legacy_bitrot_read_test.rs` | - -MinIO uses HighwayHash256 for bitrot; RustFS's default `HighwayHash256S` is -compatible, with a legacy-key variant retained for older shards. - -### Inline data - -Small objects are inlined into the `xl.meta` container after the CRC trailer -rather than written as a separate `part.1`. Handling lives in -`crates/filemeta/src/filemeta/inline_data.rs` (e.g. `physical_data_dir` and the -shared-data-dir accounting), and the inline block is appended/consumed by the -codec in `crates/filemeta/src/filemeta/codec.rs`. This mirrors MinIO's inline -data feature and the `null`/version-id keying used for the inline map -(`data_key_for_version` at `crates/filemeta/src/filemeta.rs:69`, legacy key at -`:77`). - -### `xl.meta` interop verdict - -| Item | Done | Partial | Todo | -|---|:--:|:--:|:--:| -| Read MinIO meta_ver ≤ 3 | ✅ | | | -| Legacy meta_ver 2 + legacy checksum read | ✅ | | | -| XL2 container magic/version parity | ✅ | | | -| Reed-Solomon `rs-vandermonde` parity | ✅ | | | -| HighwayHash256 bitrot parity | ✅ | | | -| Inline data parity | ✅ | | | -| Broad fixture corpus from real MinIO writers | | ⚠️ | | -| Write-back parity for round-trip (RustFS→MinIO read) | | ⚠️ | | - -The two ⚠️ items are verification breadth, not known incompatibilities: the -current fixtures are targeted regressions (issues #2265, #2288), and there is no -CI job proving a MinIO binary can re-read a RustFS-written `xl.meta`. - ---- +MinIO stores an inlined object body as `[HighwayHash256 (32 B)][body]`. Feeding the raw inline shard through RustFS's `BitrotReader` with `HighwayHash256S` verifies the checksum and yields the exact payload; the bitrot prefix is not a format incompatibility. ## Part B — Bucket Metadata (`.metadata.bin`) -### On-disk layout - -| Aspect | RustFS value | Evidence | +| Aspect | Contract | Where | |---|---|---| -| Meta bucket | `.rustfs.sys` | `crates/ecstore/src/disk/mod.rs:29` (`RUSTFS_META_BUCKET`) | -| Bucket-config prefix | `buckets` | `crates/ecstore/src/disk/mod.rs:34` (`BUCKET_META_PREFIX`) | -| Blob file | `.metadata.bin` | `crates/ecstore/src/bucket/metadata.rs:227` (`BUCKET_METADATA_FILE`) | -| Full path | `buckets/{bucket}/.metadata.bin` | `crates/ecstore/src/bucket/metadata.rs:415-416` (`save_file_path`) | -| Header | `format: u16 LE` + `version: u16 LE`, both `= 1` | `crates/ecstore/src/bucket/metadata.rs:228-229`, checked in `check_header` at `:595-614` | -| Body | MessagePack-encoded `BucketMetadata` | `marshal_msg`/`unmarshal` at `crates/ecstore/src/bucket/metadata.rs:582-593`; read strips the 4-byte header (`unmarshal(&data[4..])` at `:1079`) | +| Path | `buckets/{bucket}/.metadata.bin` under the meta bucket (`BUCKET_METADATA_FILE`, `save_file_path`) | `crates/ecstore/src/bucket/metadata.rs` | +| Header | 4 bytes: `format: u16 LE` + `version: u16 LE`, stripped before `unmarshal` | `check_header` in `crates/ecstore/src/bucket/metadata.rs` | +| Body | MessagePack-encoded `BucketMetadata`; field names map one-to-one onto MinIO's `bucketMetadata` (PascalCase on the wire) | `BucketMetadata` in `crates/ecstore/src/bucket/metadata.rs` | +| Per-config encoding | XML for S3-XML configs, JSON for policy / quota / targets / ACL; the per-config filename constants (`policy.json`, `lifecycle.xml`, ...) are `update_config` field-selector keys, not separate files | `update_config`, `parse_all_configs` in `crates/ecstore/src/bucket/metadata.rs` | +| RustFS-only fields | `bucket_targets_config_meta_json`, `table_bucket_config_json`; a MinIO reader ignores unknown msgpack fields | `crates/ecstore/src/bucket/metadata.rs` | +| Partial interop | `bucket_targets` meta side-channel is RustFS-specific; `bucket_acl` round-trips as a blob but only canned ACLs are enforced (see [minio-rustfs-router-compatibility.md](minio-rustfs-router-compatibility.md)) | | -This is the same design as MinIO's bucket metadata: a single -`.minio.sys/buckets//.metadata.bin` blob with a 4-byte -`bucketMetadataFormat|bucketMetadataVersion` header and a msgpack body. The -filename, header shape, and format/version values (`1`/`1`) all match. The -`BucketMetadata` field names correspond one-to-one to MinIO's `bucketMetadata` -struct (`policyConfigJSON`, `lifecycleConfigXML`, `objectLockConfigXML`, …). +### Importer -> Correction to a common misconception: modern MinIO does **not** store each -> bucket config as a separate loose `versioning.json` / `lifecycle.json` file — -> it embeds them in the same `.metadata.bin` blob, with XML for the S3-XML -> configs and JSON for policy/quota/targets. The per-config filename constants -> in RustFS (`policy.json`, `lifecycle.xml`, …) are the **keys used by -> `update_config`** to select a field, not separate on-disk files. +`crates/ecstore/src/bucket/migration.rs` is a one-way, idempotent importer from a `MIGRATING_META_BUCKET` (`.minio.sys`) layout into `.rustfs.sys`, run at startup from `rustfs/src/startup_bucket_metadata.rs`: -### Interop matrix (backlog#580 items) - -Field/constant references are in `crates/ecstore/src/bucket/metadata.rs`. -"Encoding" is the payload RustFS stores in that field and must match MinIO's for -byte-level interop. Getter functions live in -`crates/ecstore/src/bucket/metadata_sys.rs`. - -| Config item | RustFS field / constant | Encoding | MinIO field | Status | -|---|---|---|---|---| -| versioning | `versioning_config_xml` / `BUCKET_VERSIONING_CONFIG` = `versioning.xml` | XML | versioningConfigXML | Done | -| quota | `quota_config_json` / `BUCKET_QUOTA_CONFIG_FILE` = `quota.json` | JSON | quotaConfigJSON | Done | -| object_lock | `object_lock_config_xml` / `OBJECT_LOCK_CONFIG` = `object-lock.xml` | XML | objectLockConfigXML | Done | -| replication | `replication_config_xml` / `BUCKET_REPLICATION_CONFIG` = `replication.xml` | XML | replicationConfigXML | Done | -| policy | `policy_config_json` / `BUCKET_POLICY_CONFIG` = `policy.json` | JSON | policyConfigJSON | Done | -| lifecycle | `lifecycle_config_xml` / `BUCKET_LIFECYCLE_CONFIG` = `lifecycle.xml` | XML | lifecycleConfigXML | Done | -| tagging | `tagging_config_xml` / `BUCKET_TAGGING_CONFIG` = `tagging.xml` | XML | taggingConfigXML | Done | -| bucket_targets | `bucket_targets_config_json` + `bucket_targets_config_meta_json` / `BUCKET_TARGETS_FILE` = `bucket-targets.json` | JSON | bucketTargetsConfigJSON (+ meta variant) | Partial | -| notification | `notification_config_xml` / `BUCKET_NOTIFICATION_CONFIG` = `notification.xml` | XML | notificationConfigXML | Done | -| encryption | `encryption_config_xml` / `BUCKET_SSECONFIG` = `bucket-encryption.xml` | XML | encryptionConfigXML | Done | -| cors | `cors_config_xml` / `BUCKET_CORS_CONFIG` = `cors.xml` | XML | corsConfigXML | Done | -| public_access | `public_access_block_config_xml` / `BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG` = `public-access-block.xml` | XML | publicAccessBlockConfigXML | Done | -| bucket_acl | `bucket_acl_config_json` / `BUCKET_ACL_CONFIG` = `bucket-acl.json` | JSON | bucketACLConfigJSON | Partial | - -Field definitions: `crates/ecstore/src/bucket/metadata.rs:274-336`. Constants: -`:227-247`. `update_config` field routing: `:678-761`. `parse_all_configs` is -invoked on load (`load_bucket_metadata_parse` at `:1043`). - -Notes on the two "Partial" rows: - -- **bucket_targets** — RustFS carries an extra `bucket_targets_config_meta_json` - field (`:288`) beyond MinIO's single targets blob. The primary - `bucket-targets.json` payload is interoperable; the meta side-channel is - RustFS-specific and a MinIO reader would ignore it. ACL enforcement itself is - bounded (S3 `PutBucketAcl`/`PutObjectAcl` accept canned ACLs only — see - [minio-rustfs-router-compatibility.md](minio-rustfs-router-compatibility.md)). -- **bucket_acl** — stored and round-tripped in the blob, but ACL grant - semantics are intentionally limited at the S3 layer. - -RustFS also defines fields with no interop requirement from backlog#580 but -worth noting so a migration tool does not choke on them: `logging_config_xml`, -`website_config_xml`, `accelerate_config_xml`, `request_payment_config_xml` -(`:242-245`), and the RustFS-only `table_bucket_config_json` -(`BUCKET_TABLE_CONFIG` = `table-bucket.json`, `:248`). A MinIO reader that does -not know `table_bucket_config_json` will ignore the unknown msgpack field. - -### Old-RustFS → new-RustFS migration - -RustFS ships a one-way importer that reads a legacy meta bucket -(`MIGRATING_META_BUCKET`) and rewrites both bucket metadata and IAM config into -the current RustFS meta bucket, skipping entries that already exist -(idempotent). See `crates/ecstore/src/bucket/migration.rs`: - -- `try_migrate_bucket_metadata` copies `buckets/{bucket}/.metadata.bin` and the - replication resync blob for each bucket (`crates/ecstore/src/bucket/migration.rs:193`). -- `try_migrate_iam_config` walks `config/iam/` and normalizes legacy IAM - records — legacy timestamp fields (`update_at` → `updatedAt`) and legacy - policy-mapping field aliases (`policies` → `policy`) are rewritten - (`normalize_iam_config_blob` at `:97`; regression test at `:428`). -- Bucket resync metadata is re-encoded through `ReplicationMigrationBridge` - (`normalize_bucket_meta_blob` at `:178`). - -This importer is the practical basis for a MinIO → RustFS bucket-metadata -migration: because the blob layout and field encodings already match, the -missing piece is a source adapter that points the importer at a MinIO -`.minio.sys` layout rather than the RustFS legacy layout. - -### Bucket-metadata interop verdict - -| Item | Done | Partial | Todo | -|---|:--:|:--:|:--:| -| `.metadata.bin` filename + header + msgpack layout parity | ✅ | | | -| Per-config field encodings (XML/JSON) match MinIO | ✅ | | | -| versioning/quota/object_lock/replication/policy/lifecycle/tagging/notification/encryption/cors/public_access round-trip | ✅ | | | -| bucket_targets primary blob | ✅ | | | -| bucket_targets meta side-channel + ACL grant semantics | | ⚠️ | | -| Old-RustFS → new-RustFS importer | ✅ | | | -| MinIO `.minio.sys` source adapter for the importer | | | ❌ | -| CI proof a MinIO-written `.metadata.bin` loads unchanged | | | ❌ | - ---- +| Function | Imports | +|---|---| +| `try_migrate_bucket_metadata` | `buckets/{bucket}/.metadata.bin` plus the replication resync blob (`normalize_bucket_meta_blob` via `ReplicationMigrationBridge`) | +| `try_migrate_iam_config` | `config/iam/` records; `normalize_iam_config_blob` rewrites legacy timestamp and policy-mapping aliases | ## Part C — Server-Side Encryption (SSE) -Reading MinIO-written SSE objects is implemented, with a deliberate build boundary. The read path lives behind the `rio-v2` feature and is a **special-purpose migration capability**: it is not compiled into released binaries or container images, and there is no short-term plan to promote it into default builds. A default build fails such reads closed with a diagnosed error (see "How default builds fail" below); a `rio-v2` build reads them, within the scenario matrix below. The read-path work was tracked in rustfs/backlog#1638 (landed across rustfs/rustfs#6191, #6784, #6785). +The `xl.meta` around a MinIO SSE object parses in every build, so such objects list, HEAD, and report plausible sizes; only payload readability depends on the build. KMS wire protocols (AWS `awsJson1_1` client in `crates/kms/src/backends/aws.rs`, MinIO KES) are non-targets. -### Scope boundary: KMS wire protocols and the production gate - -This document covers MinIO on-disk metadata and object-encryption seams only. The **AWS KMS wire protocol** and the **MinIO KES wire protocol** are explicit non-targets: RustFS's AWS backend uses the AWS SDK's `awsJson1_1` client path (`crates/kms/src/backends/aws.rs:830`), while KES compatibility is outside this interop work. Those ecosystem evaluations remain separate work in the [#1562 Production Ready exit gate](https://github.com/rustfs/backlog/issues/1562), whose compatibility criterion covers MinIO/RustFS SSE data and rolling upgrades. Closing #1638 does not by itself close that gate. - -Note the asymmetry with Parts A and B: the `xl.meta` around a MinIO SSE object parses fine, so such objects list, HEAD, and report plausible sizes. Only payload readability depends on the build and the scenario. - -### What can and cannot be migrated - -| Object class | Default build | `rio-v2` build | Notes | +| Object class | `default` / `full` | `rio-v2` | Requirement | |---|:--:|:--:|---| -| Unencrypted objects | ✅ | ✅ | Parts A and B apply. | -| Bucket metadata, IAM config | ✅ | ✅ | Via the importer, once a `.minio.sys` source adapter exists (see Part B). | -| Bucket-level default-encryption *configuration* | ✅ | ✅ | The `encryption` config blob round-trips as a blob; it does not make existing ciphertext readable. | -| SSE-S3 / SSE-KMS, MinIO builtin static KMS (`MINIO_KMS_SECRET_KEY`), single- and multipart | ❌ diagnosed | ✅ | Requires `RUSTFS_SSE_S3_MASTER_KEY` set to the same 32-byte key material as MinIO's static secret. Proven against real MinIO fixtures (rustfs/rustfs#6191). | -| SSE-C, MinIO-written | ❌ diagnosed | ✅ | Detection via MinIO's sealed-key slot; the customer key is proven by the AEAD unseal, since MinIO stores no key MD5 (rustfs/rustfs#6785). | -| Any SSE, MinIO backed by KES / KMS plugin / MinKMS | ❌ | ❌ **not planned** | The wrapped DEK is sealed by the KES service itself; it is not a Vault/Transit ciphertext RustFS could be pointed at. Re-encrypt on the MinIO side before migrating. | -| Objects sealed with legacy `DARE-SHA256` (`InsecureSealAlgorithm`) | ❌ | ❌ out of scope | Pre-DAREv2-HMAC MinIO; `parse_minio_managed_sealed_key` rejects the algorithm and the read fails closed. | -| RustFS-written SSE objects read back by MinIO | ❌ | ❌ | See "Reverse direction". | +| SSE-S3 / SSE-KMS, MinIO builtin static KMS, single- and multipart | Fail closed, diagnosed | Read | `RUSTFS_SSE_S3_MASTER_KEY` (base64, 32 bytes) equal to the source MinIO's static secret. | +| SSE-C, MinIO-written | Fail closed, diagnosed | Read | Client supplies the customer key per request; MinIO stores no key MD5, so the AEAD unseal is the key proof. | +| Any SSE, MinIO backed by KES / KMS plugin / MinKMS | Fail closed | Fail closed | Not planned. Re-encrypt or decrypt on the MinIO side first. | +| Bucket default-encryption *configuration* | Round-trips | Round-trips | A config blob; it does not make existing ciphertext readable. | -### The seams, and where they closed +### Seams -The cryptographic primitives were never the gap — RustFS implements the same DARE V2 stream format, object-key derivation, and sealing. Three seams above the cryptography rejected MinIO-written objects; all three are closed in `rio-v2` builds. +The cryptography (DARE v2 stream format, object-key derivation, sealing) was never the gap. Three metadata seams above it rejected MinIO objects; all are closed in `rio-v2` builds. Symbols are in `rustfs/src/storage/sse.rs` unless noted. -| # | Seam | Resolution | -|---|---|---| -| 1 | Managed-SSE detection required the *persisted* public `x-amz-server-side-encryption` key, which MinIO synthesizes at response time and never stores. | Closed by rustfs/rustfs#6191: `infer_minio_managed_sse_type` infers the scheme from which MinIO sealed-key slot is present (the slot also selects the sealing-key domain, so a wrong inference cannot silently derive a wrong key). Inference from the KMS key id would misclassify — MinIO writes `-S3-Kms-Key-Id` on SSE-S3 objects too. | -| 2 | MinIO's wrapped-DEK ciphertext was not accepted by any envelope parser. | Closed by rustfs/rustfs#6191: `decrypt_minio_kms_data_key` implements MinIO's builtin-KMS sealing (`sealingKey = HMAC-SHA256(master, iv)`), accepting both the raw `sealed‖iv‖nonce` layout and the legacy `{"aead": ...}` JSON. Routing is by the data key's own byte shape — RustFS's strict JSON envelopes are recognized positively, everything else goes to the MinIO decoder — because slot names cannot distinguish the writer. `LocalSseDekEnvelope` keeps `deny_unknown_fields`. | -| 3 | SSE-C detection keyed on the stored customer-algorithm header, which MinIO also never persists, and the early key check demanded a stored key MD5 MinIO does not write. | Closed by rustfs/rustfs#6785: `stored_ssec_metadata` also accepts MinIO's SSE-C sealed-key slot (rio-v2 builds only), and `verify_ssec_key_match` tolerates a missing stored MD5 for exactly that shape — the AEAD unseal remains the key proof, and a wrong key still fails there. | - -Two further single-part defects were fixed on the way (both rustfs/rustfs#6191 follow-ups): multipart classification now trusts MinIO's own `X-Minio-Internal-Encrypted-Multipart` marker instead of an ETag-length heuristic (MinIO stores *encrypted* ETags, so every single-part SSE object mis-classified as multipart), and single-part plaintext sizes are recovered by DARE reverse-size arithmetic (`dare_v2_decrypted_size`) since MinIO records an explicit size only for multipart uploads. +| Seam | Resolution | +|---|---| +| Managed-SSE detection required the persisted public `x-amz-server-side-encryption` key, which MinIO synthesizes at response time | `infer_minio_managed_sse_type` infers the scheme from which MinIO sealed-key slot is present; the slot also selects the sealing-key domain, so a wrong inference cannot derive a wrong key. | +| MinIO's wrapped-DEK ciphertext was accepted by no envelope parser | `decrypt_minio_kms_data_key` implements MinIO's builtin-KMS sealing for both the raw `sealed‖iv‖nonce` layout and the legacy `{"aead": ...}` JSON. Routing is by byte shape: `LocalSseDekEnvelope` (`deny_unknown_fields`) is recognized positively, everything else goes to the MinIO decoder. | +| SSE-C detection keyed on the stored customer-algorithm header, which MinIO also never persists, and demanded a stored key MD5 | `stored_ssec_metadata` accepts MinIO's SSE-C sealed-key slot (`rio-v2` only); `verify_ssec_key_match` tolerates a missing stored MD5 for exactly that shape. | +| Multipart classification used an ETag-length heuristic (MinIO stores encrypted ETags) | Trusts MinIO's own `X-Minio-Internal-Encrypted-Multipart` marker (`crates/utils/src/http/header_compat.rs`). | ### How default builds fail -The read fails closed: ciphertext is never served as plaintext. `is_object_encryption_marker` matches the whole `x-minio-internal-server-side-encryption-` prefix, so `ObjectInfo::is_encrypted()` is true for these objects, and the read plan refuses to construct a reader without decryption material. Since rustfs/rustfs#6784 the refusal is diagnosed: the resolver raises a typed error naming the condition — in default builds it points at the MinIO-compatible sealed format and the `rio-v2` read path it would require — and it surfaces as S3 `InvalidObjectState` (non-retryable) instead of the former undiagnosed 500 `InternalError`. List and HEAD still succeed, because `xl.meta` parses normally. - -### What a `rio-v2` migration build needs - -- A binary built with `--features rio-v2`. The feature is deliberately absent from `default` and `full` in `rustfs/Cargo.toml`; released binaries and images never include it. -- For SSE-S3/SSE-KMS objects: `RUSTFS_SSE_S3_MASTER_KEY` (base64, 32 bytes) set to the same key material as the source MinIO's `MINIO_KMS_SECRET_KEY`. For SSE-C objects: nothing server-side — the client supplies the customer key per request, as on MinIO. -- The interop harness is the evidence chain: `rustfs/src/storage/minio_generated_read_test.rs` (`#[ignore]` reader tests over real MinIO-generated fixtures, run with `--features rio-v2`), the fixture lab under `crates/rio-v2/tests/minio_fixture_lab/`, and the `minio-interop` workflow. The SSE-C lane of that harness (customer-key handout from a fixture capture to the reader test) is not wired yet; SSE-C coverage currently lives in the unit suite, which builds the MinIO shape with the same sealing primitives the fixture suite proved byte-compatible. - -Known unverified edge: MinIO seals ETags on SSE objects (`SealETag`); RustFS does not unseal them, so ETag display and `If-Match` semantics on migrated SSE objects are not guaranteed to match MinIO's. +`is_object_encryption_marker` (`crates/utils/src/http/header_compat.rs`) matches the whole `x-minio-internal-server-side-encryption-` prefix, so `ObjectInfo::is_encrypted()` is true and the read plan refuses to construct a reader without decryption material. The refusal is a typed error that names the MinIO-compatible sealed format and the `rio-v2` read path it requires, surfaced as S3 `InvalidObjectState` (non-retryable). Ciphertext is never served as plaintext. ### Reverse direction -Migrating back is also unsupported. Under `rio-v2` RustFS writes its own DEK envelope into MinIO's sealed-key metadata slots and labels it with MinIO's seal algorithm (`rustfs/src/storage/sse.rs:1830-1852`), so the metadata is MinIO-shaped while the key bytes are not MinIO-openable. Default builds do not populate those slots at all (`rustfs/src/storage/sse.rs:1796-1798`). Treat RustFS-written SSE objects as readable only by RustFS. +Under `rio-v2` RustFS writes its own DEK envelope into MinIO's sealed-key metadata slots labelled with MinIO's seal algorithm, so the metadata is MinIO-shaped while the key bytes are not MinIO-openable. Default builds do not populate those slots. Treat RustFS-written SSE objects as readable only by RustFS. Known unverified edge: MinIO seals ETags on SSE objects; RustFS does not unseal them, so ETag display and `If-Match` on migrated SSE objects are not guaranteed to match MinIO. -### Migration options +### Migration options for encrypted objects -- For static-KMS MinIO sources: run the migration through a `rio-v2` build with the shared master key (see above), either serving reads in place or copying objects out into a default-build cluster (the copy re-encrypts under RustFS's own KMS). -- For KES/MinKMS-backed sources, or when a special-purpose build is not wanted: decrypt on the MinIO side first — rewrite the affected objects as plaintext, or copy them out through MinIO's S3 endpoint, which decrypts on read — and let RustFS apply its own encryption on ingest. -- Leave encrypted objects on MinIO and migrate only unencrypted data. +1. Static-KMS source: run the migration through a `rio-v2` build with the shared master key, serving in place or copying into a default-build cluster (the copy re-encrypts under RustFS's own KMS). +2. KES / MinKMS source, or no special-purpose build wanted: decrypt on the MinIO side (rewrite as plaintext, or copy out through MinIO's S3 endpoint) and let RustFS encrypt on ingest. +3. Leave encrypted objects on MinIO and migrate only unencrypted data. -Inventory the source first — bucket default-encryption settings mean objects can be encrypted without the uploader having asked for it, so "we never set SSE headers" is not sufficient evidence that a bucket has no encrypted objects. +Inventory the source first: bucket default encryption means objects can be encrypted without the uploader asking, so "we never set SSE headers" is not evidence that a bucket has no encrypted objects. -### SSE interop verdict +## rio-v2 variant lifecycle -| Item | Done | Partial | Todo | -|---|:--:|:--:|:--:| -| DARE V2 stream format parity | ✅ | | | -| Object-key derivation / sealing parity | ✅ | | | -| Managed-SSE detection accepts MinIO-written metadata (`rio-v2`) | ✅ | | | -| MinIO builtin-KMS wrapped-DEK parser (raw + legacy JSON) | ✅ | | | -| SSE-C detection accepts MinIO-written metadata (`rio-v2`) | ✅ | | | -| Read MinIO-written SSE-S3 / SSE-KMS end to end, single- and multipart | ✅ | | | -| Read MinIO-written SSE-C end to end | | ⚠️ unit-proven; fixture-lab lane unwired | | -| Migrated-object sealed-ETag semantics | | | ❌ unverified | -| KES / MinKMS / legacy `DARE-SHA256` sources | | | ❌ not planned | -| RustFS-written SSE objects readable by MinIO | | | ❌ | -| CI proof of SSE read parity | | ⚠️ `minio-interop` workflow; nightly once re-enabled | | +`rio-v2` is a dormant, special-purpose migration variant tracked under `rustfs/backlog#1835`. ---- +| Fact | Value | +|---|---| +| Shipping status | Ships in no default build: absent from `default` and `full` in `rustfs/Cargo.toml`; released binaries and container images never include it. Enable with `--features rio-v2`. | +| Pull-request coverage | `test-and-lint-rio-v2` in `.github/workflows/ci.yml`: clippy plus `cargo nextest` for `rustfs` and `rustfs-ecstore` with `--features rio-v2`. This is the cfg-seam guard; it keeps the feature compiling and its unit suite green on every pull request. | +| Full-suite lane | `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` in `ci.yml` run only on the weekly schedule and manual dispatch (`cache-warm.yml` keeps the `ci-feat-rio` cache warm so the scheduled build fits its timeout). | +| Interop evidence | `.github/workflows/minio-interop.yml` (nightly plus manual) regenerates real MinIO backend trees via `crates/rio-v2/tests/minio_fixture_lab/` and runs the `#[ignore]` reader tests in `rustfs/src/storage/minio_generated_read_test.rs` with `--features rio-v2`. Its freshness is tracked in `.github/scheduled-validations.json`. | +| Promote-or-delete condition | The variant stays dormant until one of two things happens. **Promote**: a release commits to shipping MinIO SSE migration as a supported capability; then `rio-v2` joins `default`/`full`, the scheduled lanes run on every pull request, and this section is rewritten. **Delete**: no release commits to it and the scheduled lanes are not kept green; then the feature flag, `crates/rio-v2`, the cfg seams in `rustfs/src/storage/sse.rs`, the three `ci.yml` jobs, the `cache-warm.yml` warm step, `minio-interop.yml`, and its `scheduled-validations.json` entry are removed in one change. Either outcome must update `ARCHITECTURE.md` and the `ci.yml` job comments that cite this section. | -## Phased Plan +## Fixture Evidence -The format is already close; the plan is verification, a source adapter, and -closing the two partial encodings — not a rewrite. +Fixtures were captured from a real MinIO single-drive instance and live under `crates/filemeta/tests/fixtures/minio/` and `crates/ecstore/tests/fixtures/minio/`. These tests run in the normal `cargo test` / nextest lanes. -### Phase 1 — Read parity, proven (verification) +| Test | File | Proves | +|---|---|---| +| `parses_real_minio_object_xlmeta` | `crates/filemeta/src/filemeta.rs` | Inline, two-version plus delete-marker, and multipart `xl.meta` parse to the expected `FileInfo`. | +| `parses_real_minio_bucket_metadata_blob_without_loss` | `crates/ecstore/src/bucket/metadata.rs` | The msgpack blob decodes via MinIO's field names and `parse_all_configs` loads every config in the corpus, including MinIO's lifecycle `` and replication `DeleteMarkerReplication` / `ExistingObjectReplication` extensions. | +| `reads_minio_inline_bucket_metadata_via_bitrot` | `crates/ecstore/src/bucket/metadata.rs` | The inline shard's HighwayHash prefix verifies under `HighwayHash256S` and yields the exact `.metadata.bin` blob. | +| `migrates_real_minio_bucket_metadata_end_to_end` | `crates/ecstore/src/bucket/migration.rs` | A real `.metadata.bin` seeded under a `.minio.sys` layout is imported by `try_migrate_bucket_metadata` into `.rustfs.sys` byte-identical, through the object layer on a 4-drive `ECStore`. | +| `test_issue_2265_legacy_meta_v2_object_compatibility`, `test_issue_2288_legacy_xlmeta_compatibility` | `crates/filemeta/src/filemeta.rs` | Legacy meta_ver 2 objects with legacy checksums still read. | +| `minio_generated_read_test.rs` (`#[ignore]`, `rio-v2`) | `rustfs/src/storage/minio_generated_read_test.rs` | Byte-identical plaintext reconstruction of MinIO SSE-S3 / SSE-KMS fixtures; driven by `minio-interop.yml`. | -- Add a MinIO-writer fixture corpus for `xl.meta` (inline + multipart + - versioned + delete-marker + transitioned) and assert RustFS parses each to a - `FileInfo` equivalent to MinIO's, alongside the existing issue #2265 / #2288 - fixtures in `crates/filemeta/src/filemeta.rs`. -- Add a fixture `.metadata.bin` written by MinIO and assert - `BucketMetadata::unmarshal` + `parse_all_configs` load every field without - loss (`crates/ecstore/src/bucket/metadata.rs`). -- Exit criterion: a CI job that fails if a real MinIO-written object or bucket - blob cannot be read. +Not fixture-proven: transitioned `xl.meta`; CORS, public-access-block, and bucket-ACL configs (the SNSD corpus did not exercise them); bucket-targets credentials (MinIO stores them KMS-encrypted); the SSE-C fixture-lab lane (customer-key handout is not wired; SSE-C coverage is unit-level). -#### Phase 1 status — first fixtures landed (verified 2026-07-07) +## Out Of Scope -A real MinIO `RELEASE.2025-07-23` single-drive instance wrote a bucket with -versioning, object-lock (GOVERNANCE default), lifecycle, tagging, quota, and a -public-download policy, plus inline / versioned / multipart objects. The on-disk -`xl.meta` blobs are captured as hex fixtures -(`crates/filemeta/tests/fixtures/minio/`, `crates/ecstore/tests/fixtures/minio/`). - -Proven by regression tests: - -- **Object `xl.meta` read parity** — `parses_real_minio_object_xlmeta` - (`crates/filemeta/src/filemeta.rs`): small inline, two-object-version + delete - marker, and multipart objects all parse to the expected `FileInfo`. -- **Bucket-metadata parse parity** — `parses_real_minio_bucket_metadata_blob_without_loss` - (`crates/ecstore/src/bucket/metadata.rs`): the msgpack blob decodes via the - PascalCase MinIO field names, and `parse_all_configs` loads **all ten** config - types present in the corpus without loss — policy, lifecycle (**including - MinIO's `` extension**), object-lock, versioning, tagging, - quota, notification, encryption (SSE-S3), and replication (**including the - `DeleteMarkerReplication` / `ExistingObjectReplication` MinIO extensions**). -- **Inline bucket-metadata read parity** — `reads_minio_inline_bucket_metadata_via_bitrot` - (`crates/ecstore/src/bucket/metadata.rs`): MinIO stores an inlined object body - as `[HighwayHash256 (32B)][body]`. The "`inline_data` 前缀不同" that weisd - raised on 2026-03-06 is exactly that bitrot prefix — **not** a format - incompatibility. Feeding the raw inline shard through RustFS's `BitrotReader` - with the default `HighwayHash256S` verifies the checksum (confirming RustFS's - hash matches MinIO's) and yields the exact `.metadata.bin` blob, which then - parses. So the object-layer inline read is compatible; the earlier "extract - `fi.data` directly" concern was reading the shard before the bitrot layer - strips its prefix. -- **End-to-end migration** — `migrates_real_minio_bucket_metadata_end_to_end` - (`crates/ecstore/src/bucket/migration.rs`): on a throwaway 4-drive local - `ECStore`, a real MinIO `.metadata.bin` seeded under a `.minio.sys` layout is - migrated by `try_migrate_bucket_metadata` into `.rustfs.sys`, and the migrated - blob carries every config (policy / lifecycle / object-lock / versioning / - tagging / quota / notification / encryption / replication) byte-identical to - the source. This exercises the Phase 2 source adapter - (`MIGRATING_META_BUCKET = ".minio.sys"`) end-to-end through the object layer — - proven, not just present. - -Still to broaden: transitioned `xl.meta`; CORS, public-access-block, and bucket -ACL configs (the SNSD test binary/`mc` did not expose these); and bucket-targets -credentials, which MinIO stores KMS-encrypted (a documented partial). These run -as ordinary crate tests, so they already execute in the normal `cargo -test`/nextest CI jobs. - -### Phase 2 — MinIO source adapter for migration - -- Generalize the importer in `crates/ecstore/src/bucket/migration.rs` so the - source can be a MinIO `.minio.sys/buckets//.metadata.bin` layout, not - only the RustFS legacy meta bucket. Because the blob format matches, this is - mostly source-path plumbing plus IAM record normalization reuse. -- Exit criterion: importing a MinIO backup reproduces all backlog#580 - bucket-config items with byte-identical config payloads. - -### Phase 3 — Close the two partial encodings - -- bucket_targets: document/normalize the RustFS-only - `bucket_targets_config_meta_json` so a round-trip through MinIO and back does - not silently drop it; or fold its content into a MinIO-compatible - representation. -- bucket_acl: decide whether ACL grant semantics beyond canned ACLs are in - scope; if not, keep the blob round-trippable but document the enforcement - limit (already reflected in the router compatibility matrix). - -### Phase 4 — Round-trip / write-back parity (non-goal for migration) - -Proving a MinIO binary can re-read a *RustFS-written drive set* (the reverse -direction) is **out of scope for the migration use case**, which is one-way -MinIO → RustFS: - -- RustFS's meta bucket is `.rustfs.sys` (`crates/ecstore/src/disk/mod.rs:29`); - MinIO looks for `.minio.sys`. A MinIO binary pointed at a RustFS drive set - does not find `format.json` or bucket configs and refuses the set — this is a - set-level divergence, not an object-format one. -- The object-level `xl.meta` format *does* match (proven above), so the reverse - direction is limited by drive-set discovery, not by per-object encoding. -- The supported flow is one-way: `try_migrate_bucket_metadata` / - `try_migrate_iam_config` / `format.json` migration import a MinIO layout into - RustFS. There is no requirement to keep a live MinIO able to serve - RustFS-written drives. - -If a true bidirectional round-trip is ever needed, it would require RustFS to -optionally write the `.minio.sys` set layout — a separate feature, not part of -the interop/migration story tracked here. - ---- +- A live MinIO binary serving a RustFS-written drive set (set-level `.minio.sys` vs `.rustfs.sys` divergence). A bidirectional round-trip would require RustFS to optionally write the `.minio.sys` set layout, which is a separate feature. +- RustFS-written SSE objects readable by MinIO. +- KES / MinKMS / KMS-plugin-sealed MinIO objects. +- Objects sealed with pre-DARE-v2-HMAC MinIO seal algorithms; `parse_minio_managed_sealed_key` rejects unknown algorithms and the read fails closed. +- AWS KMS and KES wire-protocol compatibility. ## Guardrails -- This document is analysis only. Any change to `crates/filemeta` or - `crates/ecstore/src/bucket` metadata encoding is a storage-format change and - must follow the migration and readiness contracts in - [README.md](README.md) and the ecstore layout boundary rules. -- The version constants (`XL_META_VERSION`, - `BUCKET_METADATA_FORMAT`/`BUCKET_METADATA_VERSION`) are compatibility anchors. - Bumping any of them requires a read-compat path for the prior value and a - migration story, exactly as the current meta_ver 2 → 3 read path provides. +- Any change to `crates/filemeta` or `crates/ecstore/src/bucket` metadata encoding is a storage-format change and follows the migration and readiness contracts in [README.md](README.md) and the ecstore layout boundary rules. +- Do not bump a Version Anchor without a read path for the prior value; see [erasure-coding.md](erasure-coding.md) for the accept-older, reject-newer rule. +- `.github/workflows/ci.yml`, `.github/workflows/cache-warm.yml`, and `ARCHITECTURE.md` cite the [rio-v2 variant lifecycle](#rio-v2-variant-lifecycle) heading; keep it when editing this file. diff --git a/docs/architecture/minio-rustfs-router-compatibility.md b/docs/architecture/minio-rustfs-router-compatibility.md index 7055b6710..806ccc098 100644 --- a/docs/architecture/minio-rustfs-router-compatibility.md +++ b/docs/architecture/minio-rustfs-router-compatibility.md @@ -1,213 +1,50 @@ -# MinIO ↔ RustFS Router Compatibility Matrix +# MinIO ↔ RustFS Router Compatibility (Exceptions Only) -Tracks how RustFS covers the MinIO HTTP router surface, split into the S3 -data-plane router (`cmd/api-router.go` in MinIO: object + bucket APIs) and the -admin control-plane router (`cmd/admin-router.go`: admin `/v3/` and `/v4/` -APIs). Each row records the current RustFS implementation status and the -landing point in the code so the matrix can be re-verified after refactors. +**Use this when:** a client or `mc` call that works against MinIO fails against RustFS and you need to know whether the endpoint is missing, stubbed, or deliberately different. +**Source of truth:** S3 plane: the `s3s::S3` trait impl in `rustfs/src/storage/ecfs.rs`. Admin plane: `make_admin_route` in `rustfs/src/admin/mod.rs`, the registration inventory `rustfs/src/admin/route_registration_test.rs`, and the route/action guardrail [admin-route-action-snapshot.md](admin-route-action-snapshot.md). -This complements two neighbouring documents and does not duplicate them: - -- [s3-compatibility-matrix.md](s3-compatibility-matrix.md) — the release-facing - S3 compatibility claim and the Ceph s3tests lists that gate it. -- [admin-route-action-snapshot.md](admin-route-action-snapshot.md) — the admin - route/handler/authorization-action migration guardrail (the source of truth - for exact route patterns and auth contracts). - -Refs rustfs/backlog#596 rustfs/backlog#603. +This document lists only exceptions. Anything not listed here is implemented with MinIO-equivalent behavior. For the s3tests-level claim see [s3-compatibility-matrix.md](s3-compatibility-matrix.md). ## Status Legend | Status | Meaning | |---|---| -| 已实现 (implemented) | Handler is registered and performs the real operation. | -| 部分兼容 (partial) | Registered and functional, but a documented subset of the MinIO behavior is rejected or unsupported. | -| 已注册未完成 (registered, incomplete) | Route is registered but the handler returns `NotImplemented` (a behavior contract, not a real implementation). | -| 缺失 (missing) | No RustFS route/handler for the MinIO endpoint. | -| 行为不一致 (behavior differs) | Implemented but intentionally diverges from MinIO's response contract. | +| 缺失 (missing) | No RustFS route or handler for the MinIO endpoint. | +| 部分兼容 (partial) | Registered and functional, but a documented subset of MinIO behavior is rejected. | +| 已注册未完成 (registered, incomplete) | Route is registered; the handler returns `NotImplemented` as a behavior contract. | +| 行为不一致 (behavior differs) | Implemented, but intentionally diverges from MinIO's response contract. | -Prefixes: RustFS registers admin routes under the canonical `/rustfs/admin` -prefix and accepts `/minio/admin` as a compatibility alias via router -canonicalization (see -[admin-route-action-snapshot.md](admin-route-action-snapshot.md)). Admin paths -below are shown relative to that prefix (e.g. `/v3/info`). +Admin paths are relative to the canonical `/rustfs/admin` prefix; `/minio/admin` is accepted as an alias via router canonicalization. ---- +## S3 Data Plane -## Part 1 — S3 Data Plane (MinIO `cmd/api-router.go`) +All `s3s::S3` trait methods are implemented in `rustfs/src/storage/ecfs.rs` except the following. -RustFS implements the S3 surface through the `s3s` service trait in -`rustfs/src/storage/ecfs.rs`, delegating to use-case layers under -`rustfs/src/app/`. Line numbers are indicative landing points on the branch -this matrix was written against and may drift; the file paths are stable. - -### Bucket-level operations - -| MinIO / S3 operation | Status | RustFS landing point | +| S3 operation | Status | Detail | |---|---|---| -| CreateBucket | 已实现 | `rustfs/src/storage/ecfs.rs` (`create_bucket`) | -| DeleteBucket | 已实现 | `rustfs/src/storage/ecfs.rs` (`delete_bucket`) | -| HeadBucket | 已实现 | `rustfs/src/storage/ecfs.rs` (`head_bucket`) | -| ListBuckets | 已实现 | `rustfs/src/storage/ecfs.rs` (`list_buckets`) | -| GetBucketLocation | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_bucket_location`) | -| ListObjects (v1) | 已实现 | `rustfs/src/storage/ecfs.rs` (`list_objects`) | -| ListObjectsV2 | 已实现 | `rustfs/src/storage/ecfs.rs` (`list_objects_v2`) | -| ListObjectVersions | 已实现 | `rustfs/src/storage/ecfs.rs` (`list_object_versions`) | -| ListMultipartUploads | 已实现 | `rustfs/src/storage/ecfs.rs` (`list_multipart_uploads`) | -| Get/PutBucketVersioning | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_bucket_versioning`, `put_bucket_versioning`) | -| Get/Put/DeleteBucketPolicy | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_bucket_policy`, `put_bucket_policy`, `delete_bucket_policy`) | -| GetBucketPolicyStatus | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_bucket_policy_status`) | -| Get/Put/DeleteBucketTagging | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_bucket_tagging`, `put_bucket_tagging`, `delete_bucket_tagging`) | -| Get/Put/DeleteBucketLifecycle | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_bucket_lifecycle_configuration`, `put_bucket_lifecycle_configuration`, `delete_bucket_lifecycle`) | -| Get/Put/DeleteBucketReplication | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_bucket_replication`, `put_bucket_replication`, `delete_bucket_replication`) | -| Get/Put/DeleteBucketEncryption | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_bucket_encryption`, `put_bucket_encryption`, `delete_bucket_encryption`) | -| Get/PutObjectLockConfiguration | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_object_lock_configuration`, `put_object_lock_configuration`) | -| Get/Put/DeletePublicAccessBlock | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_public_access_block`, `put_public_access_block`, `delete_public_access_block`) | -| Get/Put/DeleteBucketCors | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_bucket_cors`, `put_bucket_cors`, `delete_bucket_cors`) | -| GetBucketNotificationConfiguration | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_bucket_notification_configuration`) | -| PutBucketNotificationConfiguration | 已实现 | `rustfs/src/storage/ecfs.rs` (`put_bucket_notification_configuration`) | -| Get/PutBucketRequestPayment | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_bucket_request_payment`, `put_bucket_request_payment`) | -| Get/PutBucketLogging | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_bucket_logging`, `put_bucket_logging`) | -| Get/Put/DeleteBucketWebsite | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_bucket_website`, `put_bucket_website`, `delete_bucket_website`) | -| Get/PutBucketAccelerateConfiguration | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_bucket_accelerate_configuration`, `put_bucket_accelerate_configuration`) | -| GetBucketAcl | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_bucket_acl`) | -| PutBucketAcl | 部分兼容 | `rustfs/src/storage/ecfs.rs` (`put_bucket_acl`) — canned-ACL headers only; XML grant policies return `NotImplemented`. | -| GetBucketReplicationMetrics | 缺失 | No S3-path handler; replication metrics are exposed via the admin API `/v3/replicationmetrics` instead. | -| GetBucketOwnershipControls | 缺失 | Not implemented (matches the "bucket ownership controls: planned" note in `s3-compatibility-matrix.md`). | -| Put/DeleteBucketOwnershipControls | 缺失 | Not implemented. | +| GetBucketReplicationMetrics | 缺失 | No `get_bucket_replication_metrics`; replication metrics are exposed via admin `/v3/replicationmetrics`. | +| GetBucketOwnershipControls | 缺失 | No handler; s3tests entries remain in `scripts/s3-tests/unimplemented_tests.txt`. | +| PutBucketOwnershipControls, DeleteBucketOwnershipControls | 缺失 | No handler. | +| DeleteBucketNotification, DeleteBucketLogging, DeleteBucketRequestPayment, DeleteBucketAccelerate | 部分兼容 | No distinct DELETE handlers; clear the config by writing an empty configuration through the PUT path. | +| PutBucketAcl, PutObjectAcl | 部分兼容 | Canned-ACL headers only; XML grant bodies return `NotImplemented` (`put_bucket_acl`, `put_object_acl`). | +| GetObjectTorrent | 行为不一致 | `get_object_torrent` returns `404 NoSuchKey` by design, not `501 NotImplemented`, so clients degrade gracefully. | -Note on delete verbs: several S3 sub-resource DELETE operations -(DeleteBucketNotification, DeleteBucketLogging, DeleteBucketRequestPayment, -DeleteBucketAccelerate) are not exposed as distinct handlers; the corresponding -config is cleared by writing an empty configuration through the PUT path. Treat -these as 部分兼容 at the client level. +## Admin Control Plane -### Object-level operations +Every route asserted in `rustfs/src/admin/route_registration_test.rs` is registered. Exceptions: -| MinIO / S3 operation | Status | RustFS landing point | +| MinIO admin family | Status | Detail | |---|---|---| -| GetObject | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_object`) | -| PutObject | 已实现 | `rustfs/src/storage/ecfs.rs` (`put_object`) | -| DeleteObject | 已实现 | `rustfs/src/storage/ecfs.rs` (`delete_object`) | -| DeleteObjects (multi-delete) | 已实现 | `rustfs/src/storage/ecfs.rs` (`delete_objects`) | -| HeadObject | 已实现 | `rustfs/src/storage/ecfs.rs` (`head_object`) | -| CopyObject | 已实现 | `rustfs/src/storage/ecfs.rs` (`copy_object`) | -| GetObjectAcl | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_object_acl`) | -| PutObjectAcl | 部分兼容 | `rustfs/src/storage/ecfs.rs` (`put_object_acl`) — canned-ACL headers only; XML grants return `NotImplemented`. | -| Get/Put/DeleteObjectTagging | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_object_tagging`, `put_object_tagging`, `delete_object_tagging`) | -| GetObjectAttributes | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_object_attributes`) | -| Get/PutObjectLegalHold | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_object_legal_hold`, `put_object_legal_hold`) | -| Get/PutObjectRetention | 已实现 | `rustfs/src/storage/ecfs.rs` (`get_object_retention`, `put_object_retention`) | -| RestoreObject (POST restore) | 已实现 | `rustfs/src/storage/ecfs.rs` (`restore_object`) | -| SelectObjectContent | 已实现 | `rustfs/src/storage/ecfs.rs` (`select_object_content`) → `rustfs/src/app/select_object.rs` | -| CreateMultipartUpload | 已实现 | `rustfs/src/storage/ecfs.rs` (`create_multipart_upload`) | -| UploadPart / UploadPartCopy | 已实现 | `rustfs/src/storage/ecfs.rs` (`upload_part`, `upload_part_copy`) | -| CompleteMultipartUpload | 已实现 | `rustfs/src/storage/ecfs.rs` (`complete_multipart_upload`) | -| AbortMultipartUpload | 已实现 | `rustfs/src/storage/ecfs.rs` (`abort_multipart_upload`) | -| ListParts | 已实现 | `rustfs/src/storage/ecfs.rs` (`list_parts`) | -| PostObject (POST form upload) | 已实现 | Routed via the POST-object marker into the put-object path (`rustfs/src/app/object/put.rs`). See the "POST Object form upload checksum handling: planned" note in `s3-compatibility-matrix.md`. | -| GetObjectTorrent | 行为不一致 | `rustfs/src/storage/ecfs.rs` (`get_object_torrent`) — returns `404 NoSuchKey` by design (not `501 NotImplemented`) so clients degrade gracefully. | +| Batch jobs (`/v3/start-job`, `/v3/list-jobs`, `/v3/status-job`, `/v3/describe-job`, `/v3/cancel-job`) | 已注册未完成 | `rustfs/src/admin/handlers/batch_job.rs`: `start-job` returns `NotImplemented` for known job types (`KNOWN_JOB_TYPES`) and `InvalidRequest` for unknown ones; `list-jobs` returns an empty list; status/describe/cancel return no-such-job. See [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md) for why `keyrotate` must keep refusing. | +| Service control (`POST /v3/service`) | 行为不一致 | `ServiceHandle` in `rustfs/src/admin/handlers/system.rs`: `restart` and `stop` both initiate graceful shutdown (the process manager must relaunch; no in-process restart); `freeze` / `unfreeze` toggle a global freeze flag under `ServiceFreezeAdminAction`. `rustfs/src/admin/route_policy.rs` still classifies the route as deferred `NotImplemented`. | +| Inspect data (`GET|POST /v3/inspect-data`) | 行为不一致 | `InspectDataHandler` in `system.rs` returns the raw bytes of one exact `volume` + `file`, size-capped, instead of MinIO's encrypted raw-drive-file archive. The bounded archive lives at `POST /v4/inspect/archive` (`rustfs/src/admin/handlers/inspect_archive.rs`). `route_policy.rs` still classifies the v3 route as deferred `NotImplemented`. | +| Pools decommission / cancel / clear | 部分兼容 | `rustfs/src/admin/handlers/pools.rs` returns `NotImplemented` when endpoints are not initialized (single-pool or uninitialized clusters). | +| `/v3/top/drives`, `/v3/top/net` | 缺失 | Only `/v3/top/locks` is registered (`rustfs/src/admin/handlers/diagnostics.rs`). | +| Bucket / site replication per-object diff | 缺失 | `/v3/replicationmetrics` and site-replication status exist; no diff endpoint. | +| MRF (most-recent-failures) replication metrics breakdown | 缺失 | Only the generic `/v3/metrics` stream and replication metrics wire (`rustfs/src/admin/replication_metrics_wire.rs`). | -For the gate-level view of which of these are covered by executable s3tests, -defer to [s3-compatibility-matrix.md](s3-compatibility-matrix.md); this table is -the router/handler view, not the test-list view. +Formerly-missing families that are now registered and therefore not exceptions: `/v3/healthinfo`, `/v3/obdinfo`, `/v3/force-unlock`, `/v3/top/locks`, `/v3/speedtest*`, `/v3/log`, `/v3/trace`, `/v3/profile`, `/v3/profiling/*`, `/v3/idp/{ldap|openid}/*`, `/v3/idp-config/*`. ---- +## Update Rule -## Part 2 — Admin Control Plane (MinIO `cmd/admin-router.go`) - -Router assembly is `rustfs/src/admin/mod.rs::register_admin_routes`; the exact -route patterns, handler ownership, and authorization actions are the guardrail -in [admin-route-action-snapshot.md](admin-route-action-snapshot.md). This table -maps MinIO admin route families to RustFS status. - -### Implemented / registered families - -| MinIO admin family | Status | RustFS landing point | -|---|---|---| -| STS / is-admin probe | 已实现 | `rustfs/src/admin/handlers/sts.rs`, `is_admin.rs` | -| User lifecycle (list/add/info/remove/status) | 已实现 | `rustfs/src/admin/handlers/user_lifecycle.rs`, `user.rs` | -| Groups | 已实现 | `rustfs/src/admin/handlers/group.rs` | -| Service accounts / access keys | 已实现 | `rustfs/src/admin/handlers/service_account.rs` | -| Canned policies + builtin policy attach/detach + policy-entities | 已实现 | `rustfs/src/admin/handlers/policies.rs` | -| IAM import/export | 已实现 | `rustfs/src/admin/handlers/user_iam.rs`, `user.rs` | -| Account info | 已实现 | `rustfs/src/admin/handlers/account_info.rs` | -| Config KV (get/set/del/help/history/restore + `/v3/config`) | 已实现 | `rustfs/src/admin/handlers/config_admin.rs` | -| Server info / storageinfo / datausageinfo | 已实现 | `rustfs/src/admin/handlers/system.rs` | -| Metrics stream (`/v3/metrics`) | 已实现 | `rustfs/src/admin/handlers/metrics.rs` via `system.rs` | -| Runtime capabilities (`/v4/runtime/capabilities`) | 已实现 | `rustfs/src/admin/handlers/system.rs` | -| Pools list/status | 已实现 | `rustfs/src/admin/handlers/pools.rs` | -| Pools decommission/cancel/clear | 部分兼容 | `rustfs/src/admin/handlers/pools.rs` — returns `NotImplemented` when endpoints are not initialized (single-pool / uninitialized clusters). | -| Rebalance start/status/stop | 已实现 | `rustfs/src/admin/handlers/rebalance.rs` | -| Heal + background-heal status | 已实现 | `rustfs/src/admin/handlers/heal.rs` | -| Tier (list/stats/verify/add/edit/remove/clear) | 已实现 | `rustfs/src/admin/handlers/tier.rs` | -| Quota (legacy + bucket-scoped + stats/check) | 已实现 | `rustfs/src/admin/handlers/quota.rs` | -| Bucket metadata export/import | 已实现 | `rustfs/src/admin/handlers/bucket_meta.rs` | -| Scanner status | 已实现 | `rustfs/src/admin/handlers/scanner.rs` | -| Notification targets (list/arns/put/reset) | 已实现 | `rustfs/src/admin/handlers/event.rs` | -| Audit targets (list/put/reset) | 已实现 | `rustfs/src/admin/handlers/audit.rs` | -| Module switches | 已实现 | `rustfs/src/admin/handlers/module_switch.rs` | -| Plugin catalog + instances (`/v4/plugins/*`) | 已实现 | `rustfs/src/admin/handlers/plugins_catalog.rs`, `plugins_instances.rs` | -| Extension catalog + instances (`/v4/extensions/*`) | 已实现 | `rustfs/src/admin/handlers/extensions.rs` | -| Object ZIP download (`/v3/zip-downloads`) | 已实现 | `rustfs/src/admin/handlers/object_zip_download.rs` | -| Cluster snapshot (`/v4/cluster/snapshot`) | 已实现 | `rustfs/src/admin/handlers/cluster_snapshot.rs` | -| Bucket-level remote targets (list/metrics/set/remove) | 已实现 | `rustfs/src/admin/handlers/replication.rs` | -| Site replication (add/remove/info/status/peer/resync + devnull/netperf) | 已实现 | `rustfs/src/admin/handlers/site_replication.rs` | -| Admin profiling (`/debug/pprof/profile`, `/debug/pprof/status`) | 已实现 | `rustfs/src/admin/handlers/profile_admin.rs`, `profile.rs` | -| TLS debug (`/debug/tls/status`) | 已实现 | `rustfs/src/admin/handlers/tls_debug.rs`, `profile.rs` | -| KMS management / dynamic / keys | 已实现 | `rustfs/src/admin/handlers/kms_management.rs`, `kms_dynamic.rs`, `kms_keys.rs` | -| OIDC public + config | 已实现 | `rustfs/src/admin/handlers/oidc.rs` | -| Table catalog (Iceberg) | 已实现 | `rustfs/src/admin/handlers/table_catalog/mod.rs` | - -### Registered-but-incomplete - -| MinIO admin family | Status | RustFS landing point | -|---|---|---| -| Service restart/stop (`POST /v3/service`) | 已注册未完成 | `rustfs/src/admin/handlers/system.rs` — handler returns `NotImplemented`. | -| Inspect data (`GET|POST /v3/inspect-data`) | 已注册未完成 | `rustfs/src/admin/handlers/system.rs` — handler returns `NotImplemented`. | - -These registered-but-`NotImplemented` routes are behavior contracts; per the -migration rules in [admin-route-action-snapshot.md](admin-route-action-snapshot.md), -implementing or removing them is a behavior-change PR. - ---- - -## Gaps Only — Missing Admin Endpoints (follow-up checklist) - -The following MinIO admin `/v3/` route families have **no** RustFS registration -today. This is the actionable checklist for closing admin-API parity. Verified -against `rustfs/src/admin/mod.rs` and `rustfs/src/admin/handlers/` on the branch -this doc was written on. - -- [ ] **Server profiling start/stop** — MinIO `/v3/profile` (bulk profiling - session). RustFS only exposes `/debug/pprof/profile` and - `/debug/pprof/status`, which are a different, single-shot pprof surface. -- [ ] **Health info** — MinIO `/v3/healthinfo` (cluster health report / subnet - diagnostics). No RustFS route. -- [ ] **LDAP / generic IDP config CRUD** — MinIO `/v3/idp/{ldap|openid}/...` - config management. RustFS exposes OIDC config under `/v3/oidc/*` only; there - is no LDAP IDP config route. -- [ ] **Bucket / site replication diff** — MinIO replication-diff endpoints. - RustFS exposes `/v3/replicationmetrics` (metrics) and site-replication - status, but no per-object diff. -- [ ] **MRF metrics** — MinIO's most-recent-failures replication metrics - breakdown. RustFS has only the generic `/v3/metrics` stream. -- [ ] **Batch jobs** — MinIO `/v3/batch`, `/v3/list-batch-jobs`, job - describe/cancel. No RustFS batch API. -- [ ] **Distributed locks introspection** — MinIO `/v3/force-unlock` and - `/v3/top/locks`. No RustFS locks-management API. -- [ ] **Speedtest / perf** — MinIO `/v3/speedtest` (object/drive/net perf). - RustFS has `netperf`/`devnull` **only** inside the site-replication family, - not as standalone admin speedtest endpoints. -- [ ] **Console log stream** — MinIO `/v3/log` (kstream / log search). No RustFS - route. -- [ ] **Top introspection** — MinIO `/v3/top/locks`, `/v3/top/drives`, - `/v3/top/net`. No RustFS unified `top` family. -- [ ] **Trace stream** — MinIO `/v3/trace`. A `trace.rs` handler skeleton - exists under `rustfs/src/admin/handlers/` but its registration function is - **not** called from `register_admin_routes`, so no route is live. - -When one of these lands, register it in `rustfs/src/admin/mod.rs`, extend -`rustfs/src/admin/route_registration_test.rs`, update -[admin-route-action-snapshot.md](admin-route-action-snapshot.md) with the -route/handler/action rows, and move the item out of this checklist. +When an exception above changes state, edit its row here in the same PR that changes the handler, and extend `rustfs/src/admin/route_registration_test.rs` and [admin-route-action-snapshot.md](admin-route-action-snapshot.md) for admin routes. Do not add "implemented" rows to this document; absence from this list is the implemented claim. diff --git a/docs/architecture/obs-ecstore-dependency-inventory.md b/docs/architecture/obs-ecstore-dependency-inventory.md index 1cdb2fe96..6791c9fe2 100644 --- a/docs/architecture/obs-ecstore-dependency-inventory.md +++ b/docs/architecture/obs-ecstore-dependency-inventory.md @@ -1,65 +1,35 @@ # Observability ECStore Dependency Inventory -This inventory closes the first `rustfs/backlog#735` step: make every -observability dependency on ECStore visible before introducing traits or moving -dependency direction. +**Use this when:** adding, removing, or moving any `rustfs_ecstore` or `rustfs_storage_api` reference inside `crates/obs`. +**Source of truth:** the `use` block at the top of `crates/obs/src/metrics/storage_api.rs`; the guard in `scripts/check_architecture_migration_rules.sh`. -No behavior or crate movement is planned in inventory PRs. The current boundary -is `crates/obs/src/metrics/storage_api.rs`; all direct `rustfs_ecstore` and -`rustfs_storage_api` source references in `rustfs-obs` must stay in that file -until the contracts below are extracted. +`rustfs-obs` still depends on `rustfs-ecstore` (`crates/obs/Cargo.toml`). Every direct reference is confined to one boundary file so the dependency can later be replaced by provider traits without touching collectors. ## Dependency Inventory -| Current symbol in `crates/obs/src/metrics/storage_api.rs` | Consumed by | Classification | Purpose | -|---|---|---|---| -| `rustfs_ecstore::api::storage::ECStore` as `ObsStore` | `stats_collector.rs` | Type dependency | Concrete object-store handle used to call storage admin methods and data-usage loaders. | -| `rustfs_storage_api::{BucketOperations, BucketOptions, StorageAdminApi}` | `stats_collector.rs` | Type and trait dependency | Method-resolution and associated type contracts for bucket listing, backend info, and storage info. | -| `rustfs_ecstore::api::runtime::object_store_handle` | `stats_collector.rs` | Runtime dependency | Resolves the currently published object-store handle for metric collection. | -| `rustfs_ecstore::api::data_usage::load_data_usage_from_backend` | `stats_collector.rs` | Behavior dependency | Loads bucket/object usage and is projected into obs-local DTOs before collectors consume it. | -| `rustfs_ecstore::api::capacity::{get_total_usable_capacity, get_total_usable_capacity_free}` | `stats_collector.rs` | Behavior dependency | Computes usable and free capacity from ECStore storage info. | -| `rustfs_ecstore::api::bucket::metadata_sys::get_quota_config` | `stats_collector.rs` | Behavior dependency | Reads per-bucket quota limits used in bucket usage metrics. | -| `rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor` | `runtime_sources.rs`, `stats_collector.rs` | Type and runtime dependency | Reads replication bandwidth reports from the global bucket monitor handle. | -| `rustfs_ecstore::api::runtime::bucket_monitor` | `runtime_sources.rs` | Runtime dependency | Resolves the global bucket bandwidth monitor for metric collection. | -| `rustfs_ecstore::api::bucket::replication::get_global_replication_stats` | `storage_api.rs` snapshot helpers | Runtime dependency | Reads replication status, transfer, failure, and site-replication stats, then projects them into obs-local snapshot DTOs. | -| `rustfs_ecstore::api::bucket::lifecycle::bucket_lifecycle_ops::{GLOBAL_ExpiryState, GLOBAL_TransitionState}` | `runtime_sources.rs`, `stats_collector.rs` | Runtime dependency | Reads lifecycle expiry and transition queue counters. | -| `rustfs_ecstore::api::error::Result` as `ObsEcstoreResult` | `stats_collector.rs` | Type dependency | Preserves ECStore error propagation while data-usage behavior remains ECStore-owned. | +The authoritative list is the `pub(crate) use rustfs_ecstore::api::...` block in `crates/obs/src/metrics/storage_api.rs`; it is not copied here. Each import belongs to one of three coupling categories: -## Classification +| Category | Covers | Examples (aliases defined in the boundary file) | +|---|---|---| +| Type coupling | Concrete ECStore types and storage-api traits used for method resolution | `ObsStore`, `ObsEcstoreResult`, `ObsBucketBandwidthMonitor`, the `rustfs_storage_api` trait imports | +| Runtime handle coupling | Resolving process-wide handles for metric collection | object-store handle, bucket monitor, expiry and transition state handles (`rustfs_ecstore::api::runtime::*`), replication stats read inside the snapshot helpers | +| Behavior coupling | ECStore-owned computations whose output is projected into obs-local DTOs | data-usage loading, compression totals, quota lookup, usable-capacity math | -The remaining coupling is not just a dependency declaration problem: - -- type coupling: `ObsStore`, `ObsEcstoreResult`, `ObsBucketBandwidthMonitor`, - `StorageAdminApi`, `BucketOperations`, and `BucketOptions`; -- runtime handle coupling: object-store handle, bucket monitor, replication - stats inside snapshot helpers, expiry state, and transition state; -- behavior coupling: data-usage loading, quota lookup, and capacity math. - -Removing `rustfs-ecstore` from `crates/obs/Cargo.toml` is unsafe until those -three categories have replacement contracts and compile coverage. +Collectors consume only the aliases and the obs-local DTOs. Removing `rustfs-ecstore` from `crates/obs/Cargo.toml` is unsafe until all three categories have replacement contracts and compile coverage. ## Extraction Plan -1. Keep all direct ECStore and storage-api imports centralized in - `crates/obs/src/metrics/storage_api.rs`. -2. Keep projecting ECStore data-usage and replication stats output into - obs-local DTOs before collectors consume it. -3. Introduce obs-owned provider traits for storage info, bucket info, quota, - data usage, replication, bandwidth, and lifecycle queue snapshots. -4. Implement those traits in ECStore or an ECStore-owned adapter crate after the - trait shapes are covered by focused tests. -5. Remove the `rustfs-ecstore` dependency from `rustfs-obs` only after metrics - behavior is unchanged through the provider traits. +1. Keep all direct ECStore and storage-api imports centralized in `crates/obs/src/metrics/storage_api.rs`. +2. Keep projecting ECStore data-usage and replication stats into obs-local DTOs before collectors consume them. +3. Introduce obs-owned provider traits for storage info, bucket info, quota, data usage, replication, bandwidth, and lifecycle queue snapshots. +4. Implement those traits in ECStore or an ECStore-owned adapter crate once the trait shapes are covered by focused tests. +5. Remove the `rustfs-ecstore` dependency from `rustfs-obs` only after metrics behavior is unchanged through the provider traits. ## Guardrails -The architecture guard enforces this inventory boundary: +Enforced by `scripts/check_architecture_migration_rules.sh`: -- `crates/obs/src/metrics/storage_api.rs` is the only `rustfs-obs` source file - allowed to reference `rustfs_ecstore` or `rustfs_storage_api`; -- raw replication stats handles and ECStore replication stat methods must stay - behind the snapshot helpers in `crates/obs/src/metrics/storage_api.rs`; -- `rustfs-obs` must not add `storage_compat.rs` or `ecstore_compat.rs` - passthrough bridges; -- future extraction PRs must update this inventory and the guard in the same - reviewed change when a dependency category is removed. +- `crates/obs/src/metrics/storage_api.rs` is the only `rustfs-obs` source file allowed to reference `rustfs_ecstore` or `rustfs_storage_api`. +- Raw replication stats handles and ECStore replication stat methods stay behind the snapshot helpers in that file. +- `rustfs-obs` must not add passthrough bridge modules (a second `storage_api.rs`, an `ecstore_compat.rs`, or similar) that re-export ECStore items to other crates. +- An extraction PR that removes a dependency category updates this inventory and the guard in the same change. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index b674a1047..253cf1329 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -1,60 +1,20 @@ # RustFS Architecture Evolution -This document set tracks the architecture migration from -[`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660). +**Use this when:** you need the historical framing of the architecture-migration program or the phase order that the migration contracts assume. +**Source of truth:** [README.md](README.md) is the index of architecture documents; the per-topic contracts it lists are authoritative. ## Baseline -- Baseline branch: `upstream/main` -- Baseline commit: `61f0dfbc40f748be313be84d834d8259cf3e19c9` -- Baseline title: `fix(ecstore): invalidate wiped disk id cache (#3251)` -- First migration PR type: `docs-only` +The architecture-migration program (`rustfs/backlog#660`) closed in 2026-07. Its original baseline commit predates the current `main` lineage and is no longer reachable from `main`; treat it as historical. The guardrails the program introduced remain enforced by `scripts/check_architecture_migration_rules.sh`. ## Core Principle -Cut wrong dependency directions with directories and contracts first, migrate global -state in small steps next, and split crates only after boundaries are stable. Storage -hot-path behavior must not drift during this migration. - -## Architecture Documents - -- [`runtime-lifecycle.md`](runtime-lifecycle.md): runtime, AppContext, - startup/readiness, and shutdown contracts. -- [`readiness-matrix.md`](readiness-matrix.md): request-surface behavior, - runtime dependency readiness, probe semantics, and preservation rules. -- [`s3-tables-support-matrix.md`](s3-tables-support-matrix.md): supported, - preview, reference-only, and not-claimed S3 Tables and Iceberg REST Catalog - surfaces. -- [`storage-control-data-plane.md`](storage-control-data-plane.md): boundaries - between StorageCore, ECStore, ClusterControlPlane, and BackgroundControllers. -- [`background-services-inventory.md`](background-services-inventory.md): current - scanner, heal, lifecycle, replication, config reload, metrics, and shutdown - surface before BackgroundController work. -- [`background-controller-contract.md`](background-controller-contract.md): - desired/current/status/reconcile vocabulary and lifecycle boundaries for - future read-only BackgroundController work. -- [`crate-boundaries.md`](crate-boundaries.md): PR types, crate direction, - compatibility rules, and migration guardrails. -- [`global-state-crate-split-plan.md`](global-state-crate-split-plan.md): late - global-state cleanup, runtime-source boundaries, fallback removal rules, and - crate-split evaluation criteria. -- [`obs-ecstore-dependency-inventory.md`](obs-ecstore-dependency-inventory.md): - observability-to-ECStore dependency inventory, classification, and extraction - guardrails. -- [`ecstore-config-consumer-inventory.md`](ecstore-config-consumer-inventory.md): - current `ecstore::config::{Config, KV, KVS}` definitions, consumers, - migration risks, and do-not-change contract. -- [`ecstore-api-facade-inventory.md`](ecstore-api-facade-inventory.md): current - `rustfs_ecstore::api` facade groups, external consumer boundaries, shrink - rules, and split dependency inventory. -- [`config-model-boundary-adr.md`](config-model-boundary-adr.md): target crate, - module path, dependency rules, and verification gates for moving the pure - server-config model. -- [`compat-cleanup-register.md`](compat-cleanup-register.md): temporary - compatibility code that must be removed later. +Cut wrong dependency directions with directories and contracts first, migrate global state in small steps next, and split crates only after boundaries are stable. Storage hot-path behavior must not drift during this migration. ## Phase Order +Historical sequencing of the migration phases. All phases are closed; the diagram is kept because later documents refer to phase names. + ```mermaid flowchart LR G["Phase 0: Baseline and guardrails"] @@ -80,10 +40,4 @@ flowchart LR GS --> CR ``` -The first implementation sequence is conservative: - -1. Record baseline and migration context. -2. Establish PR and compatibility rules. -3. Add dependency and loss-prevention checks in a separate `ci-gate` PR. -4. Inventory `ecstore::config::{Config, KV, KVS}` before moving any code. -5. Decide the config model boundary before extracting or migrating consumers. +The document index is [README.md](README.md). The ECStore facade boundary that the storage phases converged on is described in [ecstore-api-facade-inventory.md](ecstore-api-facade-inventory.md). diff --git a/docs/architecture/placement-repair-invariants.md b/docs/architecture/placement-repair-invariants.md index 81b28ba6b..acdda004d 100644 --- a/docs/architecture/placement-repair-invariants.md +++ b/docs/architecture/placement-repair-invariants.md @@ -1,53 +1,38 @@ # Placement And Repair Invariants -This inventory covers `G-012` for `rustfs/backlog#666`. It records the current -object placement, readiness, lock quorum, scanner, and repair boundaries that -later scheduler or topology work must preserve. +**Use this when:** changing anything that resolves an object to a pool, set, or disk, or that admits scanner or heal work; these are the behaviors later scheduler or topology work must preserve. +**Source of truth:** `Sets::get_disks_by_key` / `get_hashed_set_index` in `crates/ecstore/src/core/sets.rs`; `DistributionAlgoVersion` in `crates/ecstore/src/layout/format.rs`; `crc_hash` / `sip_hash` in `crates/utils/src/hash.rs`; `ScannerCycleBudget` in `crates/scanner/src/scanner_budget.rs`; `crates/scanner/src/scanner_heal_admission_baseline.rs`. ## Object To Set Hash Rule -Objects reach a set through `Sets::get_disks_by_key`, which calls -`get_hashed_set_index` on the object key: +Objects reach a set through `Sets::get_disks_by_key`, which calls `get_hashed_set_index` on the object key: - `DistributionAlgoVersion::V1` uses `crc_hash(input, set_count)`. -- `DistributionAlgoVersion::V2` and `V3` use - `sip_hash(input, set_count, format_id_bytes)`. -- The format ID is part of the V2/V3 distribution seed, so changing the seed, - object key, set count, or algorithm changes placement. +- `DistributionAlgoVersion::V2` and `V3` use `sip_hash(input, set_count, format_id_bytes)`. +- The format ID is part of the V2/V3 distribution seed, so changing the seed, object key, set count, or algorithm changes placement. -Preservation rule: every object read, write, list, heal, repair, and -decommission path that resolves a set for an existing object must preserve the -same object key and format distribution algorithm. +Preservation rule: every object read, write, list, heal, repair, and decommission path that resolves a set for an existing object must preserve the same object key and format distribution algorithm. ## Pool, Set, And Disk Assignment Boundary Pool selection is separate from set hashing: -- Existing objects are discovered across pools and resolved to the best current - pool candidate before reads or updates continue. -- New object writes select an available pool from current per-pool free-space - inputs after suspended or rebalancing pools are skipped. -- Set selection inside a pool still uses the object-to-set hash rule above. -- Disk index assignment comes from endpoint and format metadata, not from a - scheduler decision. +- Existing objects are discovered across pools and resolved to the best current pool candidate before reads or updates continue. +- New object writes select an available pool from current per-pool free-space inputs after suspended or rebalancing pools are skipped. +- Set selection inside a pool uses the object-to-set hash rule above. +- Disk index assignment comes from endpoint and format metadata, not from a scheduler decision. -Boundary rule: schedulers may influence admission, worker concurrency, or -buffer sizing, but they must not rewrite pool, set, or disk indexes. +Boundary rule: schedulers may influence admission, worker concurrency, or buffer sizing, but they must not rewrite pool, set, or disk indexes. ## Readiness And Lock Quorum Boundary -Runtime readiness currently checks storage and lock health independently: +Runtime readiness checks storage and lock health independently: -- Storage readiness requires every observed set to meet write quorum based on - the set drive count and storage class data/parity shape. -- Lock readiness aggregates per-set lock-client host quorum and fails fast if - any set loses quorum. -- Object and bucket mutations acquire namespace locks through the existing - storage lock wrappers before changing object or bucket state. +- Storage readiness requires every observed set to meet write quorum based on the set drive count and storage class data/parity shape. +- Lock readiness aggregates per-set lock-client host quorum and fails fast if any set loses quorum. +- Object and bucket mutations acquire namespace locks through the existing storage lock wrappers before changing object or bucket state. -Boundary rule: readiness and lock quorum must stay set-aware. A global healthy -disk count or global connected-host count is not sufficient when any individual -set is below quorum. +Boundary rule: readiness and lock quorum must stay set-aware. A global healthy disk count or global connected-host count is not sufficient when any individual set is below quorum. ## Scanner Budget Preservation @@ -55,43 +40,45 @@ Scanner cycles are bounded by `ScannerCycleBudget`: - Runtime budget cancels the child token after the configured duration. - Object budget cancels after the configured object count. -- Directory budget rejects additional directories and cancels with the - directories reason. +- Directory budget rejects additional directories and cancels with the directories reason. - Partial-cycle metrics and checkpoints use the budget reason. -Preservation rule: later scheduler work can change how scan cycles are admitted -only if it preserves the budget reason, checkpoint reason, and child-token -cancellation behavior. +Preservation rule: later scheduler work can change how scan cycles are admitted only if it preserves the budget reason, checkpoint reason, and child-token cancellation behavior. ## Heal Admission Preservation -Scanner and background repair work enter the heal manager through explicit -admission: +Scanner and background repair work enter the heal manager through explicit admission: -- Scanner object heal requests are low priority and may be accepted, merged, - rejected as full, or dropped. -- Required/high-priority heal candidates escalate on non-admission instead of - silently disappearing. -- Heal queue admission deduplicates queued and active work unless the request - explicitly forces admission. -- Full queues can drop low-priority work or displace lower-priority work for a - higher-priority request according to current manager rules. +- Scanner object heal requests are low priority and may be accepted, merged, rejected as full, or dropped. +- Required/high-priority heal candidates escalate on non-admission instead of silently disappearing. +- Heal queue admission deduplicates queued and active work unless the request explicitly forces admission. +- Full queues can drop low-priority work or displace lower-priority work for a higher-priority request according to current manager rules. -Preservation rule: repair scheduling changes must keep admission outcomes -observable and must not convert rejected or dropped repair work into silent -success. +Preservation rule: repair scheduling changes must keep admission outcomes observable and must not convert rejected or dropped repair work into silent success. + +### Scanner/heal admission entry points + +No cluster-wide coordinator or second generation token exists for scanner/heal admission; each entry point keeps its own guard. `crates/scanner/src/scanner_heal_admission_baseline.rs` `include_str!`s the scanner sources and asserts the named guards are still present, so a rename or guard removal fails that test instead of silently leaving this table stale. It also encodes the investigation matrix (scanner read and heal read may overlap; heal write conflicts with scanner reads; data-movement write conflicts with all work; independent sets stay concurrent) without claiming production enforces it. + +| Work | Entry point | Current guard | Fallback / namespace semantics | +|---|---|---|---| +| Scanner read/list | `nsscanner_disk` in `crates/scanner/src/scanner_io/io_disk.rs` | Per-disk `start_scan()` guard; bucket lifecycle/replication/object-lock reads precede `scan_data_folder` | Scanner keeps its local disk and durable cursor; no HealManager set-level admission is consulted | +| Scanner metadata read | Object-size and metadata branches in `crates/scanner/src/scanner_folder.rs` | Scanner cycle budget and per-disk scan marker | Corrupt metadata records the pending scanner ledger; MRF is a hint, not the durable owner | +| Scanner heal admission | `send_required_scanner_heal_request` in `crates/scanner/src/scanner_folder.rs` | Manager queue dedup and pending ledger (`update_pending_scanner_heal_after_admission`) | MRF `Enqueued` / `Coalesced` is ledger-only; rejected MRF keeps immediate heal plus ledger | +| Heal auto scan | `start_auto_disk_scanner` in `crates/heal/src/heal/manager/auto_scan.rs` | Queue-first then active-task check; replacement recovery blocklist | Scanning disks remain candidates when degraded quorum needs them; they are not globally excluded | +| Heal object read | `heal_object` in `crates/ecstore/src/set_disk/ops/heal.rs` | Namespace write lock (`get_write_lock`) unless `no_lock`; reads file info before commit | The namespace lock is object-scoped and does not claim scanner cycle ownership | +| Disk selection | `get_online_disks_with_healing_and_info` in `crates/ecstore/src/set_disk/ops/locking.rs` | Healing disks are ordered after new disks; scanning disks may remain candidates | Degraded/quorum fallback is preserved | +| Data movement | `wait_for_data_movement_admission` in `crates/ecstore/src/data_movement/backpressure.rs` | Storage-owned backpressure on foreground pressure; no second coordinator | Any future admission token must be validated at the final metadata/format/delete commit (see [unified-object-generation.md](unified-object-generation.md)) | + +Rules: cancellation or a local lease alone is not a fence; if a fixture ever demonstrates a stale destructive write, the fix extends the storage-owned generation/admission primitive and validates the token at the final commit rather than adding a coordinator. ## Behavior Change Gates Any later placement or repair PR must use the following gates: -- Placement gate: prove object-to-set hashing is unchanged for existing object - keys and format algorithms. -- Pool gate: prove pool selection does not choose suspended or rebalancing - pools unless the existing path already allows it. +- Placement gate: prove object-to-set hashing is unchanged for existing object keys and format algorithms. +- Pool gate: prove pool selection does not choose suspended or rebalancing pools unless the existing path already allows it. - Quorum gate: prove storage readiness and lock readiness remain per-set. - Scanner gate: prove scan budget reason and checkpoint mapping remain stable. -- Heal gate: prove low-priority scanner heal, forced heal, duplicate merge, and - queue-full outcomes remain distinct. -- Rollback gate: if a new scheduler sidecar is disabled, placement and repair - must fall back to the current direct ECStore/scanner/heal behavior. +- Heal gate: prove low-priority scanner heal, forced heal, duplicate merge, and queue-full outcomes remain distinct. +- Rollback gate: if a new scheduler sidecar is disabled, placement and repair must fall back to the current direct ECStore/scanner/heal behavior. diff --git a/docs/architecture/readiness-matrix.md b/docs/architecture/readiness-matrix.md index 22551eb4b..caaa33c03 100644 --- a/docs/architecture/readiness-matrix.md +++ b/docs/architecture/readiness-matrix.md @@ -1,8 +1,7 @@ # Readiness Matrix -This document records the current request and dependency behavior around -startup readiness. It is a behavior-preservation baseline for architecture -migration work, not a new readiness policy. +**Use this when:** changing what a request surface does before storage or IAM is ready, changing probe semantics, or adding a runtime dependency that readiness must wait for. +**Source of truth:** `rustfs/src/server/readiness.rs` (probe paths, `Retry-After`), `crates/common/src/readiness.rs` (`StorageReady`, `IamReady`, `FullReady`), `crates/config/src/constants/health.rs` (`RUSTFS_HEALTH_*` gates). This matrix is a behavior-preservation baseline, not a new readiness policy. ## Request Behavior Matrix @@ -23,7 +22,7 @@ migration work, not a new readiness policy. | IamReady | Inline IAM bootstrap or deferred IAM recovery publication. | Yes. | Deferred recovery can publish IAM readiness after HTTP has already started. | | Lock quorum | Per-set write quorum readiness. | Yes. | Do not replace the distributed lock quorum check with node count or endpoint count. | | Peer health | `peer_health_ready` runtime status. | Only when `RUSTFS_HEALTH_PEER_READY_CHECK_ENABLE` is enabled. | The gate is disabled by default; unknown peer health degrades readiness only when enabled. | -| KMS compatibility | KMS health compatibility readiness. | Only when the KMS compatibility readiness check is enabled. | KMS startup fatality and health reporting remain separate from pure docs work. | +| KMS compatibility | KMS health compatibility readiness. | Only when `RUSTFS_HEALTH_COMPAT_KMS_READY_CHECK_ENABLE` is enabled (default off; `crates/config/src/constants/health.rs`). | When enabled, `/health/ready` additionally requires the KMS service to be running if a global KMS manager exists. KMS startup fatality and health reporting remain separate from pure docs work. | Effective `FullReady` is: diff --git a/docs/architecture/runtime-capability-contracts.md b/docs/architecture/runtime-capability-contracts.md index f67069d19..c755ed6cd 100644 --- a/docs/architecture/runtime-capability-contracts.md +++ b/docs/architecture/runtime-capability-contracts.md @@ -1,70 +1,28 @@ # Runtime Capability Contracts -This document records the `rustfs/backlog#660` PR-08 and PR-09 contract slice. -It adds read-only observability and topology snapshot shapes to -`rustfs-storage-api` without coupling the contract crate to runtime, ECStore, -admin routes, profiling, or observability implementation crates. +**Use this when:** changing the read-only observability or topology snapshot contracts in `rustfs-storage-api`, their RustFS providers, or the `storage_classes` payload of `GET /rustfs/admin/v4/runtime/capabilities`. +**Source of truth:** `ObservabilitySnapshot` in `crates/storage-api/src/observability.rs`; `TopologySnapshot` in `crates/storage-api/src/topology.rs`; `CapabilityState` and `CapabilitySnapshotError` in `crates/storage-api/src/capability.rs`; providers in `rustfs/src/runtime_capabilities.rs`; storage-class constants in `crates/ecstore/src/config/storageclass.rs`. -## Observability Snapshot Contract +## Snapshot Contracts -`ObservabilitySnapshot` records: +Field lists live on the defining types and are not repeated here. -- Runtime telemetry capability state. -- Userspace CPU and memory profiling capability state. -- Process, system, and cgroup memory sampling state. -- Platform support for target triple, OS, architecture, allocator, eBPF, and - NUMA capability. +| Contract | Defining type | RustFS provider | Rule | +|---|---|---|---| +| Observability | `ObservabilitySnapshot` | `RustFsObservabilitySnapshotProvider` | Reports runtime telemetry, profiling, memory-sampling, platform, allocator, eBPF, and NUMA capability as `CapabilityState` values without starting telemetry, profiling, allocator reclaim, or memory-observability workers. | +| Topology | `TopologySnapshot` | `EndpointTopologySnapshotProvider` | Maps `EndpointServerPools` into pool/set/disk indexes, optional stable IDs, and optional zone/rack/node/media/NUMA labels without changing endpoint construction, placement, readiness, locks, or ECStore metadata. Local file endpoint paths are never used as disk IDs or labels; extra labels go in the `additional` map so future inventory labels need no ECStore type leakage. | -Unsupported, disabled, and unknown states are represented by `CapabilityState` -instead of failing snapshot construction. The contract is intentionally read -only and does not replace existing profiling routes, telemetry APIs, exporter -pipelines, or startup behavior. - -## Topology Snapshot Contract - -`TopologySnapshot` records: - -- Pool, set, and disk identity indexes plus optional stable IDs. -- Optional zone, rack, node, media, NUMA, and additional labels. -- Topology-wide profiling, NUMA, failure-domain label, and media-label - capability states. -- Per-disk media, failure-domain, NUMA, and profiling capability states. - -Missing labels are represented as absent `Option` values. Extra topology labels -belong in the `additional` label map, so future inventory labels do not require -ECStore type leakage. +Unsupported, disabled, and unknown states are values of `CapabilityState`, not construction failures. Missing labels are `None`. Providers map implementation failures into `CapabilitySnapshotError` before crossing the contract boundary. Neither contract replaces existing profiling routes, telemetry APIs, exporter pipelines, or startup behavior. ## Boundary Rules -- No `rustfs-ecstore`, `rustfs-obs`, Axum, KMS, admin route, OTEL, eBPF, or - profiling implementation dependency is added to `rustfs-storage-api`. -- No placement, membership, NUMA pinning, profiling, startup, admin route, or - exporter behavior changes are part of this contract slice. -- Providers must map implementation failures into `CapabilitySnapshotError` - before crossing the contract boundary. - -## RustFS Provider Slice - -`rustfs/src/runtime_capabilities.rs` wires the contracts to RustFS runtime -owners through read-only providers: - -- `RustFsObservabilitySnapshotProvider` maps current dial9, profiling, - memory-sampling, platform, allocator, eBPF, and NUMA capability state without - starting telemetry, profiling, allocator reclaim, or memory-observability - workers. -- `EndpointTopologySnapshotProvider` maps `EndpointServerPools` into pool, set, - and disk topology snapshots without changing endpoint construction, - placement, readiness, locks, or ECStore metadata. Local file endpoint paths are - intentionally not used as disk IDs or labels. - -Unsupported or unavailable runtime capabilities are reported as `unsupported` -or `unknown` contract states instead of activating fallback behavior. +- `rustfs-storage-api` gains no dependency on `rustfs-ecstore`, `rustfs-obs`, Axum, KMS, admin routes, OTEL, eBPF, or profiling implementation crates. +- Providers are read-only. Adding or changing a provider changes no placement, membership, NUMA pinning, profiling, startup, admin-route, or exporter behavior. +- Unsupported or unavailable runtime capabilities are reported as `unsupported` or `unknown`; they never activate fallback behavior. ## Storage-Class Write Contract -Authenticated clients discover the storage-class write contract from -`GET /rustfs/admin/v4/runtime/capabilities`. The additive -`storage_classes` object is versioned independently from the route: +Authenticated clients discover the storage-class write contract from `GET /rustfs/admin/v4/runtime/capabilities`. The additive `storage_classes` object is versioned independently from the route: ```json { @@ -77,15 +35,11 @@ Authenticated clients discover the storage-class write contract from } ``` -`supported_write_classes` is the complete client-selectable write allowlist. -Any other value fails before object or multipart mutation with the stable S3 -error named by `unsupported_write_error`. `legacy_label_behavior` means -non-transitioned historical label-only metadata is reported as its effective -local class; actual lifecycle transition tier names remain unchanged. +| Field | Meaning | +|---|---| +| `supported_write_classes` | The complete client-selectable write allowlist. Any other value fails before object or multipart mutation with the S3 error named by `unsupported_write_error`. | +| `unsupported_write_error` | Stable S3 error code (`UNSUPPORTED_WRITE_ERROR` in `crates/ecstore/src/config/storageclass.rs`). | +| `legacy_label_behavior` | Non-transitioned historical label-only metadata is reported as its effective local class; lifecycle transition tier names are unchanged. | +| `contract_version` | Consumers must branch on it before assigning meaning to future fields. | -The values are sourced from -[`crates/ecstore/src/config/storageclass.rs`](../../crates/ecstore/src/config/storageclass.rs), -which also owns write validation and response normalization. Consumers must -branch on `contract_version` before assigning meaning to future fields. The -admin route continues to require `ServerInfoAdminAction`; capability discovery -does not weaken authentication or authorization. +Values, write validation, and response normalization are owned by `crates/ecstore/src/config/storageclass.rs`. The route continues to require `ServerInfoAdminAction`; capability discovery does not weaken authentication or authorization. diff --git a/docs/architecture/runtime-lifecycle.md b/docs/architecture/runtime-lifecycle.md index 7c77dcedb..9204a21ad 100644 --- a/docs/architecture/runtime-lifecycle.md +++ b/docs/architecture/runtime-lifecycle.md @@ -1,120 +1,57 @@ # Runtime And Lifecycle Contracts -Runtime and lifecycle work must preserve startup ordering, readiness behavior, and -shutdown semantics. +**Use this when:** moving or reordering anything in `rustfs/src/startup_*.rs`, changing readiness publication, or touching shutdown ordering. +**Source of truth:** the `rustfs/src/startup_*.rs` modules listed below; readiness semantics in [readiness-matrix.md](readiness-matrix.md); global-state targets in [global-state-inventory.md](global-state-inventory.md). + +Runtime and lifecycle work must preserve startup ordering, readiness behavior, and shutdown semantics. ## Startup And Readiness -- HTTP can listen early, but normal requests must remain behind readiness gates. -- Effective `FullReady = storage_ready && iam_ready && lock_quorum_ready && - peer_health_ready`; `peer_health_ready` is true by default unless - `RUSTFS_HEALTH_PEER_READY_CHECK_ENABLE` is enabled. -- Boot phases must keep the old fatal and non-fatal boundaries. -- AppContext migration keeps context-first lookup with global fallback until the - global path is proven unused. +- HTTP can listen early, but normal requests stay behind the readiness gate. +- The `FullReady` formula, its dependencies, and the `RUSTFS_HEALTH_PEER_READY_CHECK_ENABLE` gate are defined once in [readiness-matrix.md](readiness-matrix.md); do not restate them elsewhere. +- Boot phases keep the existing fatal and non-fatal boundaries. +- AppContext migration keeps context-first lookup with global fallback until the global path is proven unused. - Notify and audit lifecycle behavior must not drift during lifecycle movement. -- IAM and KMS startup, deferred recovery, and fatal boundary behavior must not be - changed by pure movement PRs. -- Request-surface and dependency details are tracked in - [`readiness-matrix.md`](readiness-matrix.md). +- IAM and KMS startup, deferred recovery, and fatal-boundary behavior must not be changed by pure movement PRs. -## Service Registry Scope +## Startup Module Ownership -`ServiceRegistry` is only for lifecycle and shutdown ordering. It must not become a -general dependency injection container. +Each module owns one concern; orchestration order is owned by `startup_services`, `startup_lifecycle`, and `startup_shutdown`. Movement PRs may pass handles between modules but must not reorder the steps a module owns. -Allowed responsibilities: - -- Register start and stop order. -- Expose read-only status snapshots. -- Coordinate graceful shutdown. - -Disallowed responsibilities: - -- Construct arbitrary dependencies for business logic. -- Hide globals behind a service-locator API. -- Change startup side effects while moving code. +| Module (`rustfs/src/`) | Owns | +|---|---| +| `startup_entrypoint.rs` | CLI command dispatch into preflight and the runtime lifecycle. | +| `startup_preflight.rs` | License init, external env compatibility, runtime foundation bootstrap. | +| `startup_runtime.rs` | Runtime foundation orchestration; outbound TLS fatal boundary when configured material fails to load. | +| `startup_runtime_hooks.rs` | Startup diagnostics, profiling hook dispatch, default crypto provider installation. | +| `startup_tls_material.rs` | Outbound TLS material loading, global publication, generation recording, TLS metrics init. | +| `startup_runtime_sources.rs` | Process-local runtime source publication (port, buffer profile, KMS manager, TLS generation). | +| `startup_fs_guard.rs` | Unsupported-filesystem policy enforcement for endpoint paths. | +| `startup_deadlock.rs` | Deadlock detector state logging. | +| `startup_server.rs` | HTTP listener start and `ServiceStateManager` publication. | +| `startup_storage.rs` | Endpoints, local disks, ECStore, lock clients, global config, background replication init; `StorageReady`. | +| `startup_bucket_metadata.rs` | Bucket metadata system init, legacy meta-bucket import (`try_migrate_bucket_metadata`, `try_migrate_iam_config`), resync intents. | +| `startup_iam.rs` | IAM init, deferred recovery, `IamReady` publication. | +| `startup_auth.rs` | OIDC and federated identity setup. | +| `startup_notification.rs` | Notification system and bucket notification configuration. | +| `startup_audit.rs` | Event notifier and audit system start. | +| `startup_observability.rs` | Auto-tuner, update check, server info, compression totals. | +| `startup_background.rs` | Scanner, heal, bitrot self-test, workload-admission provider publication. | +| `startup_protocols.rs` | FTP/FTPS/SFTP/WebDAV sidecar start and shutdown senders. | +| `startup_optional_runtime_sidecars.rs` | Handles, shutdown planning, and shutdown execution for optional sidecars that are not readiness boundaries (currently protocol servers only). New sidecars enter here with explicit shutdown handles and status snapshots, not ad hoc work in `startup_services`. | +| `startup_services.rs` | Orchestration order of runtime service startup: KMS, optional runtimes, audit, metadata, IAM, auth, notification, background services, observability. | +| `startup_lifecycle.rs` | Ready publication, global init-time publication, scanner startup, shutdown-signal wait, shutdown delegation, final stopped-state log. | +| `startup_shutdown.rs` | The shutdown sequence (see below). | +| `startup_embedded.rs`, `startup_embedded_optional.rs` | Embedded-mode reuse of the phase owners above (see below). | ## Shutdown Lifecycle Boundary -`startup_shutdown` owns the main shutdown sequence after the process receives a -shutdown signal. Startup modules may pass handles into this boundary, but they -must not reorder runtime-token cancellation, background service shutdown, -optional runtime shutdown planning, notifier/audit/profiling shutdown, HTTP -shutdown, optional runtime waits, or final service-state publication. +`startup_shutdown` owns the main shutdown sequence after the process receives a shutdown signal. Startup modules may pass handles into this boundary, but they must not reorder runtime-token cancellation, background service shutdown, optional runtime shutdown planning, notifier/audit/profiling shutdown, HTTP shutdown, optional runtime waits, or final service-state publication. -## Startup Lifecycle Boundary +## Embedded Startup Reuse -`startup_lifecycle` owns the ready-to-shutdown orchestration after runtime -services initialize. Service modules may return initialized handles into this -boundary, but they must not reorder ready publication, global init-time -publication, scanner startup, shutdown-signal wait, shutdown delegation, or the -final stopped-state log. - -## Startup Service Component Boundary - -`startup_service_components` owns individual runtime service startup component -helpers while `startup_services` preserves their orchestration order. Migration -PRs must not change KMS, optional runtime, audit, metadata, IAM, auth, -notification, background service, or observability startup ordering while moving -these helpers. - -## Optional Runtime Boundary - -`startup_optional_runtime_sidecars` owns startup handles, shutdown planning, and -shutdown execution for optional runtime services that are not readiness -boundaries. `startup_optional_runtimes` remains a compatibility handoff for the -old module path. The current owner set is protocol servers only. Future optional -sidecars must enter this boundary with explicit shutdown handles and status -snapshots instead of adding ad hoc startup or shutdown work to -`startup_services`. - -## Startup Runtime Hook Boundary - -`startup_runtime_hooks` owns runtime hook side effects that wrap the startup -foundation but are not TLS material loading: startup diagnostics, profiling -hook dispatch, and default crypto provider installation. `startup_runtime` -preserves BOOT-006 orchestration and outbound TLS fatal behavior, while -`startup_profiling` remains a compatibility handoff for the old profiling hook -path. - -## Startup TLS Material Boundary - -`startup_tls_material` owns configured outbound TLS material loading, global -outbound TLS publication, generation recording, and TLS metrics initialization. -`startup_runtime` still owns BOOT-006 ordering and must preserve the fatal -boundary when configured TLS material fails to load. - -## Embedded Startup Phase Reuse - -Embedded startup should use the same startup server and storage phase owners for -listen context, endpoint/local disk setup, storage runtime setup, readiness -publication, and replication startup. Embedded-specific behavior still owns its -stable-port requirement, one-shot global initialization guard placement, S3-only -HTTP listener, and non-fatal KMS/audit/notification policy. - -## Embedded Runtime Service Reuse - -Embedded runtime service setup should share startup service helpers for optional -service initialization, bucket metadata/IAM setup, notification setup, and -shutdown cleanup. Embedded-specific behavior still owns warning-only -KMS/audit/notification failures, no binary-only background sidecars, no state -manager, and the one-shot server handle cleanup used by embedded shutdown. - -## Embedded Lifecycle Publication Reuse - -Embedded ready publication should share startup lifecycle helpers for IAM -readiness publication, global init-time publication, and ready-state logging. -Embedded-specific behavior still owns server handle construction, endpoint -address normalization, and process-local shutdown cleanup. +Embedded startup reuses the same phase owners as the binary: server and storage phases for listen context, endpoint/local disk setup, storage runtime setup, readiness publication, and replication startup; service helpers for optional service init, bucket metadata/IAM setup, notification setup, and shutdown cleanup; lifecycle helpers for IAM readiness publication, global init-time publication, and ready-state logging. Embedded-specific behavior that stays in `startup_embedded*.rs`: stable-port requirement, one-shot global initialization guard placement, S3-only HTTP listener, warning-only KMS/audit/notification failures, no binary-only background sidecars, no state manager, server handle construction, endpoint address normalization, and process-local one-shot shutdown cleanup. ## AppContext Foundation -Early AppContext work should split resolver files and add compatibility tests before -boot extraction or consumer migration. This keeps the migration context-first while -preserving the old global fallback path during transition. - -AppContext remains a context-first facade, not a full replacement for every -process global. New migration work must keep fallback reads inside owner-local -runtime-source boundaries and follow the global-state target inventory in -[`global-state-inventory.md`](global-state-inventory.md). +AppContext is a context-first facade, not a full replacement for every process global. Resolver files are split and covered by compatibility tests before boot extraction or consumer migration, so the old global fallback path keeps working during transition. New migration work keeps fallback reads inside owner-local runtime-source boundaries and follows the target inventory in [global-state-inventory.md](global-state-inventory.md). diff --git a/docs/architecture/s3-compatibility-matrix.md b/docs/architecture/s3-compatibility-matrix.md index aa7ed8457..9393e129b 100644 --- a/docs/architecture/s3-compatibility-matrix.md +++ b/docs/architecture/s3-compatibility-matrix.md @@ -1,44 +1,30 @@ # S3 Compatibility Matrix -This matrix records the user-facing S3 compatibility claim for RustFS and ties -it to the executable Ceph s3tests lists under `scripts/s3-tests/`. +**Use this when:** writing or checking a user-facing S3 compatibility claim, or moving a Ceph s3tests case between lists. +**Source of truth:** the test lists under `scripts/s3-tests/` and the runner `scripts/s3-tests/run.sh`; counts are derived from those files and are not recorded here. ## Current Claim -RustFS provides broad S3 API compatibility for supported features. It does not -claim complete coverage of every standard or vendor-specific S3 behavior. - -The root README should use the same wording: supported S3-compatible clients and -features are covered by the compatibility matrix and test lists. +RustFS provides broad S3 API compatibility for supported features. It does not claim complete coverage of every standard or vendor-specific S3 behavior. The root README uses the same wording: supported S3-compatible clients and features are covered by the compatibility matrix and test lists. ## Test List Sources -| List | Purpose | Current count | Source | -|---|---:|---:|---| -| Implemented tests | Standard S3 tests expected to pass and used by the default local s3tests run. | 452 | `scripts/s3-tests/implemented_tests.txt` | -| Lifecycle behavior tests | Expiration behavior cases gated by the dedicated `s3-lifecycle-behavior-tests` lane (debug-accelerated day + scanner enabled). | 5 | `scripts/s3-tests/lifecycle_behavior_tests.txt` | -| Unimplemented tests | Standard S3 features planned but not yet implemented. | 17 | `scripts/s3-tests/unimplemented_tests.txt` | -| Excluded tests | Vendor-specific or intentionally unsupported behavior excluded from RustFS compatibility gating. | 273 | `scripts/s3-tests/excluded_tests.txt` | +| List | Purpose | Source | +|---|---|---| +| Implemented tests | Standard S3 tests expected to pass; the default local s3tests run. | `scripts/s3-tests/implemented_tests.txt` | +| Lifecycle behavior tests | Days-based expiration cases gated by the `s3-lifecycle-behavior-tests` lane in `.github/workflows/ci.yml`. | `scripts/s3-tests/lifecycle_behavior_tests.txt` | +| Unimplemented tests | Standard S3 features not yet passing. | `scripts/s3-tests/unimplemented_tests.txt` | +| Excluded tests | Vendor-specific or intentionally unsupported behavior excluded from RustFS gating. | `scripts/s3-tests/excluded_tests.txt` | -Counts ignore blank lines and comments. - -The lifecycle behavior lane runs real Days-based expiration cases that need -`RUSTFS_ILM_DEBUG_DAY_SECS` (Ceph `lc_debug_interval` equivalent) and an enabled -background scanner; it cannot share the default single-server gate because a -global debug day would also shrink the `x-amz-expiration` header asserted by the -`test_lifecycle_expiration_header_*` cases. See `scripts/s3-tests/run.sh` -(`IMPLEMENTED_TESTS_FILE` override) and the `s3-lifecycle-behavior-tests` job in -`.github/workflows/ci.yml`. +Counts ignore blank lines and comments; compute them from the files. The lifecycle lane runs separately because its cases need `RUSTFS_ILM_DEBUG_DAY_SECS` and an enabled scanner, and a global debug day would also shrink the `x-amz-expiration` header asserted by `test_lifecycle_expiration_header_*`; see `IMPLEMENTED_TESTS_FILE` in `scripts/s3-tests/run.sh`. ## Supported Coverage -The implemented test list currently covers the common object-storage surface: - | Area | Status | Evidence | |---|---|---| | Bucket create/delete/list/head | Supported | `implemented_tests.txt` | | Object put/get/delete/copy/head | Supported | `implemented_tests.txt` | -| CopyObject checksums (CRC32, CRC32C, CRC64NVME, SHA1, SHA256, MD5, SHA512, XXHASH3, XXHASH64, XXHASH128), including source preservation and explicit override | Supported in the first RustFS release containing this change | `crates/e2e_test/src/copy_object_checksum_test.rs` | +| CopyObject checksums (CRC32, CRC32C, CRC64NVME, SHA1, SHA256, MD5, SHA512, XXHASH3, XXHASH64, XXHASH128), including source preservation and explicit override | Supported | `crates/e2e_test/src/copy_object_checksum_test.rs` | | ListObjects/ListObjectsV2 prefix, delimiter, marker, max-keys | Supported | `implemented_tests.txt` | | Multipart upload create/upload/complete/abort and selected multipart copy/checksum/object-attribute behavior | Supported | `implemented_tests.txt` | | Bucket and object tagging | Supported | `implemented_tests.txt` | @@ -50,33 +36,25 @@ The implemented test list currently covers the common object-storage surface: | SSE-C and selected SSE-KMS edge cases | Supported | `implemented_tests.txt` | | Selected versioning, object-lock, checksum, CORS, raw request, and conditional write behavior | Supported | `implemented_tests.txt` | -"Supported" for the SSE row means RustFS encrypts and decrypts its own objects. It does not mean RustFS can read objects another implementation encrypted: objects MinIO wrote with SSE-S3, SSE-KMS, or SSE-C are not readable by RustFS today, which matters when migrating. See [MinIO file-format interoperability, Part C](minio-file-format-compat.md#part-c--server-side-encryption-sse) and rustfs/backlog#1638. +"Supported" for the SSE row means RustFS encrypts and decrypts its own objects. MinIO SSE objects (SSE-S3, SSE-KMS, SSE-C) are not readable in default builds; see [minio-file-format-compat.md Part C](minio-file-format-compat.md#part-c--server-side-encryption-sse) for the `rio-v2` migration build. -## Planned Standard Coverage +## Not Yet Passing -These are standard S3 areas that remain planned work and must not be described -as already complete: +Standard S3 areas that must not be described as complete: | Area | Status | Evidence | |---|---|---| -| Bucket access logging | Planned | `unimplemented_tests.txt` | -| POST Object form upload checksum handling | Planned | `unimplemented_tests.txt` | -| Bucket ownership controls | Planned | `unimplemented_tests.txt` | +| Bucket access logging | Handlers exist (`get_bucket_logging`, `put_bucket_logging` in `rustfs/src/storage/ecfs.rs`); the `test_*bucket_logging*` s3tests cases are still listed as unimplemented | `unimplemented_tests.txt` | +| POST Object form upload checksum handling | Not yet passing | `unimplemented_tests.txt` | +| Bucket ownership controls | No handler | `unimplemented_tests.txt` | | Multipart upload listing and part lookup compatibility edge cases | Not part of default gate | `excluded_tests.txt` | | IAM-account or multi-storage-class dependent cases | Not part of default gate | `unimplemented_tests.txt` | | Tenanted bucket policy edge cases | Needs investigation | `unimplemented_tests.txt` | ## Intentional Exclusions -`excluded_tests.txt` contains tests that should not block the RustFS -compatibility gate. They fall into two classes: - -- vendor-specific or non-portable behavior not required for RustFS S3 - compatibility; -- intentionally unsupported product behavior, such as ACL authorization. +`excluded_tests.txt` holds tests that must not block the compatibility gate: vendor-specific or non-portable behavior, and intentionally unsupported product behavior such as ACL authorization. ## Update Rule -When a planned S3 feature is implemented, move its passing test entries from -`unimplemented_tests.txt` to `implemented_tests.txt`, update this matrix, and -avoid changing README wording beyond the supported coverage. +When a feature starts passing, move its test entries from `unimplemented_tests.txt` to `implemented_tests.txt` and update the row here in the same PR. Do not change README wording beyond the supported coverage. Handler-level status (missing, stubbed, or diverging endpoints) is tracked in [minio-rustfs-router-compatibility.md](minio-rustfs-router-compatibility.md). diff --git a/docs/architecture/s3-tables-support-matrix.md b/docs/architecture/s3-tables-support-matrix.md index 309bd1573..7d476d5f9 100644 --- a/docs/architecture/s3-tables-support-matrix.md +++ b/docs/architecture/s3-tables-support-matrix.md @@ -1,327 +1,152 @@ # S3 Tables Support Matrix -This matrix records the RustFS S3 Tables surfaces that are supported, -previewed, referenced, or intentionally not claimed. It is the release-facing -boundary for the Iceberg REST Catalog work in RustFS. +**Use this when:** writing a release note, README claim, or client-compatibility statement about RustFS S3 Tables / Iceberg REST Catalog, or deciding whether a feature is supported, preview, or not claimed. +**Source of truth:** the table-catalog handlers under `rustfs/src/admin/handlers/table_catalog/`; the conformance scripts and their README in `scripts/table-catalog/`; the durable-backing cutover procedure in [docs/operations/s3-tables-cutover-runbook.md](../operations/s3-tables-cutover-runbook.md). -RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on -top of the RustFS S3 data plane. This document does not claim full parity with -the AWS S3 Tables control-plane API or with every vendor-specific Iceberg -catalog extension. +RustFS S3 Tables is an Iceberg REST Catalog and table-bucket implementation on top of the RustFS S3 data plane. It does not claim parity with the AWS S3 Tables control-plane API or with vendor-specific Iceberg catalog extensions. ## Status Labels | Label | Meaning | |---|---| | Automated | Covered by a runnable RustFS script or server test. | -| Manual/live harness | RustFS can generate pinned client package inputs, commands, expected outputs, and CI opt-in gates for a live endpoint, but the live run is not enabled by default in CI. | -| Generated harness | RustFS can generate client configuration or probe input, but live execution is not automated in CI. | +| Manual/live harness | RustFS generates pinned client inputs, commands, expected outputs, and CI opt-in gates for a live endpoint; the live run is not enabled by default in CI. | +| Generated harness | RustFS generates client configuration or probe input; live execution is not automated. | | Supported | Implemented server-side and covered by focused RustFS tests. | -| Preview / controlled | Implemented behind explicit operator action or a run-once endpoint. No automatic background claim is made. | -| Documented, not automated | Configuration or behavior is documented, but the live client run is not automated. | -| Reference only | Kept as a compatibility reference. RustFS does not claim live interoperability yet. | -| Not claimed | Out of scope for the current S3 Tables implementation. | +| Preview / controlled | Implemented behind explicit operator action or a run-once endpoint; no automatic background claim. | +| Documented, not automated | Behavior is documented; the live client run is not automated. | +| Reference only | Compatibility reference; live interoperability is not claimed. | +| Not claimed | Out of scope for the current implementation. | ## Endpoint And Profile Matrix | Surface | Status | Notes | |---|---|---| -| `/iceberg/v1` | Supported | Canonical RustFS Iceberg REST Catalog prefix. Default REST signing name is `s3`. | -| `/_iceberg/v1` | Supported compatibility alias | MinIO AIStor-style alias. The smoke profile defaults to REST signing name `s3tables`. | -| S3 object data plane | Supported | Data, metadata, manifest, and delete files remain ordinary S3 objects, with table-aware policy checks for table warehouse paths. | -| Table bucket enablement | Supported | A regular RustFS bucket can be enabled for table catalog use and then addressed as the REST catalog warehouse. | -| Catalog-vended table credentials | Automated when enabled | Disabled by default. When enabled, LoadTable vends credentials only when `X-Iceberg-Access-Delegation` contains the exact `vended-credentials` token; the dedicated credentials endpoint uses the same issuer path. | -| AWS S3 Tables endpoint shape | Profile generator | Generates the AWS catalog URI and S3 Tables warehouse ARN shape for migration docs. Full AWS S3 Tables API parity is not claimed. | -| MinIO AIStor Tables profile | Profile generator plus RustFS alias smoke | RustFS exposes the alias shape, but does not claim all AIStor private extensions. | -| Cloudflare R2 Data Catalog profile | Profile generator | Generates the catalog URI and warehouse-name shape for migration docs. Live RustFS interoperability is not claimed. | -| Alibaba OSS Tables profile | Profile generator | Generates provider endpoint, `acs:osstables` warehouse ARN, `osstables` signing-name, and `https://oss-{region}.aliyuncs.com` S3FileIO endpoint shapes for migration docs. Live RustFS interoperability is not claimed. | +| `/iceberg/v1` | Supported | Canonical REST Catalog prefix; default REST signing name `s3`. | +| `/_iceberg/v1` | Supported compatibility alias | MinIO AIStor-style alias; smoke profile defaults to signing name `s3tables`. | +| S3 object data plane | Supported | Data, metadata, manifest, and delete files are ordinary S3 objects with table-aware policy checks on warehouse paths. | +| Table bucket enablement | Supported | A regular bucket is enabled for catalog use and addressed as the REST catalog warehouse. | +| Catalog-vended table credentials | Automated when enabled | Disabled by default. LoadTable vends credentials only when `X-Iceberg-Access-Delegation` contains the exact `vended-credentials` token; the dedicated credentials endpoint uses the same issuer path. | +| AWS S3 Tables endpoint shape | Profile generator | Generates the AWS catalog URI and warehouse ARN shape for migration docs. API parity not claimed. | +| MinIO AIStor Tables profile | Profile generator plus alias smoke | Alias shape only; AIStor private extensions not claimed. | +| Cloudflare R2 Data Catalog profile | Profile generator | Catalog URI and warehouse-name shape only; live interop not claimed. | +| Alibaba OSS Tables profile | Profile generator | Endpoint, `acs:osstables` warehouse ARN, `osstables` signing name, and `https://oss-{region}.aliyuncs.com` S3FileIO endpoint shapes only; live interop not claimed. | ## Client And Engine Matrix -| Client or engine | Status | Current RustFS claim | +| Client or engine | Status | Claim | |---|---|---| -| PyIceberg | Automated | Creates namespace and table, appends rows, reloads, scans, probes metadata-location, refs, views, maintenance, diagnostics, and optional catalog-vended table credentials with an exact-prefix data-plane scope check. | -| Spark Iceberg REST catalog | Manual/live harness | RustFS can generate pinned Spark/Iceberg package inputs, REST catalog properties, SQL, run commands, expected `row_count=2`, and a CI opt-in gate for namespace creation, table creation, append, refresh, count, and cleanup. Live Spark execution and commit-conflict probing are still manual validation items unless explicitly enabled in the runner. | -| Trino Iceberg REST catalog | Manual/live harness | RustFS can generate catalog properties and a read-only `SELECT COUNT(*)` command for a table created by PyIceberg or Spark. Write compatibility is not claimed. | -| DuckDB Iceberg 1.5.5 | Automated | `duckdb_smoke.py` verifies the metadata-location read path and generic REST Catalog single-table create, insert, update, delete, merge, schema evolution, snapshots, concurrent writers, normal drop, PyIceberg cross-read, `/iceberg` with `s3` signing, and `/_iceberg` with `s3tables` signing. Staged create, purge-on-drop, and format v3 are verified as fail-closed boundaries. DuckDB's endpoint-disabled two-table mode is exercised without claiming cross-table atomicity. AWS `ENDPOINT_TYPE S3_TABLES` and catalog-vended credential integration are not claimed. | -| StarRocks Iceberg REST catalog | Documented, not automated | External catalog read-path reference only. Write compatibility is not claimed. | -| Databend | Manual/live harness | RustFS can generate an S3 stage read probe for table data files. RustFS does not claim Databend Iceberg REST Catalog integration yet. | -| Snowflake Open Catalog / Iceberg integrations | Generated harness | RustFS can generate an operator-adapted external volume/catalog SQL template. Live RustFS interoperability is not claimed. | +| PyIceberg | Automated | Namespace and table create, append, reload, scan, metadata-location, refs, views, maintenance, diagnostics, optional vended credentials with an exact-prefix data-plane scope check. | +| Spark Iceberg REST catalog | Manual/live harness | Pinned package inputs, catalog properties, SQL, expected `row_count=2`, and a CI opt-in gate for create/append/refresh/count/cleanup. Live execution and commit-conflict probing remain manual unless enabled in the runner. | +| Trino Iceberg REST catalog | Manual/live harness | Catalog properties and a read-only `SELECT COUNT(*)` against a PyIceberg- or Spark-created table. Write compatibility not claimed. | +| DuckDB Iceberg 1.5.5 | Automated | `duckdb_smoke.py` covers metadata-location read, single-table create/insert/update/delete/merge, schema evolution, snapshots, concurrent writers, drop, PyIceberg cross-read, and both signing profiles. Staged create, purge-on-drop, and format v3 are verified fail-closed. Two-table mode runs without claiming cross-table atomicity. AWS `ENDPOINT_TYPE S3_TABLES` and vended-credential integration not claimed. | +| StarRocks Iceberg REST catalog | Documented, not automated | External catalog read-path reference only. | +| Databend | Manual/live harness | S3 stage read probe for table data files only; Iceberg REST integration not claimed. | +| Snowflake Open Catalog / Iceberg integrations | Generated harness | Operator-adapted external volume/catalog SQL template only; live interop not claimed. | ## Live Evidence And Operations Matrix -| Area | Status | Current RustFS claim | +| Area | Status | Claim | |---|---|---| -| Live conformance evidence | Automated for PyIceberg and DuckDB | `engine_compatibility.py --print-live-evidence-schema` defines the required evidence schema and claim promotion boundaries. `pyiceberg_smoke.py --live-evidence-output` and `duckdb_smoke.py --live-evidence-output` write validated client evidence records after successful live smoke runs. | -| Production operations guide | Generated harness | `engine_compatibility.py --print-operations-guide` records command, evidence, pass criteria, and fail-closed signals for live conformance, durable backing cutover, maintenance, recovery, permissions, credential vending, and unsupported-claim governance. | -| Vendor compatibility gap audit | Generated harness | `engine_compatibility.py --print-vendor-audit` records provider source URLs, catalog path and warehouse shapes, signing/auth models, error/permission/maintenance validation categories, and not-claimed boundaries for AWS S3 Tables, MinIO AIStor Tables, Cloudflare R2 Data Catalog, and Alibaba OSS Tables. | -| Client claim promotion | Automated for scoped clients | PyIceberg and DuckDB claims remain bounded by their repeatable smoke entrypoints and recorded versions. Spark can be promoted only with recorded manual/live evidence; Trino remains read-only; Snowflake and vendor profiles remain reference-only without repeatable live evidence. | +| Live conformance evidence | Automated for PyIceberg and DuckDB | `engine_compatibility.py --print-live-evidence-schema` defines the evidence schema and promotion boundaries; both smoke scripts write validated evidence records via `--live-evidence-output`. | +| Production operations guide | Generated harness | `engine_compatibility.py --print-operations-guide` records commands, evidence, pass criteria, and fail-closed signals for conformance, cutover, maintenance, recovery, permissions, credential vending, and claim governance. | +| Vendor compatibility gap audit | Generated harness | `engine_compatibility.py --print-vendor-audit` records provider URLs, path and warehouse shapes, auth models, validation categories, and not-claimed boundaries for the four vendor profiles. | +| Client claim promotion | Automated for scoped clients | PyIceberg and DuckDB claims are bounded by their smoke entrypoints and recorded versions; Spark needs recorded live evidence; Trino stays read-only; Snowflake and vendor profiles stay reference-only. | ## Catalog API Matrix | Area | Status | Covered behavior | |---|---|---| -| Catalog config | Supported | `GET /v1/config` advertises RustFS catalog defaults and only the supported OpenAPI REST paths in `endpoints`. RustFS administration, maintenance, migration, diagnostics, refs, and metadata-location extensions remain available but are not presented as standard Iceberg REST endpoints. | -| Table bucket discovery | Supported | `PUT` and `GET /v1/buckets/{warehouse}` enable and inspect table bucket state. | -| Namespaces | Supported | Create, list, load, existence check, and drop namespace routes are registered on both catalog prefixes. List responses support Iceberg REST `pageSize`/`pageToken` pagination with context-bound tokens and bounded catalog-store reads. Namespace identifiers are limited to 512 ASCII characters so persisted paths and stateless continuation tokens remain bounded. | -| Tables | Supported | Create, register, list, load, existence check, rename, commit, metadata-location get/update, and drop table routes are registered on both catalog prefixes. Object-backed rename uses a bucket-scoped persistent fence, recoverable intent, and conditional publication of the destination, source tombstone, and warehouse index; the source identifier is reusable only through an ETag-conditional tombstone replacement. Table and view listings support Iceberg REST `pageSize`/`pageToken` pagination with context-bound tokens and bounded catalog-store reads. Commit identifiers must match the URL resource; unknown requirements, updates, and snapshot operations fail as bad requests; staged create, register overwrite, purge-on-drop, and v3-only encryption-key updates return an explicit unsupported-operation response. Standard statistics, partition statistics, and schema/spec cleanup updates are accepted. | -| Commit CAS | Supported | Single-table commits validate base metadata, expected version token, referenced object existence, warehouse scope, and Iceberg commit requirements before advancing the current metadata pointer. Externally supplied metadata transitions preserve monotonic column, partition, and sequence assignment watermarks and immutable definitions for retained schemas, partition specs, sort orders, and snapshots. Standard commits preserve the normal commit-token file name and use an immutable-table-scoped fallback when rename followed by source-name reuse would otherwise collide at the same generation and commit ID. The catalog does not advertise `idempotency-key-lifetime`; clients must treat standard mutation-wide `Idempotency-Key` semantics as unsupported. | -| Commit recovery | Supported | Commit log, idempotency lookup, diagnostics, and recovery routes expose staged/finalization gaps and repair safe idempotency gaps without moving the table pointer. | -| Snapshot refs | Supported | Refs can be listed, created or replaced, and deleted through catalog commits. `main` is protected and refs with explicit retention require forced delete. | -| Iceberg views | Supported | Basic create, list, load, replace, existence check, and drop routes persist view metadata with view-scoped authorization. Replace identifiers must match the URL resource, `schema-id: -1` resolves to the last added schema, one commit timestamp is used consistently, and only Iceberg view format version 1 is accepted. | -| LoadTable and table credentials endpoint | Supported | LoadTable keeps the client-provided mode unless the request negotiates `vended-credentials`. Successful vending returns one temporary session for both the table warehouse prefix and the exact current metadata location. Missing credential permission falls back to metadata-only LoadTable with an explicit reason; issuer failures remain errors. Negotiated and dedicated credential responses set `Cache-Control: no-store, private`, `Pragma: no-cache`, and `Expires: 0`. | -| Catalog diagnostics and export | Supported | Exposes recovery state, consistency state, backing manifest, recoverable commit-log WAL state, strong backing migration target, single-active-writer policy, and scale validation matrix. | -| Catalog import and rollback | Supported | Import/register and online rollback use catalog validation and commit paths rather than direct pointer mutation. Online rollback accepts only a forward-safe metadata target that preserves assignment watermarks and retained definitions. Restoring an older target that lowers those watermarks is an offline disaster-recovery operation and requires every writer to be stopped. | -| External catalog bridge | Supported operator path | Operator-supplied metadata pointer sync/import is supported for external catalog identity boundaries. Online vendor SDK polling and policy mirroring are not claimed. | -| Multi-table transactions | Not claimed | RustFS currently claims single-table commit atomicity only. | +| Catalog config | Supported | `GET /v1/config` advertises defaults and only the supported OpenAPI REST paths in `endpoints`; RustFS extensions (administration, maintenance, migration, diagnostics, refs, metadata-location) are not presented as standard endpoints. | +| Table bucket discovery | Supported | `PUT` / `GET /v1/buckets/{warehouse}` enable and inspect table bucket state. | +| Namespaces | Supported | Create, list, load, exists, drop on both prefixes; `pageSize`/`pageToken` pagination with context-bound tokens; identifiers limited to 512 ASCII characters. | +| Tables | Supported | Create, register, list, load, exists, rename, commit, metadata-location get/update, drop on both prefixes. Rename uses a bucket-scoped persistent fence, recoverable intent, and conditional publication; the source name is reusable only via an ETag-conditional tombstone replacement. Commit identifiers must match the URL; unknown requirements/updates fail as bad requests; staged create, register overwrite, purge-on-drop, and v3-only encryption-key updates return an explicit unsupported-operation response. | +| Commit CAS | Supported | Single-table commits validate base metadata, version token, referenced object existence, warehouse scope, and Iceberg requirements before advancing the pointer; external metadata transitions preserve monotonic assignment watermarks and immutable retained definitions. `idempotency-key-lifetime` is not advertised; mutation-wide `Idempotency-Key` semantics are unsupported. | +| Commit recovery | Supported | Commit log, idempotency lookup, diagnostics, and recovery routes expose and repair finalization gaps without moving the pointer. | +| Snapshot refs | Supported | List, create/replace, delete via commits; `main` is protected; refs with explicit retention need forced delete. | +| Iceberg views | Supported | Create, list, load, replace, exists, drop with view-scoped authorization; only view format version 1. | +| LoadTable and table credentials endpoint | Supported | Vending only on negotiated `vended-credentials`; one temporary session scoped to the warehouse prefix and current metadata location; missing credential permission falls back to metadata-only with an explicit reason; responses carry `Cache-Control: no-store, private`. | +| Catalog diagnostics and export | Supported | Recovery state, consistency, backing manifest, WAL state, migration target, single-active-writer policy, scale validation matrix. | +| Catalog import and rollback | Supported | Import/register and online rollback go through validation and commit paths. Online rollback accepts only forward-safe targets; restoring an older target that lowers watermarks is an offline disaster-recovery operation with all writers stopped. | +| External catalog bridge | Supported operator path | Operator-supplied metadata pointer sync/import. Vendor SDK polling and policy mirroring not claimed. | +| Multi-table transactions | Not claimed | Single-table commit atomicity only. | ## Data Plane And Credential Matrix | Area | Status | Covered behavior | |---|---|---| -| Table-aware S3 policy bridge | Supported | Ordinary S3 actions against table warehouse paths are checked through the table data-plane bridge so table policy cannot be bypassed by direct object access. | -| Reserved catalog protection | Supported | Catalog-reserved internal prefixes are protected from ordinary object mutation. | -| Static S3 credentials | Automated | The default PyIceberg smoke path uses configured S3 credentials for REST signing and object data-plane access. | -| Catalog-vended credentials | Automated when enabled | `rustfs-vended-credentials` verifies the returned table prefix, then checks `PutObject`, `HeadObject`, `GetObject`, and `DeleteObject` inside the prefix and denies access outside the prefix. | -| Credential lifetime | Supported | Vended credential TTL is server-side and clamped to a short-lived range. | -| No-long-term-data-credential bootstrap | Not claimed | The current credential-vending flow still uses the configured principal for catalog setup before table-scoped credentials are requested. | +| Table-aware S3 policy bridge | Supported | Ordinary S3 actions on warehouse paths are checked through the table bridge; table policy cannot be bypassed by direct object access. | +| Reserved catalog protection | Supported | Catalog-reserved prefixes are protected from ordinary object mutation. | +| Static S3 credentials | Automated | Default PyIceberg smoke path. | +| Catalog-vended credentials | Automated when enabled | `rustfs-vended-credentials` verifies the returned prefix, then checks Put/Head/Get/DeleteObject inside it and denies access outside it. | +| Credential lifetime | Supported | Server-side TTL clamped to a short-lived range. | +| No-long-term-data-credential bootstrap | Not claimed | Catalog setup still uses the configured principal before table-scoped credentials are requested. | ## Maintenance Matrix -| Capability | Status | Current RustFS claim | +| Capability | Status | Claim | |---|---|---| -| Metadata retention dry-run | Supported | Reports retained metadata and deletion candidates without moving the table pointer. | -| Metadata cleanup delete | Supported | Deletes only candidates that pass the safety window and current-pointer checks. | -| Ordinary bucket lifecycle expiry | Disabled for table buckets | Table bucket objects are excluded from ordinary lifecycle expiration, including already queued expiry work. Snapshot expiration and orphan cleanup remain catalog maintenance operations so referenced Iceberg files cannot be deleted outside publication fencing. | -| Snapshot expiration planning | Supported | Produces expiration plans with retained and candidate snapshots. | -| Snapshot expiration commit | Preview / controlled | Can manually commit safe snapshot expiration through the catalog. Stale plans fail closed. | -| Manifest/data/delete reachability cleanup | Supported | Reads manifest-list and manifest Avro references, reports reachable objects, and deletes only unreferenced table objects that pass the safety window. | -| Maintenance scheduler run endpoint | Preview / controlled | Lets an external scheduler durably queue one maintenance job per table, reuse an active queued job, and recover expired queued leases before requeuing. | -| Maintenance worker run endpoint | Preview / controlled | Supports queued-job claim, run-once execution, current-job backpressure, retry deferral, lease expiry recovery, and heartbeat updates. | -| Maintenance scheduler guardrails | Preview / controlled | Exposes disabled, paused, ready, queued-job handoff, active-job backpressure, retry deferral, quarantine boundary, recommended actions, and recent maintenance job audit timeline state for external schedulers and operators. | -| Maintenance audit events | Preview / controlled | Job reports and scheduler job summaries include structured audit events for planning, worker transitions, heartbeats, lease expiry recovery, and mutating quarantine operations. | -| Maintenance quarantine operations | Preview / controlled | Lets operators inspect, release, retry, or abandon the current quarantined maintenance job without moving the table pointer. | -| Compaction planning | Preview / controlled | Plans partition-local and sort-order-local binpack candidates for Parquet files and does not mix data files from different partition directories or sort orders in one rewrite group. | -| Delete-file or row-level compaction planning | Preview / controlled | Manifests with position or equality delete files produce machine-readable row-level planning and force the compaction report into manual review before any rewrite can run. | -| Compaction commit | Preview / controlled | Can commit a safe partition-local Parquet rewrite through the catalog while preserving Iceberg data file sort order IDs in the rewritten manifest. | -| Built-in periodic scheduler | Not claimed | Operators can trigger scheduler and worker ticks, but continuous in-process scheduling is not claimed. | -| Delete-file or row-level compaction execution | Not claimed | RustFS does not rewrite delete files or execute row-level compaction; those cases remain manual-review maintenance items. | +| Metadata retention dry-run | Supported | Reports retained metadata and deletion candidates without moving the pointer. | +| Metadata cleanup delete | Supported | Deletes only candidates passing the safety window and current-pointer checks. | +| Ordinary bucket lifecycle expiry | Disabled for table buckets | Table bucket objects are excluded from lifecycle expiration, including queued work; snapshot expiration and orphan cleanup stay inside catalog maintenance. | +| Snapshot expiration planning | Supported | Plans with retained and candidate snapshots. | +| Snapshot expiration commit | Preview / controlled | Manual commit through the catalog; stale plans fail closed. | +| Manifest/data/delete reachability cleanup | Supported | Reads manifest-list and manifest Avro references; deletes only unreferenced objects passing the safety window. | +| Maintenance scheduler run endpoint | Preview / controlled | Durably queues one job per table, reuses an active queued job, recovers expired leases. | +| Maintenance worker run endpoint | Preview / controlled | Claim, run-once, backpressure, retry deferral, lease expiry recovery, heartbeats. | +| Maintenance scheduler guardrails | Preview / controlled | Disabled/paused/ready state, handoff, backpressure, quarantine boundary, recommended actions, recent job audit timeline. | +| Maintenance audit events | Preview / controlled | Structured events for planning, worker transitions, heartbeats, lease recovery, quarantine mutations. | +| Maintenance quarantine operations | Preview / controlled | Inspect, release, retry, or abandon the quarantined job without moving the pointer. | +| Compaction planning | Preview / controlled | Partition-local and sort-order-local binpack candidates for Parquet; never mixes partitions or sort orders in one group. | +| Delete-file or row-level compaction planning | Preview / controlled | Position or equality delete files force machine-readable planning into manual review. | +| Compaction commit | Preview / controlled | Commits a safe partition-local Parquet rewrite preserving sort order IDs. | +| Built-in periodic scheduler | Not claimed | Ticks are operator-triggered; no continuous in-process scheduling. | +| Delete-file or row-level compaction execution | Not claimed | Manual-review maintenance item. | ## Recovery And Strong Backing Matrix -| Area | Status | Current RustFS claim | +| Area | Status | Claim | |---|---|---| -| Single-table CAS | Supported | The table pointer advances only through expected-token and expected-metadata-location validation. | -| Idempotent retry | Supported | Repeated commit IDs can return the already finalized result or surface recoverable finalization gaps. | -| Commit publication fencing | Supported with rolling-upgrade gate | Existing deployments retain exact object guards so older writers cannot mutate referenced files during publication. Set `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` only after every serving node supports table and table-bucket publication fences. In scalable mode, active table warehouse prefixes must not overlap, ordinary lifecycle expiry remains disabled for table buckets, and first enablement, first publication, drop, and warehouse relocation are serialized by the table-bucket fence. | -| Post-CAS finalization recovery | Supported | Diagnostics and recovery can repair stale or missing idempotency indexes without changing the current table pointer. | -| Catalog export | Supported | Exposes table state, commit recovery state, and backing migration information for operator inspection. | -| Strong backing state transfer | Supported | Object-backed table bucket, namespace, table, view, commit-log, and idempotency state can be materialized into the durable strong snapshot. The transfer is deterministic, ETag-CAS protected, idempotent after an interrupted finalization, validates candidate state through the restart decoder before publication, preserves resource-backed implicit namespaces, and fails closed when an inactive explicit namespace conflicts with active descendants or resources. Snapshot hydration requires a stable non-empty ETag, caps the encoded snapshot at 64 MiB, shares state and reload serialization across requests in one server context, and rejects disappearance or format-version rollback after observation. Configured durable-strong mode rejects a missing snapshot on its first catalog access after startup; only object-backed migration may initialize an empty target. | -| Durable backing migration preflight | Supported | `GET /iceberg/v1/{warehouse}/catalog/migration` and the `/_iceberg/v1` alias inspect object-backed catalog inventory, recovery blockers, warehouse prefix index readiness, active table/view identifier collisions, persistent write-fence state, target snapshot agreement, and whether every table bucket is ready for cutover. | -| Durable backing migration execution | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/migration` fences table-bucket registry changes, acquires a persistent per-bucket write fence, records whether a global strong snapshot existed before publication, drains in-flight catalog mutations, materializes the target snapshot, and reports `ready_to_enable_durable_strong`. Retries and `DELETE` may restore a known-absent initial target after an ambiguous first write, but fail closed if a previously existing or materialized global snapshot disappears. `DELETE` releases the bucket fence only while its target state has not advanced, and releases the registry fence after the last bucket is cancelled. Both mutations require `admin:MigrateTableCatalog`. | -| Strong snapshot rolling compatibility | Supported | Durable strong control-plane reads snapshot versions 1 and 2, writes version 1 by default, and writes version 2 only after both the requested and fleet-confirmed gates are enabled. A running process rejects any lower-format snapshot after observing a higher format. Once version 2 is fleet-confirmed, table data-plane resolution fails closed until the persisted snapshot is version 2; a missing table-bucket entry also fails closed instead of bypassing table-aware authorization. | -| Disaster recovery rehearsal | Manual/live harness | `failure_coverage.py --print-disaster-recovery-rehearsal` generates an operator runbook covering catalog export, diagnostics, safe recovery repair, rollback/import, durable backing migration dry-run, post-recovery loadTable, and table data-plane policy probes. | -| Scale and fault rehearsal | Manual/live harness | `failure_coverage.py --print-scale-fault-rehearsal` generates an opt-in runbook for concurrent writer stress, maintenance scheduler lease recovery, durable backing cutover preflight, recovery/rollback/import under load, and post-run evidence capture. | -| Durable strong snapshot backing cutover | Preview / controlled | Operators can select the ETag-CAS snapshot backing with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong` only after every table bucket reports `SNAPSHOT_MATERIALIZED` and `ready_to_enable_durable_strong: true`. This mode does not claim a separate external KV/WAL service, and object-only advanced operations fail closed. Version 1 backing manifests retain the legacy `STRONG_KV_WAL` and `CUT_OVER_LINEARIZABLE_READS` wire labels for client compatibility; those labels do not expand the implementation claim. | +| Single-table CAS | Supported | Pointer advances only through expected-token and expected-metadata-location validation. | +| Idempotent retry | Supported | Repeated commit IDs return the finalized result or surface recoverable finalization gaps. | +| Commit publication fencing | Supported with rolling-upgrade gate | Exact object guards protect referenced files during publication. Set `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` only after every serving node supports table and table-bucket fences. In scalable mode, warehouse prefixes must not overlap and enablement, first publication, drop, and relocation are serialized by the table-bucket fence. | +| Post-CAS finalization recovery | Supported | Repairs stale or missing idempotency indexes without changing the pointer. | +| Catalog export | Supported | Table state, commit recovery state, and backing migration information. | +| Strong backing state transfer | Supported | Object-backed catalog state is materialized into the durable strong snapshot deterministically, ETag-CAS protected, idempotent after interrupted finalization, and validated through the restart decoder before publication; conflicts between inactive explicit namespaces and active descendants fail closed. Hydration requires a stable non-empty ETag, caps the snapshot at 64 MiB, and rejects disappearance or format-version rollback after observation. Configured durable-strong mode rejects a missing snapshot on first access; only object-backed migration may initialize an empty target. | +| Durable backing migration preflight | Supported | `GET /iceberg/v1/{warehouse}/catalog/migration` (and the alias) reports inventory, recovery blockers, prefix-index readiness, identifier collisions, fence state, target agreement, and per-bucket cutover readiness. | +| Durable backing migration execution | Preview / controlled | `POST /iceberg/v1/{warehouse}/catalog/migration` fences registry changes, acquires a persistent per-bucket write fence, drains in-flight mutations, materializes the snapshot, and reports `ready_to_enable_durable_strong`. `DELETE` cancels only while the target has not advanced. Both mutations require `admin:MigrateTableCatalog`. Procedure: [s3-tables-cutover-runbook.md](../operations/s3-tables-cutover-runbook.md). | +| Strong snapshot rolling compatibility | Supported | Reads snapshot versions 1 and 2; writes version 1 by default and version 2 only after both `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2` and `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2_FLEET_CONFIRMED` are set. A process rejects a lower format after observing a higher one; once v2 is fleet-confirmed, data-plane resolution fails closed until the persisted snapshot is v2. | +| Disaster recovery rehearsal | Manual/live harness | `failure_coverage.py --print-disaster-recovery-rehearsal`. | +| Scale and fault rehearsal | Manual/live harness | `failure_coverage.py --print-scale-fault-rehearsal`. | +| Durable strong snapshot backing cutover | Preview / controlled | `RUSTFS_TABLE_CATALOG_BACKING=durable-strong` only after every table bucket reports `SNAPSHOT_MATERIALIZED` and `ready_to_enable_durable_strong: true`. No separate external KV/WAL service is claimed; the legacy `STRONG_KV_WAL` and `CUT_OVER_LINEARIZABLE_READS` labels in v1 manifests are wire-compatibility labels, not claims. | | Single active writer region | Supported policy | Diagnostics publish single-active-writer semantics and read-only replica limits. | -| Active-active multi-region writes | Not claimed | A table must not accept independent concurrent writers in multiple active regions. | - -## Durable Backing Cutover Runbook - -Use the migration dry-run before changing the table catalog backing for a -warehouse: - -1. Take an object-backed catalog backup and record the current metadata pointer - and version token for representative tables. -2. Run `GET /iceberg/v1/{warehouse}/catalog/migration` with a principal that has - `GetTableCatalogAction` on each table bucket. Treat every `blockers` entry as - fail-closed; repair commit recovery state and backfill the warehouse prefix - index before continuing. -3. Before the migration `POST`, drain every catalog writer that predates the - durable-backing migration fence and restart it on a fence-aware release. An - older writer does not recognize the persisted fence and can otherwise - mutate the object-backed source after the snapshot inventory is captured. - Keep all catalog writers on the fence-aware release until cutover completes. -4. Inventory object-only advanced operations, including maintenance workers, - catalog recovery, export, diagnostics, and external catalog bridge writes. - Quiesce mutating operations before cutover and confirm that each required - operation is supported by durable-strong mode; unsupported operations fail - closed after cutover rather than continuing against object-backed state. -5. Run `POST /iceberg/v1/{warehouse}/catalog/migration` with - `admin:MigrateTableCatalog`. This acquires the exclusive migration fence to - drain in-flight fence-aware mutations, persists the source fence while - exclusivity is held, and then copies the catalog state. -6. Repeat the preflight and materialization for every table bucket. Do not set - `RUSTFS_TABLE_CATALOG_BACKING=durable-strong` until the preflight reports - `SNAPSHOT_MATERIALIZED`, no blockers, and - `ready_to_enable_durable_strong: true`. -7. Restart with durable strong backing enabled, then verify catalog config, - table and view loads, commit idempotency, and table data-plane policy - resolution before admitting writers. -8. Before restarting into durable-strong mode, `DELETE` on the migration - endpoint can remove a migration-created target bucket snapshot and release - the source fence. After the durable-strong state advances, cancellation - fails closed; recovery requires an operator-selected restore or reverse - migration instead of restarting against the stale object-backed pointer. -9. Preserve the object-backed catalog backup until durable strong backing has - passed the operator's retention window. -10. Keep strong snapshot writes on version 1 during a rolling binary upgrade. - After every catalog writer can read version 2, set both - `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2=true` and - `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2_FLEET_CONFIRMED=true`, then restart - the catalog writers. Perform a controlled catalog write or migration - materialization and confirm that the persisted snapshot is version 2 before - serving table data-plane traffic. Setting only one gate does not change the - write format. -11. After any version 2 snapshot is persisted, do not roll catalog writers back - to a binary that only reads version 1. Current binaries preserve version 2 - even when the gates are later disabled. A running process rejects restored - version 1 content after observing version 2, but cannot distinguish an older - snapshot with the same format version from a deliberate restore. The format - high-water mark is process-local: restoring any older snapshot and restarting - every writer is a privileged disaster-recovery rollback that cannot be - inferred from the restored object alone. Recovery must restore a compatible - binary and a snapshot selected through the operator recovery procedure. -12. Migration preflight rejects an active table/view identifier collision before - it writes a migration fence. A pre-existing version 1 strong snapshot with - such a collision is loaded in cleanup-only quarantine. Ambiguous reads fail - closed; each cleanup mutation must reduce the collision set, and unrelated - writes remain blocked until all collisions are removed. Drain catalog - writers that predate cleanup quarantine before starting this repair, and - complete cleanup before the first version 2 write. Restoring any version 1 - snapshot after a writer has observed version 2 fails closed instead of - replacing the in-process catalog state. +| Active-active multi-region writes | Not claimed | A table must not accept independent concurrent writers in multiple regions. | ## Production Failure Coverage -Positive client smoke proves a client can use a table. Production failure probes -prove RustFS does not silently advance table state when a failure happens. +Failure probes prove RustFS does not silently advance table state on failure. Tracked cases: stale commit token or base metadata returns a conflict without advancing the pointer; missing metadata, manifest, data, or delete objects fail closed before commit or maintenance; concurrent writers produce one CAS winner and retryable conflicts; catalog and S3 permission denials prevent data-plane bypass; stale maintenance plans fail closed before deletion or commit; post-CAS finalization gaps are visible and safely recoverable; external catalog sync conflicts leave pointer, token, and generation unchanged; backing migration stays blocked until WAL and recovery replay are clean. -The tracked failure cases are: - -- stale commit token or stale base metadata returns a conflict without advancing - the table pointer -- missing metadata, manifest, data, or delete objects fail closed before commit - or maintenance can advance state -- concurrent writers produce a single winning CAS and retryable conflicts for - stale writers -- table catalog and ordinary S3 permission denials prevent data-plane bypass -- stale maintenance plans fail closed before object deletion or catalog commit -- post-CAS finalization gaps are visible through diagnostics and safe recovery -- external catalog sync conflicts leave pointer, token, and generation unchanged -- backing migration remains blocked until WAL and recovery replay are clean - -Do not promote a failure case from a required live probe or load test to an -automated claim until the exact RustFS build, client version, and expected -response shape are recorded. +Do not promote a failure case from live probe or load test to an automated claim until the exact RustFS build, client version, and expected response shape are recorded. ## Unsupported Or Not Claimed -RustFS does not currently claim: +Full AWS S3 Tables control-plane parity; full MinIO AIStor private extensions; full Cloudflare R2 Data Catalog or Alibaba OSS Tables interoperability; built-in periodic maintenance scheduling; active-active multi-region writes; multi-table transactions; no-long-term-data-credential bootstrap; online vendor SDK polling; external catalog policy mirroring; delete-file rewrite or row-level compaction execution; built-in SQL execution; Delta Lake or Hudi; end-to-end SQL row-level DML validation through Spark, Trino, or another engine. -- full AWS S3 Tables control-plane API parity -- full MinIO AIStor Tables private extension parity -- full Cloudflare R2 Data Catalog interoperability -- full Alibaba OSS Tables interoperability -- built-in periodic maintenance scheduling; external schedulers can queue maintenance jobs and workers can claim them, but RustFS does not claim a continuous in-process scheduler -- active-active multi-region table writes -- multi-table transactions -- no-long-term-data-credential table bootstrap -- online external catalog vendor SDK polling -- external catalog policy mirroring -- delete-file rewrite or row-level compaction execution -- built-in SQL query execution -- Delta Lake or Hudi table format support -- end-to-end SQL row-level DML validation through Spark, Trino, or another SQL engine +## Verification -## Verification Commands - -Use these commands when updating this matrix, release notes, or client -compatibility claims: - -```bash -python3 scripts/table-catalog/test_pyiceberg_smoke.py -python3 scripts/table-catalog/test_engine_compatibility.py -python3 scripts/table-catalog/test_duckdb_smoke.py -python3 scripts/table-catalog/test_failure_coverage.py -python3 scripts/table-catalog/pyiceberg_smoke.py --print-client-matrix -python3 scripts/table-catalog/pyiceberg_smoke.py --print-engine-compatibility -python3 scripts/table-catalog/pyiceberg_smoke.py --print-production-failure-coverage -python3 scripts/table-catalog/pyiceberg_smoke.py --print-vendor-profiles -python3 scripts/table-catalog/pyiceberg_smoke.py --print-production-readiness -python3 scripts/table-catalog/engine_compatibility.py --print-vendor-audit -python3 scripts/table-catalog/engine_compatibility.py --print-spark-config -python3 scripts/table-catalog/engine_compatibility.py --print-duckdb-rest-sql -python3 scripts/table-catalog/engine_compatibility.py \ - --profile aws-s3tables \ - --region us-east-1 \ - --account-id 123456789012 \ - --table-bucket analytics \ - --print-spark-config -python3 scripts/table-catalog/engine_compatibility.py \ - --metadata-location s3://rustfs-s3table-smoke/tables/table-id/metadata/v1.metadata.json \ - --print-live-conformance \ - --cleanup -python3 scripts/table-catalog/engine_compatibility.py --print-live-evidence-schema -python3 scripts/table-catalog/pyiceberg_smoke.py \ - --endpoint http://127.0.0.1:9000 \ - --bucket rustfs-s3table-smoke \ - --replace \ - --cleanup \ - --rustfs-build rustfs-v1.0.0-beta.8 \ - --git-sha "$(git rev-parse HEAD)" \ - --catalog-backing durable-strong \ - --live-evidence-output /tmp/rustfs-pyiceberg-live-evidence.json -python3 scripts/table-catalog/engine_compatibility.py \ - --warehouse rustfs-s3table-smoke \ - --namespace smoke \ - --table events \ - --print-operations-guide -python3 scripts/table-catalog/failure_coverage.py \ - --warehouse rustfs-s3table-smoke \ - --namespace smoke \ - --table events \ - --print-failure-probes -python3 scripts/table-catalog/failure_coverage.py \ - --warehouse rustfs-s3table-smoke \ - --namespace smoke \ - --table events \ - --table-warehouse-location s3://rustfs-s3table-smoke/tables/table-id \ - --print-disaster-recovery-rehearsal -python3 scripts/table-catalog/failure_coverage.py \ - --warehouse rustfs-s3table-smoke \ - --namespace smoke \ - --table events \ - --table-warehouse-location s3://rustfs-s3table-smoke/tables/table-id \ - --writer-count 8 \ - --maintenance-worker-count 2 \ - --iteration-count 50 \ - --print-scale-fault-rehearsal -``` +Commands for updating this matrix, release notes, or client claims are maintained in [scripts/table-catalog/README.md](../../scripts/table-catalog/README.md); the unit tests are `scripts/table-catalog/test_*.py`. ## Release Claim Guidance -Use conservative release wording that matches the matrix. +Acceptable: "RustFS includes a core Iceberg REST Catalog-based S3 Tables implementation with PyIceberg and DuckDB smoke coverage, table-aware S3 data-plane policy checks, controlled maintenance, catalog recovery diagnostics, manual conformance input for Spark, Trino, Databend, and Snowflake, production-failure probe harnesses, disaster-recovery and scale/fault rehearsal probes, and a machine-readable operations evidence guide." -Acceptable wording: +Do not claim: "RustFS is fully compatible with AWS S3 Tables." -> RustFS includes a core Iceberg REST Catalog-based S3 Tables implementation -> with PyIceberg and DuckDB smoke coverage, table-aware S3 data-plane policy checks, -> controlled maintenance, catalog recovery diagnostics, manual conformance -> input for Spark, Trino, Databend, and Snowflake, production-failure -> probe harnesses, disaster-recovery and scale/fault rehearsal probes, and a -> machine-readable production operations evidence guide. - -Do not claim: - -> RustFS is fully compatible with AWS S3 Tables. - -Any stronger vendor or engine claim needs a repeatable live validation harness, -the exact client versions used, and the expected response shapes recorded in the -table-catalog inventories. +Any stronger vendor or engine claim needs a repeatable live harness, the exact client versions, and the expected response shapes recorded in the table-catalog inventories. ## Related - [Table catalog conformance scripts](../../scripts/table-catalog/README.md) +- [Durable backing cutover runbook](../operations/s3-tables-cutover-runbook.md) - [Admin route action snapshot](admin-route-action-snapshot.md) - [Runtime capability contracts](runtime-capability-contracts.md) diff --git a/docs/architecture/scanner-heal-admission.md b/docs/architecture/scanner-heal-admission.md deleted file mode 100644 index f52998fe8..000000000 --- a/docs/architecture/scanner-heal-admission.md +++ /dev/null @@ -1,29 +0,0 @@ -# Scanner/Heal admission Phase 0 baseline - -This document records the current entry points and safety boundaries for backlog #1939. It is an inventory and test contract, not a lease design. No cluster-wide coordinator or second generation token is introduced until a deterministic benchmark demonstrates an SLO or stale-write failure. - -## Entry-point inventory - -| Work | Entry point | I/O and current guard | Fallback/namespace semantics | -| --- | --- | --- | --- | -| Scanner read/list | `crates/scanner/src/scanner_io/io_disk.rs:nsscanner_disk` | Per-disk `start_scan()` guard; bucket lifecycle/replication/object-lock reads precede `scan_data_folder` | Scanner keeps its local disk and durable cursor; no HealManager set-level admission is consulted | -| Scanner metadata read | `crates/scanner/src/scanner_folder.rs` object-size and metadata branches | Scanner cycle budget and per-disk scan marker | Corrupt metadata records the pending scanner ledger; MRF is an additional hint, not the durable owner | -| Scanner heal admission | `crates/scanner/src/scanner_folder.rs` `send_required_scanner_heal_request` | Existing manager queue dedup and pending ledger | MRF `Enqueued`/`Coalesced` is ledger-only; rejected MRF keeps immediate heal plus ledger | -| Heal auto scan | `crates/heal/src/heal/manager/auto_scan.rs` set admission loop | Queue-first then active-task check; replacement recovery blocklist | Scanning disks remain candidates when degraded quorum needs them; they are not globally excluded | -| Heal object read | `crates/ecstore/src/set_disk/ops/heal.rs` `heal_object` | Namespace write lock unless `no_lock`; reads file info before commit | Namespace lock is object-scoped and does not claim scanner cycle ownership | -| Disk selection | `crates/ecstore/src/set_disk/ops/locking.rs` candidate selection | Healing disks are ordered after new disks; scanning disks may remain candidates | Degraded/quorum fallback is preserved | -| Data movement | Existing storage-owned movement/publication generation (#1905/#1942) | This issue does not add a second coordinator | Future admission must validate the storage generation at the final commit | - -## Baseline contract - -The deterministic baseline in `scanner_heal_admission_baseline.rs` encodes the investigation matrix only: ScannerRead+HealRead may overlap, HealWrite conflicts with scanner reads, DataMovementWrite conflicts with all work, and independent set identities remain concurrent. It does not claim that production currently enforces the matrix. - -The production facts that must be measured before Phase 1 are scanner p99, heal p99, cursor/checkpoint delay, queue and pending-ledger depth, and starvation by set. The benchmark matrix must include restart recovery, degraded quorum/scanning-disk fallback, urgent replacement heal, and at least two independent sets. - -The executable fixture uses a fixed eight-sample restart/degraded sequence so the baseline is reproducible without wall-clock noise: two sets each receive ScannerRead, HealRead, HealWrite and a follow-up ScannerRead. Its expected synthetic p99 is 420 microseconds, maximum modeled backlog is 2, two HealWrite samples are deferred, and the independent second set still services three reads. These are fixture values, not production SLO claims; production benchmark output must replace them with measured p99, backlog and per-set wait distributions. - -The inventory test reads the current source files and asserts the named guards/fallback branches are still present (`start_scan`, pending-ledger admission, Heal queue/active checks, namespace `get_write_lock`, and scanning-disk re-append). A source rename or guard removal therefore fails the baseline instead of silently leaving stale documentation. - -Commit-time generation-fencing, lease-expiry, and lock-order tests are intentionally deferred until a Phase-0 fixture demonstrates a stale write or an SLO violation; arithmetic-only placeholders would stay green if production paths regressed. - -If a future fixture demonstrates stale destructive writes, the fix must extend the storage-owned generation/admission primitive and validate the token at the final metadata/format/delete commit. Cancellation or a local lease alone is not a fence. diff --git a/docs/architecture/storage-control-data-plane.md b/docs/architecture/storage-control-data-plane.md index 4a47cb29e..b00ca1d51 100644 --- a/docs/architecture/storage-control-data-plane.md +++ b/docs/architecture/storage-control-data-plane.md @@ -1,7 +1,7 @@ # Storage, Control Plane, And Background Controllers -This document defines migration boundaries for the storage hot path and adjacent -control-plane responsibilities. +**Use this when:** adding a storage API surface, a cluster read model, or a background-service status/reconcile surface, and you need to know which layer owns it and what must not drift. +**Source of truth:** `crates/storage-api` (trait contracts), `crates/ecstore/src/api/mod.rs` (facade groups, `api::cluster`), `crates/ecstore/src/cluster/` (control plane), [background-controller-contract.md](background-controller-contract.md) (controller vocabulary). ## Storage API Contracts @@ -42,9 +42,11 @@ It maps existing endpoint pools into the shared storage-api topology contract an an ECStore-owned static membership snapshot. It must not expose local disk paths, start health checks, mutate endpoint ownership, or change placement/readiness. The same facade also owns static pool-state, local-node storage, and peer-health -status projections. Peer health remains explicitly unknown until a later slice -wires real health signals; this document does not authorize background probes or -RPC-based health checks. +status projections. `peer_health_snapshot` in +`crates/ecstore/src/cluster/control_plane.rs` projects the internode health +tracker's per-node reachability (`PEER_HEALTH_REACHABLE` / +`PEER_HEALTH_UNREACHABLE`, or not-reported); the facade itself starts no probes +and issues no RPC-based health checks. Readiness impact for storage, lock quorum, peer health, probes, admin routes, RPC, and the S3 data plane is recorded in [`readiness-matrix.md`](readiness-matrix.md). diff --git a/docs/architecture/unified-object-generation.md b/docs/architecture/unified-object-generation.md index a5bf7b4e5..dba6aaf6a 100644 --- a/docs/architecture/unified-object-generation.md +++ b/docs/architecture/unified-object-generation.md @@ -1,384 +1,112 @@ -# Unified Per-Object Generation Authority +# Object Transaction UUID And Generation-Fencing Contract -Establishes a **single per-object generation authority** that spans object -commit, GET snapshots, garbage collection, and quota accounting, and pins the -transport, encoding, proto-evolution, and mixed-version contracts that every -consumer must obey. +**Use this when:** adding or changing anything that fences a commit, scopes a read lease, gates old-directory cleanup, binds prepared pool reads, or settles quota against "the current version of an object", or when adding a field that rides internode RPC or `xl.meta`. +**Source of truth:** `assign_object_transaction_epoch` in `crates/ecstore/src/set_disk/ops/object.rs` and `crates/ecstore/src/set_disk/ops/multipart.rs`; `FileInfo::set_object_transaction_epoch` in `crates/filemeta/src/fileinfo.rs`; `commit_rename_data_dir` and `RenameConvergence` in `crates/ecstore/src/set_disk/core/io_primitives.rs`; `PreparedPoolReadFallbackBarrier` in `crates/ecstore/src/store/rebalance.rs`; `crates/protos/src/node.proto`; env constants in `crates/config/src/constants/object.rs` and `crates/config/src/constants/internode.rs`. -This is a **design and contract document**. It changes no storage code. It is -the shared prerequisite for five implementation sub-issues under the -[#1307](https://github.com/rustfs/backlog/issues/1307) adversarial-review -program: -[#1312](https://github.com/rustfs/backlog/issues/1312) (commit fencing), -[#1313](https://github.com/rustfs/backlog/issues/1313) (read lease), -[#1314](https://github.com/rustfs/backlog/issues/1314) (prepared pool read), -[#1318](https://github.com/rustfs/backlog/issues/1318) (quota reservation), and -[#1323](https://github.com/rustfs/backlog/issues/1323) (old-dir GC). +Design tracking lives in `rustfs/backlog#1326`. This document holds only the invariants. -Tracks [rustfs/backlog#1326](https://github.com/rustfs/backlog/issues/1326). +## Authority -## Why one authority +The target contract requires **one per-object commit identity** consumed by commit fencing, read leases, cleanup, prepared reads, and quota settlement. No consumer may mint a second value and call it the same generation. -The #1307 adversarial-review verdict (issuecomment-4992565957) found that the -five sub-issues each reach for their own generation / fencing / lease token to -solve the same underlying problem — **commit mutual-exclusion plus snapshot -lifetime**. Left independent, they diverge and punch through one another: +What exists today is an **object transaction UUID**, not the target authority: -- #1323 old-dir GC can reclaim a directory still referenced by a #1313 lease if - the two disagree on what "current generation" means. -- #1312 fence epoch and #1318 quota reservation token, if derived from two - different monotonic sources, cannot be compared — a late commit fenced on one - plane can still settle quota on the other. +| Property | Current implementation | +|---|---| +| Minting | `assign_object_transaction_epoch` mints a random non-nil UUID for PUT and CompleteMultipartUpload when the object-transaction gate is active. | +| Persistence | Written through `FileInfo::set_object_transaction_epoch` into the version's internal metadata map under the dual-key contract (`x-rustfs-internal-*` / `x-minio-internal-*`). | +| Fence check | The coordinator reads the current UUID (or `Absent`) and revalidates exact equality immediately before `rename_data`. | +| Cleanup | Old-data cleanup receipts carry the committed UUID; reconciliation deletes only when the receipt UUID still equals the current object UUID. | -The fix is a single authority with one selected comparison rule, one persistence -semantics, and one transport binding, that every consumer references rather than -re-derives. +This is an equality-CAS fence and cleanup identity. It is not a monotonic epoch, is not minted by the distributed lock grant, and is not compared atomically at each disk's `xl.meta` commit point. Documents and issues must call it the *object transaction UUID*, not proof that the generation authority exists. -## Target authority and the current bounded token +### Authority modes (one must be selected) -The target contract still requires **one per-object commit identity** consumed -by commit fencing, read leases, cleanup, prepared reads, and quota settlement. -No consumer may mint a second value and call it the same generation. - -The concrete ordering semantics are not settled, however. The original #1326 -proposal requires a total-ordered, monotonic lock-grant epoch. Current main does -not implement that proposal. PR #6077 instead implements an opaque transaction -identity: - -- `assign_object_transaction_epoch` mints a random non-nil UUID for PUT and - CompleteMultipartUpload when the object-transaction gate is active. -- The UUID is written through `FileInfo::set_object_transaction_epoch` into the - dual internal metadata map. -- The coordinator reads the current UUID (or `Absent`) and revalidates exact - equality immediately before `rename_data`. -- Old-data cleanup receipts carry the committed UUID and reconciliation deletes - only when the receipt UUID still equals the current object UUID. - -This is a useful **equality-CAS fence and cleanup identity**. It is not a -monotonic epoch, is not minted by the distributed lock grant, and is not -compared atomically at each disk's `xl.meta` commit point. Until the decision -below is made, documents and issue checklists must call it the *object -transaction UUID* rather than use it as proof that the target generation -authority exists. - -### Ordering decision required - -Before #1313, #1314, or a unified quota binding can consume the authority, one -of these contracts must be selected and tested: - -1. **Total-ordered fencing epoch.** A lock grant returns a durable per-object - `(term, counter)` (or another specified total-order type). Every disk rejects - a lower epoch at the atomic metadata commit point. The value never regresses - across lock-plane restart, failover, or minority recovery. -2. **Opaque commit-generation identity.** Consumers compare only exact identity; - no `<` / `>` semantics are permitted. The authoritative commit must perform - an atomic expected-generation CAS, and all lease, cleanup, prepared-read, and - quota contracts must be rewritten in terms of “references this exact - generation,” not “lower/newer generation.” - -The current UUID implementation proves neither a durable total order nor a -per-disk atomic expected-generation CAS, so it does not by itself decide between -these options. - -### Persistence semantics if total order is selected - -A total-ordered epoch must be **monotonic across lock-plane restart and -failover**. The distributed lock entry remains in-memory; deriving a counter -from that entry alone would reset it after restart. The chosen source therefore -must be either quorum-persisted before grant or derived from a durable term whose -full `(term, counter)` comparison cannot regress. This requirement does not -apply to an opaque UUID as an ordering rule; the opaque alternative instead -requires atomic expected-identity comparison and durable crash recovery. - -## Consumer binding contracts - -### Current implementation snapshot (2026-08-31, main@9ee7b1221) - -This table separates code that exists on current main from the target contract. -Closing an implementation issue does not imply that its token is already the -unified authority. - -| Surface | Current main | Gap against this contract | +| Mode | Contract | Persistence requirement | |---|---|---| -| PUT / CompleteMultipartUpload (#1312, PR #6077) | Owned commit tasks retain the relevant guards; an opt-in gate persists a random object transaction UUID and performs a quorum metadata equality recheck before rename | no lock-grant monotonic source; no per-disk atomic epoch/CAS comparison; the live proof is the reused remote-version-state fleet proof, not a dedicated generation capability | -| Old-data cleanup (#1323, PR #6077) | JSON receipt carries transaction UUID, old dir, and committed dir; reconciliation is gated and requires UUID equality | no generation-bound read lease is consulted, so this is crash cleanup fencing rather than the full #1313/#1323 lease lifetime contract | -| Read lease (#1313) | short-term streaming/multipart path holds the namespace read lock through EOF/drop; deterministic part-boundary coverage is tracked by PR #6887 | no cross-node generation-bound lease registry, TTL reclamation, or crash recovery | -| Prepared pool read (#1314) | PR #6889 tracks a pool-local prepared identity and fails closed/refetches when pool state changes | not merged on this snapshot; pool-local identity is not a cross-pool generation authority; black-box mixed-version/rebalance coverage remains open | -| Quota reservation (#1318) | durable per-bucket ledger plus independent snapshot-lease mutation-fence tokens; issue closed after PR #6058 | reservation and settle are not bound to the object transaction UUID; the independent fence must be reconciled with the selected authority or explicitly proven to be a separate, non-generation arbitration domain | -| Internode integrity (#1327, #1541, #1542) | v2/v3 HMAC binds audience, exact method, timestamp, nonce, canonical body digest, and receiver boot epoch; body-bound RPC policy has exact-set coverage | signature/body/replay strict switches remain default-off rollout gates; generation enforcement cannot treat an unrelated fleet-version proof as proof that these strict contracts converged | +| Total-ordered fencing epoch | A lock grant returns a durable per-object `(term, counter)`; every disk rejects a lower epoch at the atomic metadata commit point; the value never regresses across lock-plane restart, failover, or minority recovery. | Quorum-persisted before grant, or derived from a durable term whose full comparison cannot regress. The in-memory distributed lock entry alone is insufficient. | +| Opaque commit-generation identity | Consumers compare exact identity only; no `<` / `>` semantics. The authoritative commit performs an atomic expected-generation CAS; lease, cleanup, prepared-read, and quota contracts are phrased as "references this exact generation". | Atomic expected-identity comparison plus durable crash recovery. | -| Consumer | How it binds generation | Key invariant | -|---|---|---| -| #1312 commit fence | selected generation is checked at `rename`, rollback restore/delete, and cleanup mutation points using the chosen ordered or exact-CAS rule | a stale writer is rejected on **all** disks; an already-ACK'd write is never rolled back | -| #1313 read lease | lease binds the exact generation observed at read time; GC runs only after every lease referencing that generation is released | lease is visible across nodes; a crashed reader's lease is reclaimed by TTL | -| #1323 old-dir GC | cleanup job carries the committed generation; before deleting `old_dir` it confirms that no lease for the generation owning that directory remains | `old_dir != committed_dir`; a still-referenced directory is never deleted | -| #1314 prepared pool read | the `PreparedPoolRead` bundle carries the generation resolved during pool lookup; the chosen pool's reader setup reuses it only after a match | generation mismatch forces a fallback to full metadata fanout | -| #1318 quota reservation | reservation / settle record binds the exact object generation (and an ordered epoch too, if that option is selected) | a late commit cannot settle quota for a different committed generation | +The current UUID proves neither a durable total order nor a per-disk atomic CAS, so it does not decide between the modes. -### Fence coverage is three disk-write points, not one (#1312 B2) +## Consumer Binding -Checking the generation only before the `rename` fanout is insufficient. The -authoritative commit sequence is `tmp sync → data-dir rename → xl.meta commit → -directory sync` in `crates/ecstore/src/disk/local.rs`, and there are two further -detachable disk-write points in -`crates/ecstore/src/set_disk/core/io_primitives.rs`: +| Consumer | Binds generation how | Key invariant | Current state | +|---|---|---|---| +| Commit fence (PUT / CompleteMultipartUpload) | Checked at `rename`, rollback restore/delete, and cleanup mutation points using the selected rule | A stale writer is rejected on **all** disks; an already-ACK'd write is never rolled back | Opt-in UUID equality recheck before rename; no per-disk atomic comparison | +| Read lease | Lease binds the exact generation observed at read time; GC runs only after every lease on that generation is released | Lease visible across nodes; crashed reader's lease reclaimed by TTL | Streaming/multipart GET holds the namespace read lock through EOF/drop (part-boundary coverage: `#6887`); no cross-node generation-bound registry | +| Old-dir GC | Cleanup job carries the committed generation and confirms no lease owns `old_dir` before deleting | `old_dir != committed_dir`; a still-referenced directory is never deleted | UUID receipt equality (`#6077`); no lease consultation | +| Prepared pool read | The prepared bundle carries the generation resolved during pool lookup; the chosen pool reuses it only after a match | Mismatch forces fallback to full metadata fanout | `PreparedPoolReadFallbackBarrier` (`#6889`) is a pool-local identity that fails closed / refetches on pool state change; it is not a cross-pool authority | +| Quota reservation | Reserve / settle record binds the exact object generation (and the ordered epoch too, if selected) | A late commit cannot settle quota for a different committed generation | Durable per-bucket ledger with independent snapshot-lease fence tokens (`#6058`); not bound to the transaction UUID | -- **Rollback restore/delete** — on quorum failure each disk can restore backup - metadata or delete the failed version. A stale writer's rollback must compare - the expected generation, otherwise it can overwrite or delete the winner's - already-committed metadata. -- **`commit_rename_data_dir`** — a cancel-then-detach disk-write point; the - coordinator's "reap all child tasks" must explicitly include it so a cancelled - writer cannot bypass fence/lease and keep deleting directories. +## Fence Coverage: Three Disk-Write Points -If generation is validated only after data-dir rename, a fenced writer -may already have renamed its data-dir into the object path, leaving a staged -orphan. Either move the fence ahead of the data-dir rename, or declare that -orphan an acceptable residue accounted for by GC metrics — the white-box -acceptance "no background disk write after release" must be rewritten -accordingly. +Checking generation only before the `rename` fanout is insufficient. The commit sequence is `tmp sync → data-dir rename → xl.meta commit → directory sync` in `crates/ecstore/src/disk/local.rs`, and `crates/ecstore/src/set_disk/core/io_primitives.rs` has two further detachable disk-write points: -Current PR #6077 performs a quorum metadata equality recheck before rename and -reaps owned commit work. That closes important cancellation windows, but it is -not evidence that every disk mutation above performs the selected generation -comparison atomically. The writer inventory and per-point CAS/ordering proof -remain acceptance work for #1326 even though #1312 is closed. +1. **Rollback restore/delete.** On quorum failure each disk can restore backup metadata or delete the failed version. A stale writer's rollback must compare the expected generation, or it can overwrite or delete the winner's committed metadata. Panic, cancel, and timeout outcomes must be reaped into coordinator convergence rather than skip rollback through an early return. +2. **`commit_rename_data_dir`.** A cancel-then-detach disk-write point; the coordinator's "reap all child tasks" must include it so a cancelled writer cannot bypass fence or lease and keep deleting directories. -### Post-commit convergence is orthogonal to the fence (#1321) +If generation is validated only after the data-dir rename, a fenced writer may already have renamed its data-dir into the object path, leaving a staged orphan. Either move the fence ahead of the data-dir rename, or declare that orphan an accepted residue accounted for by GC metrics. -The same `SetDisks::rename_data` path already returns a post-commit -convergence classification (`RenameConvergence`, rustfs/backlog#1321) that -tells the caller whether the *committed* replicas need heal to converge — -`AllSuccessIdentical` (no heal), `PartialCommit` (a replica failed/offline), -`SignatureDivergent` (committed replicas' version signatures differ), or -`Unknown` (no signature was produced, e.g. >10 versions — scanner-backstopped). -This replaced an earlier `Option>` heuristic under which any -version signature looked like "needs heal", so every healthy multipart -completion self-enqueued. +`RenameConvergence` (`AllSuccessIdentical` / `PartialCommit` / `SignatureDivergent` / `Unknown`) is a *post-commit* heal signal on the same `rename_data` path; the fence is a *commit* gate. They compose: the fence decides whether a convergence is produced, `RenameConvergence` classifies it. A fence-aware convergence variant would be an additive enum change. -Convergence is a *post-commit* signal (the write landed; do the replicas need -reconciliation), whereas the #1312 fence is a *commit* gate (a stale epoch is -rejected before the write lands, surfaced through the existing `Result::Err` -channel). They compose on the one `rename_data` path rather than competing: -the fence decides whether a convergence is produced at all, and -`RenameConvergence` classifies it once produced. A future fence-aware -convergence variant, if ever needed, is an additive change to that enum and -does not disturb the epoch comparison at the disk-write points above. +## Transport And Security -## Transport and security contract +Generation and derived tokens (lease, reservation) cross node boundaries in internode RPC bodies; every such flow must be signature-bound. -Generation and all derived tokens (lease, reservation) cross node boundaries in -internode RPC bodies. Every such flow must be signature-bound. +| Rule | Detail | +|---|---| +| HMAC scope | Target audience, exact service/method, timestamp, nonce, canonical body digest, receiver replay (boot) epoch. The receiver consumes the nonce in a bounded replay cache; a transmitted-but-unconsumed nonce is not replay protection. | +| Current substrate | RPC v2/v3 in `crates/ecstore/src/cluster/rpc/http_auth.rs` binds all of the above. Body-bound policy covers mutating disk RPCs including `RenameData`, whose versioned canonical body includes every `RenameDataRequest` field, so the `FileInfo` metadata map carrying the UUID is authenticated. | +| Strict switches | `RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT`, `RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT`, `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT` (`crates/config/src/constants/internode.rs`) are default-off rollout gates governed by [compat-cleanup-register.md](compat-cleanup-register.md). A generation capability may claim strong transport binding only after the relevant strict modes have converged fleet-wide. | +| Acceptance tests per consumer | Method substitution, canonical body tamper, nonce replay, receiver restart, stripped-strict-metadata negatives. | -### RPC signature binding (#1312 B3, #1313, #1318) +## Encoding Rules -**Requirement.** The canonical body carrying a generation or derived token must -be folded into the internode HMAC. The authenticated scope binds the target -audience, exact service/method, timestamp, nonce, canonical body digest, and -receiver replay epoch. The receiver must consume the nonce in a bounded replay -cache; transmitting a nonce without receiver-side consumption is not replay -protection. +| Rule | Reason | +|---|---| +| **Do not bump `XL_META_VERSION` or `XL_HEADER_VERSION`** (`crates/filemeta/src/filemeta.rs`). | `decode_xl_headers` in `crates/filemeta/src/filemeta/codec.rs` rejects newer values outright; a bump makes every new `xl.meta` unreadable by rolling-upgrade old nodes and by MinIO. See [minio-file-format-compat.md](minio-file-format-compat.md). | +| **Do not add generation as a `FileInfo` struct field.** | Internode RPC serializes `FileInfo` with two msgpack encoders: positional-array encoding for the `read_version` family (a new positional field breaks mixed-version decode) and `encode_msgpack_named` (named-map) for `rename_data` in `rustfs/src/storage/rpc/node_service/disk.rs`. A field would have to be correct under both plus the JSON compatibility twin. Use the metadata map, which rides every encoder unchanged. | +| **Metadata-map dual key.** | The UUID lives under `x-rustfs-internal-*` / `x-minio-internal-*`; missing, malformed, nil, or conflicting dual values fail closed when fencing is active. | +| **No sidecar unless atomic.** | An epoch sidecar outside `xl.meta` is admissible only if it commits at the same atomic/CAS point as `xl.meta` with a specified crash-recovery protocol. None is implemented. | +| **Regression guard.** | The real-MinIO `xl.meta` interop fixtures in `crates/filemeta/src/filemeta.rs` must keep passing: objects written by a new node stay readable by old RustFS nodes and by MinIO in both upgrade directions. | -**Current substrate (verified on main).** The original legacy-only description -is obsolete: +### Wire-encoding window (JSON and msgpack) -- RPC v2 binds target audience, exact method, POST, timestamp, nonce, and body - digest. -- Body-bound policy covers mutating disk RPCs including `RenameData`; its - versioned canonical body includes every `RenameDataRequest` field, so the - `FileInfo` metadata map carrying the transaction UUID is authenticated. -- PR #5425 extended canonical-body enforcement to implemented non-disk mutating - unary RPCs and added an exact policy/handler coverage partition. -- PR #5455 added the receiver boot epoch and rotating replay scope so signatures - captured before a receiver restart are rejected after capability convergence. - -The rollout switches -`RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT`, -`RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT`, and -`RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT` remain default-off for rolling -compatibility. The compatibility register and fallback/overflow metrics govern -their fleet convergence. Therefore a generation capability may claim strong -transport binding only when the relevant strict modes have converged; the -object-transaction gate's current remote-version-state fleet proof is not, by -itself, proof of RPC signature/body/replay strictness. - -Acceptance for each generation consumer includes method substitution, canonical -body tamper, nonce replay, receiver restart, and stripped-strict-metadata -negative tests. Generation rollout must also record which strict-mode evidence -authorized enforcement. - -### Encoding contract (#1312 B1) - -The on-disk persistence of generation must not perturb the file format: - -- **Do not bump `XL_META_VERSION` / `XL_HEADER_VERSION`.** - `crates/filemeta/src/filemeta/codec.rs` rejects `meta_ver > 3` and - `header_ver > 3` outright (`decode_xl_headers`), and both constants are `3` - (`crates/filemeta/src/filemeta.rs:53-54`). Bumping either makes every new - `xl.meta` unreadable by rolling-upgrade old RustFS nodes and by MinIO — a - total read failure, not a graceful downgrade. -- **Do not add generation as a `FileInfo` struct field.** The internode RPC layer serializes `FileInfo` with two different msgpack encoders depending on the call site: `encode_msgpack` uses rmp_serde's default **array** (positional) encoding for the `read_version` family, where a new positional field breaks decode across mixed-version nodes; `encode_msgpack_named` uses `.with_struct_map()` (named-map) encoding for `rename_data` (`crates/ecstore/src/cluster/rpc/remote_disk.rs`), which is more tolerant but still requires `#[serde(default)]` and MinIO-side agreement. Because a `FileInfo` field would have to be correct under *both* encoders and under the JSON compatibility twin (see "Wire-encoding migration" below), do not add one — use the metadata map, which rides through every encoder unchanged. -- **Where it lives today.** The object transaction UUID uses the version's - internal metadata map under the dual-key contract - (`x-rustfs-internal-*` / `x-minio-internal-*`) via - `set_object_transaction_epoch`. Missing, malformed, nil, or conflicting dual - values fail closed when fencing is active. -- **Sidecars are not an equivalent alternative.** A future sidecar is admissible - only if it commits atomically with `xl.meta` and has a specified crash-recovery - protocol. No such protocol is implemented, so a sidecar cannot be selected by - an implementation issue merely because this document mentions one. -- **Regression guard.** Preserve the #4377 real-MinIO `xl.meta` interop - regression (the fixture family around `crates/filemeta/src/filemeta.rs`): - objects written by a new node must still be readable by old RustFS nodes and - by MinIO, in both upgrade and downgrade directions. - -### Wire-encoding migration (JSON → msgpack) interaction - -The internode RPC layer retains a JSON/msgpack rolling-compatibility window, and -generation-bearing fields must respect it. - -- **Dual-field transport.** Each dual-encoded RPC field exists twice in `crates/protos/src/node.proto`: a JSON `string` field and a msgpack `bytes _bin` field (e.g. `file_info` #4 alongside `file_info_bin` #7 on `RenameDataRequest`). Senders emit both; receivers `decode_msgpack_or_json` prefer the `_bin` form and fall back to the JSON string only when `_bin` is empty (`crates/ecstore/src/cluster/rpc/remote_disk.rs`). -- **Capability flags, default off.** `rustfs_protos::internode_rpc_msgpack_only()` only drops the redundant JSON copy when both `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true` and `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=true` are deliberately enabled after the JSON-fallback metric reads zero fleet-wide and the convergence runbook is followed. Generation follows the same default-off, fleet-confirmed, metric-reads-zero rollout discipline, but a msgpack proof is not itself a generation capability proof. -- **Generation must ride both encodings during the window.** If epoch lives in the version's internal metadata map, that map is carried inside `FileInfo`, so it is present in both the msgpack `_bin` and JSON copies automatically — good. But any new *top-level* generation datum must be added to **both** the msgpack and JSON representations (and, for msgpack, be safe under both the array and named-map encoders). A field added to only one encoding is silently lost the moment a peer falls back to the other — exactly the failure the JSON-fallback metric exists to catch. -- **Signature binds a canonical form.** `RenameDataRequest` now has a versioned, - injective canonical-body encoder that covers both compatibility fields and is - authenticated independently of whichever JSON/msgpack decoder branch a peer - consumes. A generation-capable strict request must reject missing or - mismatched canonical-body metadata; it must not silently downgrade to an - unauthenticated JSON twin. +- Dual-encoded RPC fields exist twice in `crates/protos/src/node.proto`: a JSON `string` field and a msgpack `bytes *_bin` field (e.g. `file_info` and `file_info_bin` on `RenameDataRequest`). Senders emit both; receivers (`decode_msgpack_or_json` in `crates/ecstore/src/cluster/rpc/remote_disk.rs`) prefer `_bin` and fall back to JSON only when `_bin` is empty. +- `rustfs_protos::internode_rpc_msgpack_only()` drops the JSON copy only when both `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY` and `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED` are set after the JSON-fallback metric reads zero fleet-wide. +- Generation inside the `FileInfo` metadata map is carried in both copies automatically. Any new *top-level* generation datum must be added to both encodings and be safe under both msgpack encoders; a field in only one encoding is silently lost when a peer falls back. +- `RenameDataRequest` has a versioned, injective canonical-body encoder covering both compatibility fields; a strict generation-capable request must reject missing or mismatched canonical-body metadata rather than downgrade to the unauthenticated JSON twin. ### Proto evolution -No top-level proto field is required by the current metadata-map UUID. If a -future ordered epoch or explicit expected-generation is added to proto, it uses -**proto3 `optional`** (explicit presence). A non-optional scalar is forbidden: -an old coordinator talking to a new disk decodes absence as a plausible zero. +No top-level proto field is required by the metadata-map UUID. If an ordered epoch or explicit expected-generation is ever added to proto, it uses **proto3 `optional`** (explicit presence). A non-optional scalar is forbidden: an old coordinator talking to a new disk decodes absence as a plausible zero. -### Mixed-version gate — one direction +## Mixed-Version Gate: One Direction -When generation enforcement is not explicitly requested, or fleet confirmation -is absent, behavior falls back to current semantics. Fail-closed is reserved for -an explicit administrator-confirmed strict rollout. +When generation enforcement is not explicitly requested, or fleet confirmation is absent, behavior falls back to current semantics. Fail-closed is reserved for an explicit administrator-confirmed strict rollout. -Current object transaction fencing follows that direction: +| Flag (`crates/config/src/constants/object.rs`) | Default | Effect | +|---|---|---| +| `RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE` | false | With either flag absent, PUT/MPU neither persists nor consumes the transaction UUID. | +| `RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED` | false | With both enabled, failure to obtain or retain the live fleet proof rejects the commit before rename. | -- `RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE` and - `RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED` both default false. -- With either flag absent, PUT/MPU does not persist or consume the transaction - UUID. -- With both flags enabled, failure to obtain or retain the live fleet proof - rejects the commit before rename. +The fleet proof is currently borrowed from the remote-version-state writer rollout. It proves membership/process-epoch convergence for that feature only; it does not prove an epoch type, per-disk CAS support, or RPC strict-mode convergence, and must not be treated as the final generation handshake. -This is an opt-in strict gate, not a negotiated generation capability. The -proof is currently borrowed from the remote-version-state writer rollout. It -proves current membership/process-epoch convergence for that feature, but does -not prove an epoch type, per-disk generation CAS support, or RPC strict-mode -convergence. Treating it as the final handshake is forbidden without an -explicit proof mapping for those properties. +### Capability negotiation (target) -## Capability negotiation +Generation enforcement requires one **live fleet proof** containing at least: the selected authority version and comparison mode; the current membership/topology fingerprint and process epochs; support for every required disk mutation point; RPC signature/body/replay strict convergence; and the on-disk encoding version (the metadata-map UUID is version 1). Membership change or an old-node rejoin revokes the proof; revocation before commit fails an explicitly strict request and never rewrites or lowers a persisted generation. The proof may extend the authenticated fleet-proof machinery in `notification_sys` or the runtime capability contract; this document requires one shared token, not a mechanism. -Generation enforcement requires one **live fleet proof**, not independent -boolean guesses in each consumer. The proof contract contains at least: +## Open Decisions -1. the selected authority version and comparison mode (ordered or exact-CAS), -2. the current membership/topology fingerprint and process epochs, -3. support for every required disk mutation point, -4. RPC signature/body/replay strict convergence, and -5. the on-disk encoding version (the current metadata-map UUID is version 1). +Blockers for calling the contract implemented: -The authoritative writer enables enforcement only while every target disk in -the set is covered by a current proof. Membership change or an old node rejoin -revokes that proof. Revocation before commit fails an explicitly strict request; -when strict generation was never requested, the request remains on the legacy -path. Revocation never rewrites or lowers an already-persisted generation. - -The existing fleet-proof machinery in `notification_sys` may be reused if its -authenticated statements are extended to cover the properties above. The -runtime capability contract may instead expose the proof. This document does -not choose the storage mechanism; it requires one token whose acquisition and -revalidation semantics are shared by all consumers. - -## Implementation order - -Some original prerequisites have landed, but not in the originally proposed -form. Remaining work follows this order: - -1. **Resolve the authority mode in #1326.** Select total order or opaque - exact-CAS, specify its atomic commit point, and audit PR #6077 against it. - Do not retrofit ordering semantics onto the existing random UUID. -2. **Define the generation fleet proof.** Map generation enablement to the RPC - signature/body/replay strict proofs delivered by #1327/#1541/#1542 and to - the selected per-disk comparison capability. Keep all strict defaults off - until fallback metrics converge. -3. **Implement #1313 generation-bound read leases.** The lease registry, - cross-node visibility, TTL, and crash recovery must exist before old-dir GC - can claim the full snapshot-lifetime guarantee. #1325 supplies the required - multi-node failure tests. -4. **Bind #1314 prepared reads.** A bundle binds the exact selected generation - within its source pool. Cross-pool ordering is forbidden until a common - authority is demonstrated. Validate rebalance and mixed-version fallback in - the #1325 multi-pool harness. -5. **Reconcile #1318 quota fencing.** Either bind reserve/settle/reconcile to - the selected object generation or document and prove that its independent - snapshot-lease fence is a separate arbitration domain that cannot settle a - different generation. -6. **Re-audit #1323 cleanup.** The existing UUID receipt remains valid crash - cleanup, but full closure against active readers requires the #1313 lease - check and the selected generation semantics. - -## Open design decisions (pin before contract closure) - -The following decisions remain blockers for calling the contract implemented: - -- **Authority mode.** Choose total order or opaque exact-CAS. If total order is - selected, define the type, per-object scope, persistence, overflow, and - never-regress restart/minority-recovery tests. If opaque identity is selected, - define the atomic expected-generation CAS and remove all ordered wording. -- **Complete xl.meta-writer coverage.** Enumerate commit rename, rollback - restore/delete, cleanup, heal, transition, restore, replication, and data - movement. Each path must compare/carry the selected generation or be proved - incapable of replacing the authoritative object identity. -- **Rollback is an expected-generation CAS (#1312 B2).** The quorum-failure - rollback in `rename_data` can restore backup metadata, not just remove a - writer-private temporary file. It must execute only when the stored generation - still matches the failed writer's expected generation. Panic, cancel, and - timeout outcomes must be reaped into coordinator convergence rather than skip - rollback through an early return. -- **Sidecar is excluded unless proven atomic.** An epoch sidecar outside `xl.meta` is only admissible if it commits at the same atomic/CAS point as `xl.meta` with a defined recovery; otherwise it opens a crash gap and must be rejected in favor of the version-internal metadata map. The earlier "metadata map or sidecar" phrasing does not treat the two as equally safe. -- **Generation capability proof.** Decide whether to extend the current - authenticated fleet proof or the runtime capability contract. It must prove - authority version, mutation coverage, topology/process epoch, and RPC strict - convergence in one revalidatable token. -- **Read-lease and GC crash recovery.** Select the cross-node registry, TTL - reclamation, lease-holder crash behavior, and GC-executor recovery. The - current cleanup receipt equality check does not answer these questions. -- **Quota reserve → commit → settle binding.** The durable ledger's idempotency - exists, but its independent mutation tokens must be related to the selected - object generation with a concrete late-settle rejection test. -- **PreparedPoolRead is pool-local only.** A #1314 bundle's generation validates freshness only within the pool that produced it. It cannot order commits across different pools unless a cross-pool common authority exists; absent that, the multi-pool wait cannot be short-circuited. -- **Hot-path cost is a blocking metric.** Measure any additional consensus - write, fsync, fleet-proof lookup, lease operation, or centralized serialization - under 4 KiB and high-concurrency hot-key/hot-bucket A/B. -- **Test infrastructure.** #1325 still lacks the complete 4-node × 4-drive, - 2-pool, directed network-fault, and large-object budget needed for restart, - mixed-version, and cross-node lease acceptance. - -## Acceptance for this contract - -- [x] Architecture document exists and is linked from the architecture index. -- [x] Transport signature, encoding, proto presence, mixed-version direction, - and capability-proof requirements are defined once. -- [x] Current implementations are separated from target guarantees; a closed - child issue is not treated as proof of unified generation binding. -- [ ] Authority mode and atomic comparison semantics are selected and tested. -- [ ] #1312 / #1313 / #1314 / #1318 / #1323 bodies reference this document and - use the selected authority terminology. -- [ ] #1313 and #1314 bind the selected generation and pass #1325 multi-node / - multi-pool failure tests. -- [ ] #1318 either binds reserve/settle to the selected generation or provides - an accepted proof that its separate fence cannot cross-settle generations. -- [ ] #1323 reconciliation checks both committed generation and active - generation-bound leases. -- [ ] Generation strict enablement is backed by one live proof that includes RPC - signature/body/replay strict convergence and per-disk comparison support. +1. **Authority mode.** Total order or opaque exact-CAS. Do not retrofit ordering semantics onto the existing random UUID. +2. **Complete `xl.meta`-writer coverage.** Enumerate commit rename, rollback restore/delete, cleanup, heal, transition, restore, replication, and data movement; each path compares/carries the selected generation or is proved incapable of replacing the authoritative identity. +3. **Rollback as expected-generation CAS.** The quorum-failure rollback in `rename_data` restores backup metadata, not just a private temp file; it must run only when the stored generation still matches the failed writer's expectation. +4. **Generation capability proof.** Extend the fleet proof or the runtime capability contract; one revalidatable token. +5. **Read-lease and GC crash recovery.** Cross-node registry, TTL reclamation, lease-holder crash behavior, GC-executor recovery. +6. **Quota reserve → commit → settle binding.** Relate the ledger's independent mutation tokens to the selected generation, with a concrete late-settle rejection test, or prove the fence is a separate arbitration domain that cannot cross-settle. +7. **Prepared reads stay pool-local.** `PreparedPoolReadFallbackBarrier` validates freshness only within the pool that produced it; cross-pool ordering requires a common authority, and the multi-pool wait cannot be short-circuited without one. +8. **Hot-path cost is a blocking metric.** Measure any added consensus write, fsync, fleet-proof lookup, lease operation, or centralized serialization under 4 KiB and hot-key/hot-bucket A/B. +9. **Test infrastructure.** Multi-node, multi-pool, directed network-fault, and large-object budget for restart, mixed-version, and cross-node lease acceptance. diff --git a/docs/architecture/workload-admission-contracts.md b/docs/architecture/workload-admission-contracts.md index 15f7ce9e5..a1bea9ea3 100644 --- a/docs/architecture/workload-admission-contracts.md +++ b/docs/architecture/workload-admission-contracts.md @@ -1,130 +1,40 @@ # Workload Admission Contracts -This document records the `rustfs/backlog#660` PR-05 and PR-07 scheduler -preservation and runtime workload-class contract slice. +**Use this when:** adding a workload class or snapshot provider, consuming admission state from a background job, or deciding whether a job can "join" admission (it cannot; see Observation Surface Only). +**Source of truth:** `WorkloadClass`, `AdmissionState`, `WorkloadAdmissionSnapshot`, `WorkloadAdmissionRegistrySnapshot`, `WorkloadAdmissionSnapshotProvider`, and `foreground_pressure` in `crates/concurrency/src/workload.rs`; the provider and consumer files named below. -## Preservation Coverage +## Contract Shapes -The `rustfs-concurrency` tests pin the current reusable admission-facing -behavior before later snapshot extraction: +`rustfs-concurrency` owns the read-only shapes. `WorkloadClass` enumerates the admission categories (the variants are the source of truth); `AdmissionState`, `WorkloadAdmissionSnapshot`, and `WorkloadAdmissionRegistrySnapshot` are status shapes for runtime owners to fill. They do not replace the scheduler, request guards, scanner, heal, replication, or ECStore placement behavior. `GetObjectQueueSnapshot` permit semantics (saturated, over-available, zero-total) and worker-slot over-release clamping are pinned by `rustfs-concurrency` tests; scheduler buffer/priority behavior is pinned by `rustfs-io-core` and `rustfs/src/storage/concurrency/` tests. -- Worker slot over-release remains clamped by the configured worker limit. -- `GetObjectQueueSnapshot` preserves saturated, over-available, and zero-total - permit semantics. +## Class To Provider Table -The former reusable scheduler and backpressure-pipe facades (and their -preservation tests) were removed as zero-caller dead code in backlog#1025; -scheduler buffer/priority behavior is now pinned by `rustfs-io-core` and -`rustfs/src/storage/concurrency` tests. +| Class | Provider (`impl WorkloadAdmissionSnapshotProvider`) | `active` / `queued` / `limit` source | Reports `Unknown` when | +|---|---|---|---| +| `ForegroundRead` | `ConcurrencyManager` in `rustfs/src/storage/concurrency/manager.rs` (source of truth); re-exposed unchanged by the RustFS runtime provider | disk-read permits in use / `None` (the semaphore exposes no waiter count) / configured max concurrent disk reads | the storage registry has no entry | +| `ForegroundWrite` | none | none | always: no write-specific admission owner exposes a read-only surface yet | +| `Metadata` | `RustFsWorkloadAdmissionSnapshotProvider` in `rustfs/src/workload_admission.rs` | `Open` once the bucket metadata runtime handle exists; no counts | bucket metadata runtime not initialized | +| `Scanner` | same | scanner active work-unit counter / none / none | the counter is zero (idle and uninitialized are indistinguishable) | +| `Repair` | same | heal active tasks / heal queue length / `None` (limits live behind the async heal manager state) | heal manager not initialized | +| `Replication` | same | active regular + large-object + MRF workers / site replication queue count / `None` (limits owned by the async pool and resize policy) | replication runtime not initialized, or queue stats currently locked | -## Workload Class Contract +## Observation Surface Only -`WorkloadClass` defines the required future admission categories: +This is an observation surface only. Permit acquisition, priority assignment, buffer sizing, storage media detection, request guards, queue capacity, heal admission and priority merge/drop policy, replication worker resize and MRF handling, scanner cycle scheduling, bucket metadata loading and locks, and object write paths are unchanged by any provider. There is no runtime admission API for a background job to join; a job that needs bounded contention must bound it itself (see [kms-bulk-rekey-contract.md](kms-bulk-rekey-contract.md)). -- Foreground read. -- Foreground write. -- Metadata. -- Scanner. -- Repair. -- Replication. +Consumers that read the snapshot to self-throttle exist, and they do not change the owners' decisions: -`AdmissionState`, `WorkloadAdmissionSnapshot`, and -`WorkloadAdmissionRegistrySnapshot` define read-only status shapes for later -runtime owners. They do not replace the current scheduler, request guard, -scanner, heal, replication, or ECStore placement behavior. +| Consumer | File | Behavior | +|---|---|---| +| Data-movement backpressure (decommission, rebalance) | `crates/ecstore/src/data_movement/backpressure.rs` (`wait_for_data_movement_admission`, `foreground_pressure`) | Delays the next data-movement step while `ForegroundRead` or `ForegroundWrite` usage exceeds the configured high-water percent. ECStore receives the provider through `set_workload_admission_snapshot_provider` (`crates/ecstore/src/lib.rs`), published from `rustfs/src/startup_background.rs`; with no provider the step is admitted immediately. | +| Heal manager mainline throttle | `crates/heal/src/heal/manager.rs` (`new_with_workload_provider`) | When `mainline_throttle_enable` is set, defers heal work while `ForegroundRead` or `ForegroundWrite` utilization exceeds the configured high-water percents; with no provider or the throttle disabled, heal pacing is unchanged. | ## Boundary Rules -- `rustfs-concurrency` owns this reusable contract surface. -- The contract does not depend on `rustfs-ecstore` or RustFS binary runtime - state. -- No scheduler decision logic, queue capacity, Tokio runtime default, scanner - admission, heal admission, replication admission, placement, membership, or - NUMA behavior changes are part of this slice. +- `rustfs-concurrency` owns the contract surface and does not depend on `rustfs-ecstore` or RustFS binary runtime state. +- Adding a class or provider changes no scheduler decision logic, queue capacity, Tokio runtime default, scanner/heal/replication admission, placement, membership, or NUMA behavior. +- Providers report `Unknown` rather than blocking or guessing when their owner is uninitialized or its stats are not immediately observable. -## Set-Local Snapshot Extraction +## Provider Composition -The RustFS storage `ConcurrencyManager` now implements -`WorkloadAdmissionSnapshotProvider` for local foreground-read admission: - -- `ForegroundRead` reports local disk-read permit usage through - `GetObjectQueueSnapshot`. -- `active` is the number of disk-read permits currently in use. -- `limit` is the configured maximum concurrent disk reads. -- `queued` remains `None` because the current semaphore does not expose waiter - counts. -- Scanner, repair, replication, foreground write, and metadata entries remain - `Unknown` until their owning runtime components expose read-only status. - -This is an observation surface only. Permit acquisition, priority assignment, -buffer sizing, storage media detection, request guards, and queue behavior are -unchanged. - -## Heal Repair Snapshot Extraction - -The RustFS integration layer now exposes a read-only repair admission snapshot -from the heal runtime counters: - -- `Repair` reports the current heal active task count. -- `queued` reports the current heal queue length. -- `limit` remains `None` because the configured heal queue and concurrency - limits live behind the async heal manager state. -- Other workload classes remain `Unknown` in this provider until their owning - runtime components expose read-only status. - -This is an observation surface only. Heal request admission, queue capacity, -priority merge/drop policy, task scheduling, retry handling, and repair -behavior are unchanged. - -## Replication Snapshot Extraction - -The RustFS integration layer now exposes a read-only replication admission -snapshot from the existing replication pool and queue statistics: - -- `Replication` reports active regular, large-object, and MRF worker counts. -- `queued` reports the current site replication queue count when queue stats - are immediately observable. -- `limit` remains `None` because replication worker limits remain owned by the - async replication pool and resize policy. -- If the replication runtime has not initialized, or queue stats are currently - locked, the snapshot reports `Unknown` instead of blocking or guessing. - -This is an observation surface only. Replication admission, queue channel -capacity, worker resize behavior, MRF handling, target dispatch, and resync -behavior are unchanged. - -## RustFS Runtime Owner Snapshot Extraction - -The RustFS integration layer now extends the workload admission registry with -additional read-only owner mappings: - -- `ForegroundRead` reuses the storage `ConcurrencyManager` disk-read permit - snapshot so the RustFS-level provider exposes the same active and limit - counts as the storage-local provider. -- `Scanner` reports the existing scanner active work-unit counter. When the - counter is zero, the snapshot remains `Unknown` because the current counter - cannot distinguish an idle scanner from a scanner that has not initialized. -- `Metadata` reports `Open` once the bucket metadata runtime handle is - available, and `Unknown` before initialization. -- `ForegroundWrite` remains `Unknown` until a write-specific admission owner - exposes a read-only surface. - -This is an observation surface only. Disk-read permit acquisition, scanner -cycle scheduling, bucket metadata loading, metadata locks, object write paths, -and queue behavior are unchanged. - -## Provider Composition Boundary - -`WorkloadAdmissionRegistrySnapshot::overlay` composes provider-owned registry -snapshots without mutating runtime owners: - -- The storage concurrency provider remains the source of truth for - `ForegroundRead`. -- The RustFS runtime owner provider overlays metadata, scanner, repair, - replication, and foreground-write status on top of the storage registry. -- Matching workload classes are replaced by the later provider snapshot; new - classes are appended without reordering existing unrelated entries. - -This keeps the later controller/status layer consuming a single read-only -registry while preserving the existing storage, scanner, heal, replication, and -metadata ownership boundaries. +`WorkloadAdmissionRegistrySnapshot::overlay` composes provider-owned registries without mutating runtime owners: the storage concurrency provider is the source of truth for `ForegroundRead`; the RustFS runtime owner provider overlays metadata, scanner, repair, replication, and foreground-write status on top; matching classes are replaced by the later snapshot and new classes are appended without reordering. `workload_admission_registry_snapshot` in `rustfs/src/workload_admission.rs` is the single composed registry consumers read. diff --git a/docs/operations/atomic-object-undo.md b/docs/operations/atomic-object-undo.md index 94944ec21..df099eee9 100644 --- a/docs/operations/atomic-object-undo.md +++ b/docs/operations/atomic-object-undo.md @@ -1,5 +1,8 @@ # Atomic object undo precondition +**Use this when:** you need the `x-rustfs-expected-current-version-id` precondition for undo-style CopyObject restores or delete-marker removal, or you are changing how it is parsed or enforced. +**Source of truth:** `rustfs/src/app/object/shared.rs` (`expected_current_version_id` header parser), `rustfs/src/app/object/copy.rs` (CopyObject enforcement), `crates/ecstore/src/set_disk/ops/object.rs` (`expected_current_version_id` checks under the namespace write lock). + RustFS supports a destination-side version precondition for the two S3 operations used to undo changes in a versioned bucket: diff --git a/docs/operations/authing-oidc-integration.md b/docs/operations/authing-oidc-integration.md deleted file mode 100644 index fe6a036cc..000000000 --- a/docs/operations/authing-oidc-integration.md +++ /dev/null @@ -1,240 +0,0 @@ -# Authing OIDC Integration Runbook - -This runbook helps operators connect the RustFS Console to Authing through standard OpenID Connect. The examples use the default RustFS provider id, `default`. - -## 1. Integration Model - -RustFS expects a standards-compliant OpenID Connect provider, not an Authing-specific plugin. The Authing application must provide: - -- issuer metadata through `.well-known/openid-configuration` -- authorization endpoint -- token endpoint -- JWKS or another verifiable ID token signature path -- authorization-code flow that returns an `id_token` - -The RustFS browser login flow is: - -1. The user opens the RustFS OIDC authorize endpoint. -2. RustFS creates `state`, `nonce`, and a PKCE S256 challenge. -3. The browser is redirected to Authing. -4. Authing redirects back to RustFS with `code` and `state`. -5. RustFS exchanges the code with `client_id`, `client_secret`, and the PKCE verifier. -6. RustFS validates the ID token signature, issuer, audience, expiry, and nonce. -7. RustFS reads identity and authorization claims from the ID token. -8. RustFS maps claim values to RustFS policy names and issues one-hour STS credentials for the Console. - -## 2. Required Values - -Collect these values before deployment: - -| Value | Example | Notes | -| --- | --- | --- | -| Public RustFS browser origin | `https://rustfs.example.com` | The scheme and authority users open in the browser. | -| Provider id | `default` | This runbook uses the default provider. | -| RustFS callback URL | `https://rustfs.example.com/rustfs/admin/v3/oidc/callback/default` | Register this exact URL in Authing. | -| Authing application domain | `https://example.authing.cn` | Use the value shown in the Authing application. | -| Authing issuer | `https://example.authing.cn/oidc` | Copy the issuer from Authing; do not guess the path. | -| Authing App ID | `` | RustFS `client_id`. | -| Authing App Secret | `` | RustFS `client_secret`. | -| RustFS scopes | `openid,profile,email,roles` | `openid` is required; include `roles` when Authing emits role claims. | - -Authing deployments can use different issuer paths, such as `/oidc` or `/oauth/oidc`. Always copy the issuer from the Authing console and verify that discovery returns the same `issuer` value. - -## 3. Authing Configuration - -### 3.1 Create the Application - -1. Open the Authing console. -2. Create a self-hosted application named `RustFS Console`. -3. Record the App ID, App Secret, application domain, issuer, and discovery URL. - -### 3.2 Configure OIDC - -Use these protocol settings: - -| Setting | Value | -| --- | --- | -| Protocol | OpenID Connect | -| Grant type | Authorization Code | -| Response type | `code` | -| Token endpoint authentication | `client_secret_post` | -| PKCE | Allow or require `S256` | -| ID token signing algorithm | `RS256` recommended | - -RustFS sends the client secret in the request body. Do not configure Authing to reject `client_secret_post`. - -### 3.3 Register the Redirect URL - -Add this exact callback URL in Authing: - -```text -https://rustfs.example.com/rustfs/admin/v3/oidc/callback/default -``` - -The scheme, host, port, path, and provider id must match the RustFS configuration. - -### 3.4 Map Roles to RustFS Policies - -RustFS does not call Authing authorization APIs. It reads `roles` or `groups` from the ID token and maps each value to a RustFS policy name. - -Recommended policy names: - -| Authing claim value | RustFS policy | Purpose | -| --- | --- | --- | -| `consoleAdmin` | `consoleAdmin` | Full Console, admin, KMS, and S3 access. | -| `readwrite` | `readwrite` | S3 read/write access. | -| `readonly` | `readonly` | S3 read-only access. | -| `writeonly` | `writeonly` | S3 write-only access. | -| `diagnostics` | `diagnostics` | Diagnostic admin access. | - -For initial validation, assign a test user the `consoleAdmin` role and confirm that the ID token contains: - -```json -{ - "roles": ["consoleAdmin"] -} -``` - -`claim_prefix` only prepends a fixed string. It does not perform arbitrary role mapping. Keep Authing role values equal to RustFS policy names unless you already created policies with a fixed prefix. - -## 4. RustFS Configuration - -### 4.1 Environment Variables - -Set the OIDC provider and the public browser origin: - -```bash -export RUSTFS_BROWSER_REDIRECT_URL="https://rustfs.example.com" - -export RUSTFS_IDENTITY_OPENID_ENABLE=on -export RUSTFS_IDENTITY_OPENID_CONFIG_URL="" -export RUSTFS_IDENTITY_OPENID_CLIENT_ID="" -export RUSTFS_IDENTITY_OPENID_CLIENT_SECRET="" -export RUSTFS_IDENTITY_OPENID_SCOPES="openid,profile,email,roles" -export RUSTFS_IDENTITY_OPENID_REDIRECT_URI="https://rustfs.example.com/rustfs/admin/v3/oidc/callback/default" -export RUSTFS_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC=off -export RUSTFS_IDENTITY_OPENID_DISPLAY_NAME="Authing" -export RUSTFS_IDENTITY_OPENID_EMAIL_CLAIM="email" -export RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM="preferred_username" -export RUSTFS_IDENTITY_OPENID_ROLES_CLAIM="roles" -``` - -For short-lived connectivity testing only, you may temporarily add: - -```bash -export RUSTFS_IDENTITY_OPENID_ROLE_POLICY="consoleAdmin" -``` - -Do not keep `role_policy=consoleAdmin` in production unless every Authing user for this client should receive full Console access. - -Restart RustFS after changing OIDC settings. - -### 4.2 Admin Config - -If the deployment manages OIDC through compatible admin configuration commands, set the provider like this: - -```bash -mc admin config set rustfs identity_openid \ - enable=on \ - config_url="" \ - client_id="" \ - client_secret="" \ - scopes="openid,profile,email,roles" \ - redirect_uri="https://rustfs.example.com/rustfs/admin/v3/oidc/callback/default" \ - redirect_uri_dynamic=off \ - display_name="Authing" \ - email_claim="email" \ - username_claim="preferred_username" \ - roles_claim="roles" - -mc admin service restart rustfs -``` - -`RUSTFS_BROWSER_REDIRECT_URL` is a process environment variable, not an `identity_openid` provider key. Configure it in the RustFS service environment even when the provider itself is stored through admin config. - -### 4.3 Redirect URL Priority - -RustFS builds browser-facing URLs with this priority: - -1. Provider `redirect_uri`, when configured, is used for the OIDC callback URL sent to Authing. -2. `RUSTFS_BROWSER_REDIRECT_URL`, when configured, is used as the public origin for OIDC callback generation when no provider `redirect_uri` exists, and for Console success redirects and logout fallback redirects. -3. Request headers are used only when provider dynamic redirects are enabled and no browser redirect URL is configured. - -For reverse-proxy or load-balancer deployments, set `RUSTFS_BROWSER_REDIRECT_URL` to avoid depending on `Host` and `X-Forwarded-Proto` for Console redirects. OIDC authorize and callback requests must still reach the same RustFS node because in-flight OIDC `state` is local to the node. - -## 5. Validation - -### 5.1 Validate Authing Discovery - -```bash -AUTHING_ISSUER="" - -curl -fsS "$AUTHING_ISSUER/.well-known/openid-configuration" | jq '{ - issuer, - authorization_endpoint, - token_endpoint, - jwks_uri, - id_token_signing_alg_values_supported, - code_challenge_methods_supported, - token_endpoint_auth_methods_supported, - scopes_supported -}' -``` - -Check that: - -- `issuer` exactly matches `RUSTFS_IDENTITY_OPENID_CONFIG_URL` -- `authorization_endpoint`, `token_endpoint`, and `jwks_uri` are present -- `code_challenge_methods_supported` includes `S256` -- `token_endpoint_auth_methods_supported` includes `client_secret_post` -- `scopes_supported` includes `openid`, `profile`, `email`, and any role scope you need - -### 5.2 Validate RustFS Provider Visibility - -```bash -curl -fsS "https://rustfs.example.com/rustfs/admin/v3/oidc/providers" | jq -``` - -The response should include the Authing provider unless `hide_from_ui` is enabled. - -### 5.3 Test Browser Login - -Open: - -```text -https://rustfs.example.com/rustfs/admin/v3/oidc/authorize/default -``` - -Expected flow: - -1. Browser redirects to Authing. -2. The user signs in. -3. Authing redirects to `/rustfs/admin/v3/oidc/callback/default?code=...&state=...`. -4. RustFS validates the ID token and issues STS credentials. -5. The browser lands on the RustFS Console and can use the expected permissions. - -## 6. Troubleshooting - -| Symptom | Common cause | Fix | -| --- | --- | --- | -| `/oidc/providers` does not show Authing | OIDC provider did not load, or RustFS was not restarted | Check environment variables and restart RustFS. | -| Authing reports redirect mismatch | Callback URL differs between Authing and RustFS | Use the exact `/rustfs/admin/v3/oidc/callback/default` URL. | -| RustFS reports missing `code` or `state` | Proxy dropped the query string | Preserve the full callback URL and query string. | -| Token exchange fails | Wrong client secret or unsupported token auth method | Confirm `client_secret_post` is allowed. | -| RustFS reports no `id_token` | Missing `openid` scope or non-OIDC OAuth flow | Include `openid` and use OIDC authorization code flow. | -| ID token verification fails | Issuer, audience, signing algorithm, or JWKS mismatch | Compare discovery metadata with RustFS config; prefer `RS256`. | -| Login succeeds but access is denied | No matching RustFS policy claim | Ensure `roles` or `groups` is in the ID token and equals a RustFS policy name. | -| Console redirects to an internal host | Missing `RUSTFS_BROWSER_REDIRECT_URL` or incorrect proxy headers | Set `RUSTFS_BROWSER_REDIRECT_URL` to the public browser origin. | -| Invalid or expired OIDC state | Callback reached a different RustFS node | Configure load-balancer session affinity for authorize and callback requests. | - -## 7. Production Checklist - -- [ ] RustFS and Authing use HTTPS. -- [ ] Authing redirect URL is exact, not a broad wildcard. -- [ ] `RUSTFS_BROWSER_REDIRECT_URL` is set to the public RustFS browser origin. -- [ ] `RUSTFS_IDENTITY_OPENID_REDIRECT_URI` matches the registered Authing callback URL. -- [ ] Authing emits role or group claims in the ID token. -- [ ] Claim values match RustFS policy names. -- [ ] `role_policy=consoleAdmin` is not used as a permanent production shortcut. -- [ ] The load balancer preserves query strings. -- [ ] OIDC authorize and callback requests have session affinity to the same RustFS node. diff --git a/docs/operations/container-resource-detection.md b/docs/operations/container-resource-detection.md index bfd6b5d7d..4b16c2a65 100644 --- a/docs/operations/container-resource-detection.md +++ b/docs/operations/container-resource-detection.md @@ -1,203 +1,70 @@ -# Container Resource Detection +# Container resource detection -RustFS automatically detects container resource limits (CPU and memory) from cgroup v1/v2. This ensures correct resource allocation and accurate metrics in containerized environments (Kubernetes, Docker, etc.). +**Use this when:** RustFS runs under a cgroup CPU or memory limit (Kubernetes, Docker) and you need to know which limit it detected, how to override it, or which log line and metrics expose it. +**Source of truth:** `rustfs/src/cgroup_resources.rs` (detection, `ContainerResources`, `container_resources()`), `rustfs/src/memory_observability.rs` (metrics), `rustfs/src/server/runtime.rs` (Tokio thread sizing consumer), `rustfs/src/startup_entrypoint.rs` (startup log call). -## Problem +RustFS resolves the CPU core count and the memory limit once at startup (cached in a `OnceLock`) and uses them for Tokio worker/blocking-thread sizing, the memory budget, and memory metrics. Precedence is override env var, then cgroup limit, then host value from `sysinfo`. -When RustFS runs in a container, the underlying system libraries report the **host's** total CPU cores and memory, not the container's limits. This leads to: +## Detection rules -1. **Over-provisioned Tokio threads**: Too many worker and blocking threads -2. **Incorrect memory metrics**: `rustfs_memory_usage_percent` shows host-based percentage -3. **Memory budget errors**: Object data cache sized to host RAM instead of container limit -4. **OOMKills**: Container exceeds its memory limit and gets killed +| Resource | Order | Source | Rule | +| --- | --- | --- | --- | +| CPU | 1 | cgroup v2 `/sys/fs/cgroup/cpu.max` | `" "` (or bare `""` with period 100000) gives `ceil(quota / period)`; `"max"` or a zero quota means no limit. | +| CPU | 2 | cgroup v1 `/sys/fs/cgroup/cpu/cpu.cfs_quota_us` with `cpu.cfs_period_us` | `ceil(quota / period)`; a zero, unparsable, or `u64::MAX` quota means no limit. | +| CPU | 3 | host | `sysinfo` CPU count, minimum 1. | +| Memory | 1 | cgroup v2 `/sys/fs/cgroup/memory.max` | bytes; `"max"` means no limit. | +| Memory | 2 | cgroup v1 `/sys/fs/cgroup/memory/memory.limit_in_bytes` | bytes; values `>= 1 << 62` mean no limit. | +| Memory | 3 | host | `sysinfo` total memory. | -## Solution +cgroup reads are compiled only for Linux; other platforms always take the host branch. `cgroup_detected` is true when at least one of the two cgroup reads returned a limit. -RustFS now detects cgroup limits directly from the filesystem: +## Environment variables -- **CPU**: `/sys/fs/cgroup/cpu.max` (v2) or `/sys/fs/cgroup/cpu/cpu.cfs_quota_us` (v1) -- **Memory**: `/sys/fs/cgroup/memory.max` (v2) or `/sys/fs/cgroup/memory/memory.limit_in_bytes` (v1) +Names are the constants `ENV_DISABLE_CGROUP_DETECTION`, `ENV_OVERRIDE_CPU_CORES`, and `ENV_OVERRIDE_MEMORY_BYTES` in `rustfs/src/cgroup_resources.rs`. -The effective resource limits are the **minimum** of host and cgroup values. +| Variable | Accepted values | Effect | +| --- | --- | --- | +| `RUSTFS_DISABLE_CGROUP_DETECTION` | `1` or `true` (case-insensitive) | Skip cgroup reads; host values apply unless overridden. | +| `RUSTFS_OVERRIDE_CPU_CORES` | integer `> 0` | Replaces the CPU core count regardless of cgroup or host. | +| `RUSTFS_OVERRIDE_MEMORY_BYTES` | integer `> 0`, bytes | Replaces the memory limit regardless of cgroup or host. | -## Detection Logic +Non-positive or unparsable override values are ignored. Changes take effect on process restart. -### CPU Detection +## Startup log -1. Read cgroup v2 `/sys/fs/cgroup/cpu.max` - - Format: `"$QUOTA $PERIOD"` or `"max"` (unlimited) - - Calculate: `cores = ceil(quota / period)` -2. Fallback to cgroup v1 `/sys/fs/cgroup/cpu/cpu.cfs_quota_us` - - Calculate: `cores = ceil(quota / period)` -3. Fallback to host CPU count from `sysinfo` +`log_container_resources` emits exactly one of these lines with `cpu_cores` and `memory_bytes` fields (the INFO variants also carry `memory_mib`): -### Memory Detection - -1. Read cgroup v2 `/sys/fs/cgroup/memory.max` - - Value in bytes or `"max"` (unlimited) -2. Fallback to cgroup v1 `/sys/fs/cgroup/memory/memory.limit_in_bytes` - - Very large values (≥2^62) indicate unlimited -3. Fallback to host memory from `sysinfo` - -## Environment Variables - -### Disable Cgroup Detection - -```bash -RUSTFS_DISABLE_CGROUP_DETECTION=1 -``` - -Disables cgroup detection entirely. Useful for testing or when cgroup filesystem is not accessible. - -### Override CPU Cores - -```bash -RUSTFS_OVERRIDE_CPU_CORES=4 -``` - -Overrides detected CPU cores. Takes precedence over cgroup detection. - -### Override Memory Limit - -```bash -RUSTFS_OVERRIDE_MEMORY_BYTES=2147483648 -``` - -Overrides detected memory limit in bytes. Takes precedence over cgroup detection. +| Message | Level | Condition | +| --- | --- | --- | +| `container resources (overridden by environment variables)` | INFO | an override env var was applied; also carries `cgroup_detected` | +| `container resources (detected from cgroup)` | INFO | no override, at least one cgroup limit read | +| `container resources (using host values)` | DEBUG | neither override nor cgroup limit | ## Metrics -### New Metrics +Gauges emitted from `rustfs/src/memory_observability.rs`. -| Metric | Description | -|--------|-------------| -| `rustfs_memory_effective_total_bytes` | Effective memory total (host or cgroup) | -| `rustfs_cgroup_detected` | Whether cgroup limits were detected (1=yes, 0=no) | -| `rustfs_cgroup_cpu_cores_limit` | Detected CPU cores limit | -| `rustfs_cgroup_memory_limit_bytes` | Detected memory limit | - -### Updated Metrics - -| Metric | Change | -|--------|--------| -| `rustfs_memory_total_bytes` | Now uses effective memory (cgroup-aware) | -| `rustfs_memory_usage_percent` | Now calculated against effective memory | - -## Startup Logging - -RustFS logs detected container resources at startup: - -``` -INFO container resources (detected from cgroup) cpu_cores=2 memory_bytes=1073741824 memory_mib=1024 -``` - -or - -``` -INFO container resources (overridden by environment variables) cpu_cores=4 memory_bytes=2147483648 memory_mib=2048 -``` - -## Examples - -### Kubernetes with Resource Limits - -```yaml -resources: - limits: - cpu: "2" - memory: "1Gi" - requests: - cpu: "500m" - memory: "512Mi" -``` - -RustFS will detect: -- CPU cores: 2 -- Memory: 1 GiB (1073741824 bytes) - -### Docker with CPU and Memory Limits - -```bash -docker run --cpus=2 --memory=1g rustfs/rustfs:latest -``` - -RustFS will detect: -- CPU cores: 2 -- Memory: 1 GiB - -### Manual Override - -```bash -export RUSTFS_OVERRIDE_CPU_CORES=4 -export RUSTFS_OVERRIDE_MEMORY_BYTES=2147483648 -``` - -RustFS will use: -- CPU cores: 4 -- Memory: 2 GiB +| Metric | Meaning | +| --- | --- | +| `rustfs_memory_effective_total_bytes{basis}` | Effective memory total. `basis` is `cgroup` when a cgroup limit was detected, else `host`; an override does not change the basis label. | +| `rustfs_container_cpu_cores` | Effective CPU cores. | +| `rustfs_container_memory_bytes` | Effective memory limit in bytes. | +| `rustfs_container_cgroup_detected` | `1` when a cgroup limit was read, else `0`. | +| `rustfs_container_overridden` | `1` when an override env var was applied, else `0`. | +| `rustfs_memory_total_bytes`, `rustfs_memory_usage_percent` | Computed against the effective total (`record_memory_usage` in `crates/io-metrics/src/lib.rs`). | ## Troubleshooting -### Cgroup Detection Not Working +When effective values look like the host rather than the container: -1. Check if cgroup filesystem is mounted: - ```bash - ls -la /sys/fs/cgroup/ - ``` +1. Confirm detection is not disabled: `env | grep RUSTFS_DISABLE_CGROUP_DETECTION`. +2. Find the startup line: `grep "container resources" `. The `(using host values)` variant is DEBUG, so raise the log level if no variant appears. +3. Inspect the cgroup filesystem inside the container: -2. Check cgroup version: - ```bash - stat -fc %T /sys/fs/cgroup/ - ``` - - `cgroup2fs` = cgroup v2 - - `tmpfs` = cgroup v1 +```bash +stat -fc %T /sys/fs/cgroup/ # cgroup2fs = v2, tmpfs = v1 +cat /sys/fs/cgroup/cpu.max /sys/fs/cgroup/memory.max # v2 +cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us /sys/fs/cgroup/memory/memory.limit_in_bytes # v1 +``` -3. Check if limits are set: - ```bash - # cgroup v2 - cat /sys/fs/cgroup/cpu.max - cat /sys/fs/cgroup/memory.max - - # cgroup v1 - cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us - cat /sys/fs/cgroup/memory/memory.limit_in_bytes - ``` - -### Metrics Show Host Values - -If `rustfs_memory_effective_total_bytes` shows host memory instead of cgroup limit: - -1. Verify cgroup detection is not disabled: - ```bash - echo $RUSTFS_DISABLE_CGROUP_DETECTION - ``` - -2. Check startup logs for cgroup detection: - ```bash - grep "container resources" /logs/rustfs.log - ``` - -3. Use environment variable override as workaround: - ```bash - export RUSTFS_OVERRIDE_MEMORY_BYTES=1073741824 - ``` - -## Implementation Details - -### Files Modified - -- `rustfs/src/cgroup_resources.rs` - Core cgroup detection logic -- `rustfs/src/container_config.rs` - Container configuration with overrides -- `rustfs/src/memory_observability.rs` - Updated memory metrics -- `rustfs/src/server/runtime.rs` - Updated Tokio runtime configuration -- `rustfs/src/startup_entrypoint.rs` - Startup logging - -### Performance Impact - -- **Startup**: One-time detection adds ~1ms overhead -- **Runtime**: Cached values, no repeated filesystem reads -- **Memory**: Negligible (<1KB for cached values) - -### Thread Safety - -All detection functions are thread-safe and use `OnceLock` for caching. +4. If the runtime does not expose limits to the container, pin them with `RUSTFS_OVERRIDE_CPU_CORES` / `RUSTFS_OVERRIDE_MEMORY_BYTES` and confirm `rustfs_container_overridden` reads `1`. diff --git a/docs/operations/dial9-runtime-profiling.md b/docs/operations/dial9-runtime-profiling.md index eff939932..4a69e0324 100644 --- a/docs/operations/dial9-runtime-profiling.md +++ b/docs/operations/dial9-runtime-profiling.md @@ -1,5 +1,8 @@ # dial9 Tokio Runtime Profiling +**Use this when:** you need Tokio runtime-level evidence (which task held a worker, long polls, park/unpark behaviour) that Prometheus metrics and `tracing` spans cannot provide, or you are building or running the opt-in `dial9` profiling binary. +**Source of truth:** `crates/obs/src/telemetry/dial9/mod.rs` (session setup), `crates/obs/src/metrics/collectors/dial9.rs` (metrics), `crates/config/src/constants/runtime.rs` (`RUSTFS_RUNTIME_DIAL9_*` and defaults), `.config/make/build.mak` (`build-profiling`), `crates/obs/build.rs` (feature/cfg pairing check). + `dial9-tokio-telemetry` records Tokio runtime-level events — poll start/end, worker park/unpark, task spawn/terminate, and optionally async backtraces of stalled tasks — into binary trace segments. @@ -60,9 +63,8 @@ That is `cargo build --release --bin rustfs --features dial9` with dumps and S3 upload are both unavailable, for the reasons given above and below. `crates/obs/build.rs` fails the build if the `dial9` feature is enabled without -`--cfg tokio_unstable`. This is deliberate: an environment `RUSTFLAGS` *replaces* -the value from `.cargo/config.toml` rather than appending to it, so the flag used -to disappear silently whenever anything else set `RUSTFLAGS`. +`--cfg tokio_unstable`, so a mismatched `RUSTFLAGS` cannot produce a binary that +silently records nothing. For CPU profiling with usable stacks, add `-C force-frame-pointers=yes`. diff --git a/docs/operations/drive-timeout-tuning.md b/docs/operations/drive-timeout-tuning.md index 2976db091..9122e93b2 100644 --- a/docs/operations/drive-timeout-tuning.md +++ b/docs/operations/drive-timeout-tuning.md @@ -1,5 +1,8 @@ # Drive Timeout Tuning +**Use this when:** `ListObjects`/`ListObjectsV2` on a large prefix fails with `Io error: timeout`, or RustFS runs on HDD-class, network, or throttled storage and you need to widen per-operation drive liveness budgets. +**Source of truth:** `crates/config/src/constants/drive.rs` (`DEFAULT_DRIVE_*_TIMEOUT_SECS`, `DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS`), `crates/config/src/constants/object.rs` (`DEFAULT_OBJECT_DISK_READ_TIMEOUT`), `crates/config/src/constants/capacity.rs` (`DEFAULT_CAPACITY_MAX_TIMEOUT_SECS`), `crates/ecstore/src/cache_value/metacache_set.rs` (walk stall handling and `rustfs_list_path_raw_stall_total`). + This document describes the per-operation drive timeout knobs and the drive-timeout profile. It is written for operators running RustFS on slow or high-latency storage (HDD-class disks, network block devices, throttled @@ -79,10 +82,11 @@ for the full list and defaults. `ListObjects`/`ListObjectsV2` on a large prefix either: - returns `500 InternalError` with `Io error: timeout`; or -- (on older builds) returns HTTP 200 with `IsTruncated=false` after fewer keys - than the bucket actually holds — a **silent** truncation that S3 clients - (`mc`, minio-go, SDK pagination loops) cannot detect, because - `IsTruncated=false` is the protocol's only end-of-listing signal. +- on builds without the failure contract below, returns HTTP 200 with + `IsTruncated=false` after fewer keys than the bucket actually holds — a + **silent** truncation that S3 clients (`mc`, minio-go, SDK pagination loops) + cannot detect, because `IsTruncated=false` is the protocol's only + end-of-listing signal. Every "missing" object remains readable by exact key via `GetObject` / `StatObject`; only the listing is affected. @@ -97,21 +101,10 @@ single `readdir` exceed the budget on a perfectly healthy disk, especially on HDD-class or throttled storage. That trips a drive timeout, which the listing path escalates and surfaces to the client. -### The silent variant is fixed; the loud 500 is tuned away +### Failure contract -As of the walk-stall rework (merged to `main`, first released in **1.0.0-beta.9**): - -- **The silent variant is eliminated.** A walk that dies mid-stream can no - longer be consumed as a clean end-of-listing. Once a walk has streamed any - entries and then stalls, the failure is recorded as a hard drive timeout and - escalated on that erasure set, so the client always sees an error — never a - well-formed short page. This is locked by the - `list_path_raw_returns_timeout_when_producer_fails_after_partial_entry` - regression test in `crates/ecstore/src/cache_value/metacache_set.rs`. -- **The remaining 500 is an operator-tunable, not a data-integrity bug.** A - genuinely wide flat directory can still exhaust the default 5s stall budget on - slow storage and fail the listing loudly. The supported mitigation is to widen - the budget. +- A walk that stalls after streaming any entries fails as a hard drive timeout escalated on that erasure set; the client always sees an error, never a well-formed short page. Locked by `list_path_raw_returns_timeout_when_producer_fails_after_partial_entry` in `crates/ecstore/src/cache_value/metacache_set.rs`. +- The remaining `500` on a genuinely wide flat directory is an operator tunable, not a data-integrity bug: widen the stall budget as below. ### Mitigation @@ -129,12 +122,6 @@ Raise the walk stall budget, or select the high-latency profile: -e RUSTFS_DRIVE_TIMEOUT_PROFILE=high_latency ``` -> Note: on releases at or before `1.0.0-beta.8`, the foreground listing path was -> bounded by the *total* wall-clock knob `RUSTFS_DRIVE_WALKDIR_TIMEOUT_SECS` -> instead of the stall budget. If you cannot upgrade, raise that knob — but -> upgrading to `1.0.0-beta.9` or later is strongly preferred, because only the -> newer builds convert the *silent* truncation into a detectable error. - The most durable fix for pathologically wide directories is to shard keys under additional prefix levels so no single directory holds an enormous flat child set; the stall budget then never has to bound one giant `readdir`. diff --git a/docs/operations/durability-modes.md b/docs/operations/durability-modes.md index 6cbf8e830..a543237cc 100644 --- a/docs/operations/durability-modes.md +++ b/docs/operations/durability-modes.md @@ -1,5 +1,8 @@ # Durability modes (drive sync tiers) +**Use this when:** choosing or debugging the fsync tier (`strict|relaxed|none|legacy-off`) for a deployment or a single bucket, or changing any write-path sync behaviour. +**Source of truth:** `crates/ecstore/src/disk/local.rs` (`ENV_RUSTFS_DURABILITY_MODE`, `ENV_RUSTFS_DRIVE_SYNC_ENABLE`, mode resolution and per-write-point sync decisions), `crates/ecstore/src/bucket/durability.rs` (`ENV_NEW_BUCKET_DURABILITY_MODE`, per-bucket override), `rustfs/src/admin/handlers/durability.rs` (admin API), `crates/ecstore/src/bucket/metadata_sys.rs` (`BUCKET_METADATA_REFRESH_INTERVAL`). + RustFS lets operators choose how much fsync work runs on the object write path. The default (`strict`) preserves the fully synced behavior RustFS has always shipped; the relaxed tiers are **opt-in** trades of power-loss @@ -81,13 +84,13 @@ failure: MinIO's default posture (no per-object fsync) but means small objects have the widest loss window. - Durability of acknowledged writes therefore rests on **erasure-coded - redundancy across other nodes** plus the unclean-shutdown heal introduced - in PR #4221 converging the affected drive afterwards. + redundancy across other nodes** plus the unclean-shutdown heal converging + the affected drive afterwards. Deployment rule for `relaxed`: only multi-node clusters whose nodes sit in **independent power domains** (separate feeds/UPS). If all nodes can lose -power simultaneously — the exact incident class that motivated PR #4221 — -`relaxed` can lose recently acknowledged objects cluster-wide. Single-node +power simultaneously, `relaxed` can lose recently acknowledged objects +cluster-wide. Single-node deployments must stay on `strict`. **`none`.** No fsync on the object data path at all; acknowledged objects can @@ -119,7 +122,7 @@ staged in tmp still commits with full `strict` durability. The durability mode is server-side configuration only; it cannot be raised or lowered by any request header. -## Per-bucket durability (phase 2) +## Per-bucket durability A bucket can override the process-wide mode with its own tier. The override is stored in the bucket's metadata (a `durability.json` entry in @@ -146,8 +149,8 @@ configuration plane. ### New-bucket default -A bucket created after this feature ships gets a `relaxed` override **seeded -into its own metadata** at creation time (rustfs/backlog#1811), so it opts +A newly created bucket gets a `relaxed` override **seeded into its own +metadata** at creation time, so it opts into MinIO's default posture (object data still fdatasynced; xl.meta and directory-entry fsyncs left to the page cache) without touching the process-wide default. This is a gradual migration: @@ -226,20 +229,10 @@ bucket on power failure. ## Performance expectations -The often-quoted 26x PUT throughput delta was measured on macOS with the old -binary switch fully **off** (equivalent to `none`/`legacy-off`), where -`F_FULLFSYNC` heavily amplifies sync cost. `relaxed` keeps the per-shard -fdatasync, so its gain is necessarily smaller and must be measured on the -target platform (Linux ext4/xfs) before being relied on. Do not use `none` -numbers to size `relaxed`. - -## Scope - -Phase 1 (rustfs/backlog#926) shipped the global, per-process tier configured -by environment variable. Phase 2 (rustfs/backlog#938) adds the per-bucket -override described above, configured through the admin API and stored in -bucket metadata. `mc admin` integration for the per-bucket tier is a -follow-up. +Measure `relaxed` on the target platform (Linux ext4/xfs) before relying on a +number. Throughput deltas measured with sync fully **off** (`none` or +`legacy-off`) do not transfer: `relaxed` keeps the per-shard fdatasync, so its +gain is necessarily smaller. Do not use `none` numbers to size `relaxed`. ## Related Recovery Guides diff --git a/docs/operations/get-path-experimental-switches.md b/docs/operations/get-path-experimental-switches.md deleted file mode 100644 index 015db346c..000000000 --- a/docs/operations/get-path-experimental-switches.md +++ /dev/null @@ -1,35 +0,0 @@ -# GET Path Experimental Performance Switches - -This document records two experimental environment switches on the object GET -path. Both default to **off**, are read once at startup, and exist to support -staged performance work — they are not general tuning knobs. Until this -document existed they were referenced only by performance harness scripts, -which made them look like orphans during dead-code sweeps; they are kept -deliberately (rustfs/backlog#1832). - -## RUSTFS_GET_SEEK_BUFFER_ENABLE - -- Type: boolean (`true`/`false`), default `false`. -- Read once at startup in `rustfs/src/app/object_usecase.rs`. -- When enabled, small GET responses may be served through an in-memory seek - buffer, providing seek support without re-reading the object. The seek-buffer - code path is unit-test gated; whether the path stays or graduates to default - is a post-1.0 maintainer decision — do not remove either the switch or the - gated path as dead code. - -## RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE - -- Type: boolean (`true`/`false`), default `false`. -- Read once at startup in `rustfs/src/app/object_usecase.rs`. -- When enabled, GET responses attribute output-handoff stage timing in the GET - stage metrics, at a small per-request bookkeeping cost. Used by the A/B - performance runbooks (`scripts/run_get_codec_streaming_smoke.sh`, - `scripts/test_get_1mib_abba_stage_metrics.sh`) to compare handoff cost - between configurations. - -## Operational guidance - -Leave both switches unset in production. Enable them only when following a -performance runbook that asks for them, and unset them afterwards — both are -startup-latched, so changing a value requires a process restart to take -effect. diff --git a/docs/operations/heal-concurrency-safety-notes-zh.md b/docs/operations/heal-concurrency-safety-notes-zh.md deleted file mode 100644 index ffb1e90c7..000000000 --- a/docs/operations/heal-concurrency-safety-notes-zh.md +++ /dev/null @@ -1,115 +0,0 @@ -# Heal 并发安全说明（对象级 healing 标记对标审计结论） - -对应 backlog rustfs/backlog#1874（父 #1862，HS-12）。本文回答一个问题：MinIO 在 heal -期间对对象打 `x-minio-healing:true` 元数据标记以防"heal 提交与并发删除/版本清理互毁" -（cmd/xl-storage.go RenameData 的 healing 分支），RustFS 是否需要同款防御。 - -**结论：不需要。** RustFS 不存在 MinIO 用 healing 标记防御的那类竞争：所有会触达同一 -`(bucket, object)` 提交面的路径都在同一把对象级 namespace 写锁上互斥，且 heal 的锁 -guard 覆盖 rename 提交全程；MinIO 需要标记的根因（RenameData 提交内部与版本清理逻辑 -交错）在 RustFS 的提交模型中不存在。RustFS 已有一个瞬态 healing 旗标用于另一目的 -（见下文 §2），并有并发不变量回归测试锁定本结论（§5）。 - -## 1. 两个防御模型的对照 - -MinIO：heal 时对对象写 `x-minio-healing:true`（持久元数据标记），后续任何 RenameData -提交看到该标记就跳过版本清理/legacy purge 逻辑——防御发生在锁外，靠元数据让路。 - -RustFS：三层防御，全部不依赖持久对象标记： - -1. **锁内互斥**：heal 与一切前台/后台写路径的提交点在同一把 `(bucket, object)` ns 写锁 - 上串行（分布式部署为 quorum 锁 RPC，单机为进程内锁管理器；锁粒度是对象级，version - 恒为 None）。 -2. **提交模型隔离**：rename_data 提交内没有会与 heal 交错的版本清理逻辑；被替换旧版本 - 的 data_dir 物理删除被移出提交临界区（commit tail），且只删已被新提交替换的 unshared - 目录。 -3. **瞬态 healing 旗标**：`FileInfo::set_healing`（crates/filemeta/src/fileinfo.rs）在 - heal 提交的内存 FileInfo 上打 `"healing"` 内部键，rename_data 据此允许先清空 stale - 目标 data_dir 再 rename——解决 heal 复用 data_dir 做 in-place 修复时 rename(2) 无法 - 替换非空目录的文件系统语义冲突（EEXIST/ENOTEMPTY）。该键是瞬态的，不落盘 - （`is_skip_meta_key`），与 MinIO 的持久标记目的不同。非 heal 提交撞上非空目标 - data_dir 会显式失败，有测试锁定两个方向的行为。 - -## 2. 交点矩阵 - -中心路径：`heal_object_with_explicit_version_regen`（crates/ecstore/src/set_disk/ops/heal.rs， -下称 heal.rs）在入口取 `(bucket, object)` ns 写锁，guard 绑定到函数作用域末尾，覆盖 -quorum 元数据读取 → EC 重建 → 逐盘 rename 提交 → tmp 清理 → HEAL_RENAME_INCOMPLETE -部分提交返回 → 孤儿 data_dir 回收的全过程。并发侧逐交点判定： - -| # | 并发路径 | 并发侧锁 | 判定 | 关键证据 | -|---|---|---|---|---| -| 1 | PUT 对象提交 | `put_object_commit` 对象写锁，rename_data 在锁内 | 同锁串行 | ops/object.rs 提交锁段 + rename 调用点 | -| 2 | PUT 旧 data_dir tail 清理 | drop 对象锁后的 `commit_rename_data_dir`，无锁 | 无锁并发，语义安全（见 §3.1） | object.rs drop 后 tail 段；io_primitives.rs | -| 3 | DELETE 单对象/版本 | `delete_object` 对象写锁，delete_version 在锁内 | 同锁串行 | object.rs delete_object 锁段 | -| 4 | DELETE 批量 | 批量逐对象写锁（dist 走批量锁 RPC） | 同锁串行 | object.rs delete_objects 锁段 | -| 5 | CompleteMultipart | 对象写锁 + upload 路径锁双锁，rename 在锁内 | 同锁串行 | ops/multipart.rs 提交锁段 | -| 6 | CompleteMultipart tail 清理 | drop 对象锁后的旧 data_dir 删除 | 无锁并发，语义安全（见 §3.1） | multipart.rs drop 后 tail 段 | -| 7 | AbortMultipart | 仅 multipart bucket 的 upload 路径锁 | 锁 key 不相交，但资源不相交（abort 不触对象 data_dir/xl.meta）→ 无实际交点 | multipart.rs abort 锁段 | -| 8 | ILM expiry（含 DeleteAllVersions） | DeleteAllVersions 走 `delete_prefix_object=true` → 仍取对象锁；FreeVersionTask 显式取锁；noncurrent 批量走批量锁 | 同锁串行 | bucket_lifecycle_ops.rs 消费端链路 | -| 9 | 纯 prefix 删除（绕锁能力面） | `delete_prefix`-only 不取子对象锁 | 无锁并发，但生产调用方为零（见 §3.2） | object.rs delete_object 锁条件 | -| 10 | 孤儿 data_dir 回收 reclaim_orphan_data_dirs | 函数本体无锁；唯一生产调用方在 heal 锁内 | heal 流程内=锁内串行 | heal.rs 收尾调用；io_primitives.rs | -| 11 | 旧清理 receipt 对账 reconcile_old_data_cleanup_receipts | 函数本体无锁；调用点在 heal 锁内 + epoch fence 防误删 | 锁内串行 | object.rs 对账函数 | -| 12 | replication | 数据面为远端 HTTP 写（不落本地盘）；本地元数据回写走对象锁 | 同锁串行 / 无交点 | replication_resyncer.rs 链路 | -| 13 | data_movement / rebalance / decommission 源清理 | 显式取对象锁 + 版本未变复核 + guard 复用（no_lock 只是复用已持锁） | 同锁串行 | data_movement/mod.rs 源清理 | -| 14 | copy_object | 目标对象锁 / 走 put 链锁 | 同锁串行 | object.rs copy_object 锁段 | -| 15 | 另一 heal 任务（跨 HealType/force_start） | dedup key 跨类型不相交 + force_start 跳过去重 → 任务级可并发 | 最终在 ns 写锁上串行 | heal/manager.rs dedup key 构成 | -| 16 | admin `no_lock=true` heal | 客户端可控绕锁 | 无锁并发，明示运维选项（见 §3.3） | admin/handlers/heal.rs 透传 | -| 17 | stale multipart 清理 | multipart bucket 的 upload 路径锁 | 资源不相交 → 无交点 | bucket_lifecycle_ops.rs 清理链路 | - -## 3. 残留窗口定性 - -### 3.1 PUT/CompleteMultipart commit tail（交点 2/6） - -写路径提交成功、释放对象锁之后，才 best-effort 删除被替换的旧 data_dir（注释明示有意 -不阻塞下一操作）。该删除与并发 heal 对同一旧 data_dir 的读取/重建存在竞态窗口，但语义 -安全： - -- 删除目标是已被新提交替换的 unshared data_dir；heal 的 canonical 元数据来自 quorum - 仲裁（ETag/mod_time），此时 quorum 已指向新版本，heal 不会把已替换版本当作 canonical - 复活； -- 竞态最坏后果 = heal 当轮对旧版本的一次 transient 失败/空转，重试轮自然收敛；清理 - residue 会上报并重新入队 heal（`report_old_data_dir_cleanup`）； -- 换盘重建等长 heal 走 per-version 显式版本请求，quorum 元数据在锁内读取，不受 tail - 影响。 - -### 3.2 纯 prefix 删除（交点 9） - -`delete_prefix && !delete_prefix_object` 的路径不取子对象锁（对象名空间锁无法保护前缀 -递归删除），与并发 heal 存在理论复活窗口（heal 在 prefix 删除进行中依据旧 quorum 元 -数据重建某版本）。全仓库核对结论：该路径的**生产调用方为零**——所有生产 `delete_prefix: -true` 调用点均同时设置 `delete_prefix_object: true`（从而取对象锁）或在测试模块内。这 -是 API 能力面的暴露而非行为风险。若未来有调用方需要纯 prefix 删除，须在调用点证明与 -heal/scanner 的隔离（例如 bucket 级停扫围栏）。 - -### 3.3 admin `no_lock=true`（交点 16） - -admin heal 请求可透传客户端 `nolock` 参数绕过 ns 锁（与 MinIO madmin 的同名选项对齐）。 -这是运维明示选项：使用即自负与并发写的竞争责任。文档化即可，不建议收紧。 - -## 4. heal 侧自身的不变量保障 - -- dedup key 跨 HealType 不相交（object/metadata/mrf/ecdecode/prefix 各自键面）+ admin - `force_start` 可跳过去重 → 同对象可能同时存在多个 heal 任务，但它们的执行体全部在 - `heal_object` 入口的 ns 写锁上串行（生产入口均 `no_lock=false`）； -- read-repair 的本地 TTL 预留只去重自身来源，不拦截其他来源的 heal——同样由 ns 锁兜底； -- healing 旗标不落盘，故不存在"标记残留导致后续提交错误让路"的反向风险。 - -## 5. 回归测试 - -以下两个并发不变量测试随本审计加入 `crates/ecstore/src/set_disk/ops/heal.rs` 测试模块： - -- `heal_racing_version_delete_never_resurrects_the_deleted_version`：注入 doomed 版本 - shard 损坏后，版本化 DELETE 与 Deep heal 真并发（同一把锁争用），断言已删除版本不被 - 复活、存活版本完好； -- `heal_racing_unversioned_overwrites_preserves_the_last_commit`：非版本化覆盖提交（激活 - commit tail 旧 data_dir 删除）与 Deep heal 循环竞态，断言最终 current 恰为最后一次 - 提交（etag 级一致）。 - -## 6. 结论 - -MinIO 的 `x-minio-healing` 是锁外元数据防御，前提是其 RenameData 提交内部存在与 heal -交错的版本清理逻辑；RustFS 的提交模型把这类交错从根上消除（提交面锁内互斥 + 清理外 -移到 tail + tail 只删 unshared 旧目录），因此引入持久对象级 healing 标记没有对应的竞争 -可防，反而会引入 FileInfo 落盘格式变更与标记残留清理两类新成本。维持现状，本对标疑点 -关闭。 diff --git a/docs/operations/heal-scanner-parity-notes-zh.md b/docs/operations/heal-scanner-parity-notes-zh.md deleted file mode 100644 index 027926747..000000000 --- a/docs/operations/heal-scanner-parity-notes-zh.md +++ /dev/null @@ -1,109 +0,0 @@ -# Heal/Scanner 配置与语义对照（MinIO parity 决策记录） - -对应 backlog rustfs/backlog#1878（父 #1862，批 HS-14/HS-16/HS-18）。本页沉淀三项"决策 + 文档化"结论：scanner idle 节流语义对照与迁移警告（HS-14）、单机默认扫描周期决策（HS-16）、stale multipart 与 tmp/.trash 清理三段核对（HS-18），并顺带收录 bitrot_cycle 与 alert_excess_folders 两项已确认的默认值差异。所有 MinIO 侧结论均于 2026-08 按 minio/minio master 逐源码核对（引用文件为上游路径），不转述二手资料。 - -运行时旋钮的完整清单、状态端点与调参流程见 [Scanner Runtime Controls](scanner-runtime-controls.md)；excess 告警阈值差异见 [Scanner Excess Alerts](scanner-excess-alerts_zh.md)；heal 并发模型对照见 [Heal 并发安全说明](heal-concurrency-safety-notes-zh.md)。 - -## 1. HS-14：scanner idle 节流语义对照 - -### RustFS 当前语义（三因子） - -RustFS 的 scanner 步进节流由三个因子共同决定（crates/scanner/src/sleeper.rs）： - -1. **总闸 `scanner.idle_mode` / `RUSTFS_SCANNER_IDLE_MODE`（默认 `true`）**：`false` 时所有节流 sleep 全部跳过，scanner 全速推进；`true` 时按下面两因子计算 sleep。 -2. **速度档**（`scanner.speed` / `RUSTFS_SCANNER_SPEED`，默认 `default`）：档位表与 MinIO 完全一致（见下表）。目录级 sleep = `1ms × factor`（上限 `max_wait`）；对象级 sleep = `本对象处理耗时 × factor`，下限 1ms、上限 `max_wait`。 -3. **前台读退避下限**：`current_foreground_read_activity()` 取并发 GetObject 请求数（rustfs/src/storage/concurrency/request_guard.rs 的 `GetObjectGuard`）与流式读计数（`ForegroundReadGuard`）的较大值，换算为 `10ms × 活跃读数`、封顶 250ms 的下限；该下限对目录级与对象级 sleep 都生效（`.max(foreground_sleep)`），且**可以超过速度档的 `max_wait`**（自身封顶 250ms）。速度档为 `fastest`（factor=0）时预设 sleep 为 0，但只要 `idle_mode=true`，前台读下限仍然生效。 - -| 速度档 | sleep factor | 单次 sleep 上限 | 周期间隔 | -|---|---:|---:|---:| -| `fastest` | 0 | 0 | 1s | -| `fast` | 1× | 100ms | 1m | -| `default` | 2× | 1s | 1m | -| `slow` | 10× | 15s | 1m | -| `slowest` | 100× | 15s | 30m | - -实际行为矩阵（RustFS）： - -| `idle_mode` | 速度档 | 前台并发读 = 0 | 前台并发读 > 0 | -|---|---|---|---| -| `false` | 任意 | 完全不休眠，全速 | 完全不休眠，全速（前台退避也被总闸关闭） | -| `true` | `fastest` | 预设 sleep = 0，等效全速 | 每步 sleep = 前台读下限（10ms×读数，封顶 250ms） | -| `true` | 其余档 | 每步 sleep = 预设值（1ms~15s 封顶） | 每步 sleep = max(预设值, 前台读下限) | - -周期间隔的解析优先级为 env `RUSTFS_SCANNER_CYCLE` > 持久化 `scanner.cycle` > `scanner.start_delay` > 启动期默认覆盖（当前恒无）> 速度档派生（crates/scanner/src/runtime_config.rs）。另有 `scanner.yield_every_n_objects`（默认 128）的协作式让出，与节流 sleep 相互独立。 - -### MinIO 当前语义（master 逐源码核对） - -MinIO 的对应开关是 `scanner:idle_speed` / `MINIO_SCANNER_IDLE_SPEED`（internal/config/scanner/scanner.go）：取值为空串或 `on`（默认）时 `IdleMode=0`，取值 `off` 时 `IdleMode=1`。启动/配置加载时一次性写入 `scannerIdleMode`（cmd/config-current.go），扫描侧闭包 `weSleep = scannerIdleMode.Load() == 0`（cmd/xl-storage-disk-id-check.go）：**`on`（默认）= 目录级与对象级节流 sleep 始终插入（按速度档 factor，minSleep 100µs）；`off` = 两条节流路径完全不 sleep，全速扫描**。当前上游没有任何按 S3 请求/磁盘活动动态调整节流的逻辑——这是静态开关。 - -命名具有误导性，是历史残留：2024-01 之前 `weSleep` 由磁盘活动驱动（"Entire queue is full, so we sleep"，即有并发 S3/heal 活动才 sleep），minio/minio#18734（commit 7705605b）把该活动门替换为上述静态配置（初版取值 `throttled`/`full`，后改为 `on`/`off`），上游残留注释 "default is throttled when idle"、"Sleep always or based on incoming S3 requests" 均是替换前的语义描述，与现行代码不符。 - -### 对照与迁移警告 - -| 维度 | RustFS | MinIO（master） | -|---|---|---| -| 开关名 | `scanner.idle_mode` / `RUSTFS_SCANNER_IDLE_MODE` | `scanner:idle_speed` / `MINIO_SCANNER_IDLE_SPEED` | -| 取值 | 布尔 `true`/`false` | `on`/`off` | -| 默认 | `true`（节流开启） | `on`（节流开启） | -| 开 = | 节流总闸开：速度档 sleep + 前台读下限 | 节流总闸开：速度档 sleep | -| 关 = | 完全不休眠（含前台读下限一并失效） | 完全不休眠 | -| 活动耦合 | 有：前台并发读抬高 sleep 下限（10ms×读数，封顶 250ms） | 无（2024-01 起为静态开关） | -| 速度档表 | 两边完全一致（上表） | 同左 | - -迁移警告： - -- **环境变量名不可照搬**：RustFS 只读取 `RUSTFS_*` 前缀，不解析 `MINIO_SCANNER_*` 任何别名（crates/scanner、crates/utils 的 env 读取无别名链，测试还专门断言 `MINIO_SCANNER_SPEED`/`MINIO_SCANNER_CYCLE` 不泄漏生效）。照搬 `MINIO_SCANNER_IDLE_SPEED=off` 到 RustFS 会静默无效，必须改写成 `RUSTFS_SCANNER_IDLE_MODE=false`。 -- **取值词表不同**：`on/off` vs `true/false`，不能原样复制。 -- **方向澄清（修正父 issue 的预设）**：按当前上游源码，MinIO `idle_speed` 与 RustFS `idle_mode` 在"开=节流、关=全速"方向上是一致的，并非反向；父 issue 中"MinIO on=集群空闲才节流、off=始终按 delay 节流"的矩阵描述的是 2024-01 之前的活动耦合行为与反向解读，与 master 不符。真正需要写进迁移手册的差异是：MinIO 的 `idle_speed` 名称暗示"空闲时才慢"但实际是静态总闸；RustFS 的 `idle_mode=true` 在总闸之上还叠加了 MinIO 没有的前台读保护下限。 -- **`false` 是大锤**：RustFS `idle_mode=false` 会连前台读退避一起关闭，scanner 与前台读完全抢盘；仅在 benchmark 或可独占 IO 的窗口使用。 - -**决策（HS-14）：保持现状。** RustFS 语义更直观（`idle_mode` = 节流总闸，`true` 即自适应限速），且比 MinIO 多一层前台读保护；不新增 `RUSTFS_SCANNER_IDLE_SPEED` 兼容别名（无社区强诉求不做，避免双入口漂移）。本节即对照表与迁移警告的正式落点。 - -## 2. bitrot_cycle 默认差异 - -| 项 | RustFS | MinIO | -|---|---|---| -| 键 | `heal.bitrot_cycle` / `RUSTFS_SCANNER_BITROT_CYCLE_SECS`（scanner.bitrot_cycle 为兼容旧键） | `heal:bitrotscan` / `MINIO_HEAL_BITROTSCAN` | -| 默认 | 30 天（crates/config/src/constants/heal.rs 的 `DEFAULT_HEAL_BITROT_CYCLE_SECS`）：按墙钟周期把扫描切深扫（deep bitrot） | `off`（internal/config/heal/heal.go 默认 `EnableOff`）：不做周期性深扫，仅普通扫描 + 管理端手动深扫 | -| 对齐方式 | 迁移 MinIO 行为：`heal.bitrot_cycle=off` 或 `RUSTFS_SCANNER_BITROT_CYCLE_SECS=disabled` | 反向：`heal:bitrotscan=<秒>` | - -RustFS 的 30 天默认是刻意的耐用性默认（周期性全量 bitrot 校验），代价是每 30 天一轮深扫 IO；单机场景另有清洁空闲退避封顶约 42 分钟的墙钟保护（见 scanner-runtime-controls.md）。这是行为差异而非缺陷，文档化即可。 - -## 3. alert_excess_folders 默认差异 - -RustFS 默认 65538（容纳 Proxmox Backup Server 每目录 65536 chunk 的布局），MinIO 默认 50000。差异原因、另两个 excess 阈值（versions=100 相同、version_size TiB vs TB）、事件名映射与冷却语义已完整记录在 [Scanner Excess Alerts](scanner-excess-alerts_zh.md)，此处不重复。 - -## 4. HS-18：stale multipart 与 tmp/.trash 清理三段核对 - -MinIO 把"清理已删除数据"拆成三段：stale upload 先 rename 进 `.minio.sys/tmp/.trash/` 隔离（rename 快、原子）；trash 由独立例程排空；tmp 下非 trash 的旧目录单独回收。逐段核对 RustFS： - -| 段 | MinIO | RustFS | 判定 | -|---|---|---|---| -| stale multipart → 隔离 | `cleanupStaleUploadsOnDisk`（cmd/erasure-multipart.go）逐盘列出 multipart 目录，按 uploadID 目录名里的 UnixNano 判龄，超过 `stale_uploads_expiry`（默认 24h）即 `renameAll` 进 `.minio.sys/tmp/.trash/`，空 sha 目录、tmp 旧目录同法 | `cleanup_stale_multipart_uploads_in_set`（crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs）发现候选后取 ns 写锁 + 重查（`lock_stale_multipart_cleanup`），`delete_all_with_quorum` 扇出逐盘递归删除，而 LocalDisk 的递归删除内部就是 `move_to_trash`（crates/ecstore/src/disk/local.rs）把目录 rename 进 `.rustfs.sys/tmp/.trash/` | 行为等价（都是先隔离后清理）；RustFS 额外有写锁 + quorum 重查 + 锁丢失 fence（crates/ecstore/src/set_disk/ops/multipart.rs 的 `StaleMultipartCleanupGuard`），防并发 CompleteMultipartUpload 竞争，安全性强于 MinIO 的无锁 rename | -| trash 排空 | 每 `delete_cleanup_interval`（默认 5m，internal/config/api/api.go）逐盘删 `.trash` 内条目，逐条以 `deleteCleanupSleeper`（factor 5 / 25ms，cmd/globals.go）节流 | 每盘独立 `cleanup_deleted_objects_loop`，`DELETED_OBJECTS_CLEANUP_INTERVAL` = 5m（crates/ecstore/src/disk/local.rs），先排空 `.trash` 再回收 tmp 旧目录；排空为顺序 `remove_dir_all`/`remove_file`，**无逐条 sleep 节流** | 基本等价；唯一差异是 RustFS 排空不节流，trash 积压大时单轮 IO 更突发（5m 周期天然限频），文档化，如实测出现清理风暴再补节流 | -| tmp 非 trash 旧目录 | 并在 `cleanupStaleUploadsOnDisk` 内：非 `.trash` 的 tmp 目录超过 `stale_uploads_expiry`（24h）rename 进 trash（随 6h 任务） | `cleanup_stale_tmp_objects`（crates/ecstore/src/disk/local.rs）随 5m 循环执行：非 `.trash` 目录超过 `STALE_TMP_OBJECT_EXPIRY` = 24h 即 rename 进 trash；另有启动时 tmp → tmp-old 整体换名 + 后台删除的崩溃安全路径 | 行为等价（阈值同为 24h）；RustFS 检查频率 5m vs MinIO 6h，回收更及时 | - -周期与环境变量默认值对照（两边一致）： - -| 项 | RustFS | MinIO | -|---|---|---| -| stale upload 过期阈值 | `RUSTFS_API_STALE_UPLOADS_EXPIRY`，默认 24h | `MINIO_API_STALE_UPLOADS_EXPIRY`，默认 24h | -| stale multipart 清理周期 | `RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL`，默认 6h | `MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL`，默认 6h | -| trash 排空周期 | 5m（常量，暂无开关） | `MINIO_API_DELETE_CLEANUP_INTERVAL`，默认 5m | - -关于 rustfs/src/delete_tail_activity.rs：它**不覆盖三段中的任何一段**。该模块是 delete 尾部活动的进程内指标计数（inflight gauge + 耗时 histogram），供 allocator 回收压力判断（rustfs/src/allocator_reclaim.rs）使用；生产代码目前只在对象复用路径使用 `Replication`/`Notify` 两个 stage 计数，`Tail`/`Cleanup` 枚举值暂无调用点。 - -崩溃残留窗口结论： - -- trash 内部残留（排空中途崩溃）：`.trash/` 是自包含目录，下一轮 5m tick 重扫 `.trash` 自然收敛，与 MinIO 相同。 -- 跨盘扇出中途崩溃（部分盘已 rename 进 trash、其余未动）：若剩余盘数仍满足写 quorum，下一轮 6h 任务重新发现候选并重删，自然收敛；若已清理盘数超过 parity（剩余低于写 quorum），`check_multipart_upload_path_exists` 因 `FileNotFound` 不在 `OBJECT_OP_IGNORED_ERRS`（crates/ecstore/src/disk/error_reduce.rs）而判 quorum 失败，候选被跳过，残留 uploadID 目录不会被该任务收敛（不可见于 S3 API，仅占盘空间）。该窗口极窄（逐盘 rename 为毫秒级，需恰在扇出中途且已过 parity 盘时进程死亡）。MinIO 同场景会收敛（逐盘独立处理、无 quorum 闸门）。**分级：有崩溃残留窗口（极窄）→ 登记后续修复**；修复需为清理守卫提供把"已不存在"计为达成终态的专用 quorum 变体（不能改共享的 `check_multipart_upload_path_exists` 语义，它同时服务 CompleteMultipartUpload），超出本批"几行小修"边界，不在本 PR 扩 scope。 - -## 5. HS-16：单机（ErasureSD）默认扫描周期决策 - -启动期曾有预留钩子 `single_disk_default_cycle_secs`，可按维护特征（lifecycle/replication/巡检失败）为单机覆盖默认周期，但从未接线、恒返回 `None`，已删除（本批 PR）。决策：**单机默认周期保持速度档派生（`default` 档 = 60s），不做特殊覆盖**。理由：其一，无任何实测依据表明单机冷启动 ILM 延迟需要更短周期，凭空缩短只会放大空闲扫描频次；其二，单机已有清洁空闲退避（连续干净周期间隔翻倍，默认 bitrot 窗口下封顶约 42 分钟，见 scanner-runtime-controls.md），空闲时的周期压力已被消化；其三，若确有诉求，用户可用 `RUSTFS_SCANNER_CYCLE` / `scanner.cycle` 显式配置，无需内置特殊路径。需要更激进短周期的场景应先拿实测数据再议。 - -## 6. 决策摘要 - -- HS-14：保持 `RUSTFS_SCANNER_IDLE_MODE` 现语义（true=节流总闸+前台读下限，false=全速），文档化对照表与迁移警告，不做兼容别名。 -- HS-16：删除恒 `None` 的单机默认周期钩子，单机周期保持速度档派生 + 清洁空闲退避。 -- HS-18：三段清理行为等价（trash 排空无逐条节流、tmp 回收频率 5m vs 6h 两处小差异文档化）；跨盘扇出的极窄崩溃残留窗口登记后续；周期默认值 24h/6h/5m 与 MinIO 对齐。 diff --git a/docs/operations/hotpath-warp-ab-runbook.md b/docs/operations/hotpath-warp-ab-runbook.md index 3f31da00f..4dbf708f3 100644 --- a/docs/operations/hotpath-warp-ab-runbook.md +++ b/docs/operations/hotpath-warp-ab-runbook.md @@ -1,164 +1,178 @@ -# Hotpath warp A/B runbook +# Hotpath warp runbook (A/B gate and ABBA evidence) -Relative-budget A/B gate for the hotpath series (rustfs/backlog#935 HP-14). It -runs the same warp workloads against a **baseline** binary and a **candidate** -binary, across the drive-sync on/off matrix, then applies a relative budget: -a metric regressing past the fail budget fails the gate, past the warn budget -warns. This is how the macOS profiling conclusions of the HP series get -confirmed or corrected on Linux — structural wins (call counts, read -amplification) should hold; absolute numbers are whatever the rig measures. +**Use this when:** you need S3-face (PutObject / GetObject / mixed) performance evidence for a code change: the nightly relative-budget gate, a quick local A/B, or a formal ABBA run whose numbers will be quoted in a PR. +**Source of truth:** `scripts/run_hotpath_warp_ab.sh` (quick A/B rig), `scripts/run_hotpath_warp_abba.sh` (ABBA runner; also what CI executes), `scripts/hotpath_warp_ab_gate.sh` (relative-budget gate), `scripts/run_object_batch_bench_enhanced.sh` (warp driver, medians, `baseline_compare.csv`), `.github/workflows/performance-ab.yml` (CI gate). -Pieces: +This runbook compares two binaries. To sweep one `RUSTFS_*` runtime knob at a time against a fixed binary, use [object-io-tuning-ab-matrix.md](object-io-tuning-ab-matrix.md) instead. -- `scripts/run_hotpath_warp_ab.sh` — orchestrator (baseline vs candidate, - workload × drive-sync matrix). -- `scripts/hotpath_warp_ab_gate.sh` — the budget gate over the - `baseline_compare.csv` deltas the load driver emits. -- `scripts/run_object_batch_bench_enhanced.sh` — the warp driver + median + - `baseline_compare.csv` (reused, not reimplemented). -- `.github/workflows/performance-ab.yml` — nightly on `main` (post-merge - detection) plus opt-in pre-merge via the `perf-ab` label. +Two entry points share one workload matrix, one gate, and one deploy-hook shape: -Metric directions: `reqps` (put obj/s) and `throughput` (get MiB/s) are -higher-is-better; `latency` / p99 (mixed) is lower-is-better. warp is assumed -pre-installed, as elsewhere in `scripts/`. +| Entry | Script | Legs per cell | Use | +| --- | --- | --- | --- | +| Quick A/B | `scripts/run_hotpath_warp_ab.sh` | baseline → candidate | local smoke, fast triage of a suspected regression | +| ABBA | `scripts/run_hotpath_warp_abba.sh` | A1 baseline → B1 candidate → B2 candidate → A2 baseline | formal evidence; the CI gate | + +This rig is the only entry point for S3-face performance coverage. There is deliberately no in-process criterion benchmark for those operations: a criterion harness that stands up an embedded server measures the harness, not the S3 path. Micro-benchmarks stay at function level (EC encode, `xl.meta` parse, `rename_data`). Knob-level sweeps against a fixed binary are a different question; see [object-io-tuning-ab-matrix.md](object-io-tuning-ab-matrix.md). + +## Shared prerequisites + +- Linux host. A laptop is acceptable for a quick A/B smoke only; formal evidence needs a dedicated runner or a cluster. +- `warp` on `PATH`, or `--warp-bin `. +- Two Linux release binaries, baseline and candidate (build each with `cargo build --release -p rustfs --bins` at its commit; cross-compile with `cargo zigbuild --release --target x86_64-unknown-linux-gnu -p rustfs --bins` for a cluster). +- Disposable disks or data root in local mode; an isolated benchmark bucket and credentials in cluster mode. Never bench against production data. +- Readiness polling (`--health-timeout`, default 180 s in both scripts) must outlast the server's own startup budget, `DEFAULT_STARTUP_READINESS_MAX_WAIT_SECS` in `crates/config/src/constants/health.rs`; a shorter poll misreports a slow cold start as a failure. +- Metric directions: `reqps` (put obj/s) and `throughput` (get MiB/s) are higher-is-better; `latency`/p99 (mixed) is lower-is-better. ## Workload matrix -Six workloads × the drive-sync on/off matrix × baseline/candidate = 24 cells: +Every workload runs with `RUSTFS_DRIVE_SYNC_ENABLE=true` and `=false`. Quick A/B: 6 × 2 × 2 = 24 cells; ABBA: 6 × 2 × 4 = 48 cells. -| Workload | mode | size | why | +| Workload | mode | size | Why | | --- | --- | --- | --- | -| `put-4kib` / `get-4kib` | put / get | 4KiB | the #4221 fsync regression size (~-10% @4KiB) — previously invisible | -| `put-4mib` / `get-4mib` | put / get | 4MiB | bulk obj/s and MiB/s | -| `get-10mib` | get | 10MiB | the historical large-GET EOF size | -| `mixed-256k` | mixed | 256KiB | p99 latency | +| `put-4kib` / `get-4kib` | put / get | 4 KiB | small-object fsync-sensitive path | +| `put-4mib` / `get-4mib` | put / get | 4 MiB | bulk obj/s and MiB/s | +| `get-10mib` | get | 10 MiB | large-GET streaming path | +| `mixed-256k` | mixed | 256 KiB | p99 latency | -Sizes are passed to the load driver via `--sizes` (one size per cell); the -driver's `DEFAULT_SIZES` covers 1KiB..10MiB, so any of those can be added by -editing `WORKLOADS` in `scripts/run_hotpath_warp_ab.sh`. A 1KiB cell is left -out for now to keep the nightly matrix comfortably under budget; re-enable it -(one line in `WORKLOADS`) once perf-6 recalibrates the warp params. +Sizes are passed to the driver as `--sizes` (one per cell); any size in the driver's `DEFAULT_SIZES` (`scripts/run_object_batch_bench_enhanced.sh`) can be added by editing the `WORKLOADS` array in the chosen script. -CI runs a **short** warp matrix (`--duration`/`--rounds`/`--cooldown` tuned in -`.github/workflows/performance-ab.yml`) so all 24 cells fit the budget without -dropping cells. These params are deliberately noisy-but-fast for the Phase-0 -"keep the pipeline alive" goal; perf-6 recalibrates them. +## Gate -## Baseline binary cache (CI) +`scripts/hotpath_warp_ab_gate.sh` compares each metric against baseline: a regression beyond `--fail-pct` (default `FAIL_PCT=10`) fails, beyond `--warn-pct` (default `WARN_PCT=5`) warns. `--allow-regression --exemption-reason ""` records a FAIL as an exempted WARN and exits 0; use it only for a deliberate correctness trade (for example paying write cost to restore power-loss durability). -The nightly no longer builds both binaries from source. Every push to `main` -runs a `build-baseline-cache` job that builds the release binary once and stores -it in the actions cache under `rustfs-baseline-` (perf-3). The A/B job -restores the binary for `origin/main` by that key and passes it as -`--baseline-bin`; on the nightly, where the candidate commit equals the baseline -commit, the same cached binary serves both phases (`--skip-build`) and the run -does zero source builds — the common path finishes well under 50 minutes. A -cache miss (binary evicted, or not built for that SHA yet) transparently falls -back to the source double-build via `--baseline-ref origin/main`. +## Deploy-hook contract (external / cluster mode) -Each `gate.md` ends with a **Provenance** section recording the baseline and -candidate commit SHAs and whether each binary came from the cache or a source -build, plus the runner, warp version, and matrix params. `perf-5`/`perf-12` -reuse this contract for their archived baselines. +In external mode (`--endpoint `) the rig never starts or restarts RustFS. Before each phase or leg it runs `--deploy-hook ` with the context below in the environment, then waits for `http://` (default `/health`). A non-zero hook exit aborts the run. -## Local mode (quick / CI smoke) +| Variable | Quick A/B | ABBA | Value | +| --- | --- | --- | --- | +| `HOTPATH_AB_PHASE` / `HOTPATH_ABBA_PHASE` | yes | yes | `baseline` or `candidate` | +| `HOTPATH_AB_BINARY` / `HOTPATH_ABBA_BINARY` | yes | yes | selected binary path (A/B: may be empty if the hook builds its own) | +| `HOTPATH_AB_DRIVE_SYNC` / `HOTPATH_ABBA_DRIVE_SYNC` | yes | yes | `true` or `false` for this cell | +| `HOTPATH_ABBA_LEG` | — | yes | `A1`, `B1`, `B2`, `A2` | +| `HOTPATH_ABBA_WORKLOAD`, `HOTPATH_ABBA_MODE`, `HOTPATH_ABBA_SIZE`, `HOTPATH_ABBA_CELL_ID` | — | yes | cell identity | +| `HOTPATH_ABBA_DATASET_NAMESPACE`, `HOTPATH_ABBA_BUCKET` | — | yes | run namespace and per-leg benchmark bucket the hook must provision or reset | +| `HOTPATH_ABBA_DEPLOY_EVIDENCE_FILE` | — | yes | path the hook must write a non-empty evidence file to | -Builds both binaries and runs a throwaway single-node server on local disks. +The ABBA runner refuses external mode without a hook unless `--allow-unmanaged-external` is passed, and output from that mode is not formal evidence. + +Ansible-shaped hook (replace `/path/to/ansible` and the inventory group; the `config` tag must thread `RUSTFS_DRIVE_SYNC_ENABLE`, or the finer `RUSTFS_DURABILITY_MODE`, into the deployed unit): ```bash +--deploy-hook ' + set -euo pipefail + cd /path/to/ansible + cp "${HOTPATH_ABBA_BINARY:?}" roles/rustfs/files/rustfs + export RUSTFS_DRIVE_SYNC_ENABLE="${HOTPATH_ABBA_DRIVE_SYNC:?}" + ansible-playbook -f 4 -l bench rustfs-manage.yml --tags stop + ansible-playbook -f 4 -l bench rustfs-manage.yml --tags config + ansible-playbook -f 4 -l bench rustfs-manage.yml --tags binary-copy + ansible-playbook -f 4 -l bench rustfs-manage.yml --tags start +' +``` + +For the quick A/B rig use the `HOTPATH_AB_*` names. + +## Quick A/B (`run_hotpath_warp_ab.sh`) + +```bash +# build both binaries (baseline from --baseline-ref, default origin/main) and run a throwaway single-node server on local disks scripts/run_hotpath_warp_ab.sh --baseline-ref origin/main -# or with prebuilt binaries: -scripts/run_hotpath_warp_ab.sh --skip-build \ - --baseline-bin ./rustfs-main --candidate-bin ./target/release/rustfs + +# prebuilt binaries +scripts/run_hotpath_warp_ab.sh --skip-build --baseline-bin ./rustfs-main --candidate-bin ./target/release/rustfs + +# print the plan only +scripts/run_hotpath_warp_ab.sh --dry-run --skip-build --baseline-bin /tmp/base --candidate-bin /tmp/cand + +# external cluster +scripts/run_hotpath_warp_ab.sh --endpoint "$CLUSTER_ENDPOINT" --deploy-hook '' \ + --baseline-bin /path/to/rustfs-main --candidate-bin ./target/x86_64-unknown-linux-gnu/release/rustfs ``` -Preview the full plan without running anything: +Outputs under `target/hotpath-ab//`: `gate.md` (ends with a **Provenance** section: baseline/candidate SHAs, binary source, runner, warp version, matrix params; extend with `--provenance-note`) and `server-logs/-sync-.{log,env}`. On a health-check failure the rig prints the last 50 server log lines; in local mode it fails fast if the server process exits before becoming healthy. + +## ABBA evidence run (`run_hotpath_warp_abba.sh`) + +`B1` and `B2` are compared with `A1` for the candidate delta; `A2` is compared with `A1` for baseline drift. Required flags: `--baseline-bin`, `--candidate-bin`, `--baseline-revision`, `--candidate-revision`. The script enforces `--rounds >= 3`; prefer `--rounds 5` or more for formal evidence when budget allows. ```bash -scripts/run_hotpath_warp_ab.sh --dry-run --skip-build \ - --baseline-bin /tmp/base --candidate-bin /tmp/cand +# local Linux runner (throwaway data root; a reused run namespace is rejected) +scripts/run_hotpath_warp_abba.sh \ + --baseline-bin /tmp/rustfs-baseline --candidate-bin /tmp/rustfs-candidate \ + --baseline-revision "$(git rev-parse origin/main)" --candidate-revision "$(git rev-parse HEAD)" \ + --address 127.0.0.1:9000 --data-root /var/tmp/rustfs-hotpath-abba --disks 4 \ + --duration 120s --rounds 3 --cooldown 30 --concurrency 16 \ + --out-dir target/hotpath-abba/linux-local + +# production-like cluster +scripts/run_hotpath_warp_abba.sh \ + --baseline-bin /srv/rustfs-binaries/rustfs-baseline --candidate-bin /srv/rustfs-binaries/rustfs-candidate \ + --baseline-revision --candidate-revision \ + --endpoint rustfs-bench.example.internal:9000 --deploy-hook '' \ + --duration 180s --rounds 5 --cooldown 45 --concurrency 32 \ + --out-dir target/hotpath-abba/cluster-pr-XXXX ``` -## External mode (real cluster, ansible-deployed) +Add `--dry-run` to print the schedule without starting servers or warp. Output layout: -For the production-representative run, warp targets an already-running cluster -and a `--deploy-hook` swaps in each phase's binary and durability config -between the baseline and candidate phases. The hook receives context via the -environment: +```text +/ + manifest.env + abba_schedule.csv + candidate_gate.md + baseline_drift_gate.md + summary.md + ///median_summary.csv + ///baseline_compare.csv +``` -- `HOTPATH_AB_PHASE` — `baseline` or `candidate` -- `HOTPATH_AB_BINARY` — binary path (or empty; the hook may build its own) -- `HOTPATH_AB_DRIVE_SYNC` — `true` or `false` for this matrix cell +Attach to the PR or issue: `summary.md`, `candidate_gate.md`, `baseline_drift_gate.md`, `abba_schedule.csv`, every `median_summary.csv` and `baseline_compare.csv` for a failed or borderline workload, and the host telemetry used to explain saturation. Preserve the output directory unmodified. -This maps directly onto the team's ansible harness. Build the candidate with -the cross toolchain, stage both binaries, then let the hook drive -`rustfs-manage.yml`: +### Interpretation + +| Candidate gate | A2 drift gate | Interpretation | +| --- | --- | --- | +| PASS | PASS | Candidate acceptable for the measured matrix. | +| WARN | PASS | Small measurable signal; inspect telemetry and decide whether it is expected. | +| FAIL | PASS | Candidate likely regressed the workload; investigate before merge. | +| FAIL | FAIL on the same workload | Environment drift is high; rerun on a quieter runner or raise duration and rounds. | +| PASS | FAIL | Rig unstable; do not quote the numbers as proof of improvement. | + +Rules that override the table: a candidate result is actionable only when the `A2` drift for the same workload passes or is materially smaller than the `B1`/`B2` delta; never report a win or loss for a workload whose drift gate failed without a rerun. When `B1` and `B2` disagree, the cell is inconclusive even if the gate passes. Report only measured facts: deltas, drift, saturation, failed workloads. + +### CPU and memory evidence + +Warp output says whether throughput or latency changed; host telemetry says why. Collect it for the whole run and stop the collectors after the script exits. + +| Tool | Command | Answers | +| --- | --- | --- | +| `pidstat` | `pidstat -durh 5 > /telemetry/pidstat.txt &` | per-process CPU, memory, disk | +| `mpstat` | `mpstat 5 > /telemetry/mpstat.txt &` | CPU saturation and steal | +| `iostat` | `iostat -xz 5 > /telemetry/iostat.txt &` | device queue depth and latency | +| `perf` | `perf record -F 99 -g -- sleep 180` around one representative cell, then `perf report --stdio` | CPU attribution after the gate shows an effect | + +samply against a running RustFS process goes through the bounded helper, one attach window per leg or focused cell: ```bash -# 1. Build the candidate (cross-compile for the cluster target). -cargo zigbuild --release --target x86_64-unknown-linux-gnu -p rustfs --bins - -# 2. Run the A/B against the cluster; the hook deploys the phase's binary and -# applies the drive-sync config, then restarts, before each phase. -scripts/run_hotpath_warp_ab.sh \ - --endpoint "$CLUSTER_ENDPOINT" \ - --deploy-hook ' - set -euo pipefail - cd /home/xiaomage/xiaomage/ansible - # Select the phase binary and the drive-sync value for this cell. - cp "${HOTPATH_AB_BINARY:?}" ./roles/rustfs/files/rustfs - export RUSTFS_DRIVE_SYNC_ENABLE="$HOTPATH_AB_DRIVE_SYNC" - ansible-playbook -f 4 -l testing rustfs-manage.yml --tags stop - ansible-playbook -f 4 -l testing rustfs-manage.yml --tags config - ansible-playbook -f 4 -l testing rustfs-manage.yml --tags binary-copy - ansible-playbook -f 4 -l testing rustfs-manage.yml --tags start - ' \ - --baseline-bin /path/to/rustfs-main \ - --candidate-bin ./target/x86_64-unknown-linux-gnu/release/rustfs +scripts/run_samply_attach_window.sh --pid "$RUSTFS_PID" --duration-secs 180 \ + --output /telemetry/samply-A1-get-4mib.json.gz ``` -The `config` tag is responsible for threading `RUSTFS_DRIVE_SYNC_ENABLE` (or -the finer `RUSTFS_DURABILITY_MODE`) into the deployed unit — the hook exports -it so the config template can pick it up. The rig itself never restarts the -cluster; lifecycle stays with ansible. +After each window confirm the `.json.gz` profile and its `.syms.json` sidecar are non-empty, no `samply` process is still attached to the PID, and any temporary `perf_event_paranoid` change is restored; reject the cell otherwise. -## Budget and exemptions +Instrumented builds (features in `rustfs/Cargo.toml`): `--features hotpath-alloc` for allocation attribution, `--features hotpath-cpu` for CPU hotpath sections. Compare instrumented binaries only with other builds of the same mode; never use them for throughput acceptance, because the instrumentation changes what is measured. -Default budget: a metric regressing more than **10%** vs baseline fails, -more than **5%** warns. Tune with `--fail-pct` / `--warn-pct`. +## CI gate (`performance-ab.yml`) -Some regressions are the correct trade — #4221 deliberately paid a large write -cost to restore power-loss durability. For those, run with -`--allow-regression` (or add the `perf-deliberate-tradeoff` label in CI): the -FAIL is recorded and rendered as an exempted WARN, and the gate exits 0. +| Aspect | Value | +| --- | --- | +| Triggers | `schedule` (nightly cron `31 6 * * *` UTC against `main`) and `workflow_dispatch` (inputs `duration`, default `12s`; `allow_regression`, boolean). No `pull_request` trigger and no label gating. | +| Jobs | `warp-ab` (runner `sm-standard-2`, `timeout-minutes: 180`); `alert-on-failure` (opens the scheduled-failure issue; scheduled runs only). | +| Baseline commit | scheduled: head of the last successful scheduled run (falls back to the candidate itself when there is none); dispatch: `origin/main`. The baseline must be an ancestor of the candidate. | +| Binary cache | `actions/cache` inside `warp-ab`, key `rustfs-baseline-`. Miss: source build (same-commit runs build once and reuse the binary for both phases); a run whose gate passes saves the candidate binary under `rustfs-baseline-` for the next night. | +| Command | `scripts/run_hotpath_warp_abba.sh --duration --rounds 3 --cooldown 5 --health-timeout 180 --baseline-revision --candidate-revision --baseline-bin ... --candidate-bin ...` | +| Exemption | `allow_regression=true` on dispatch adds `--allow-regression --exemption-reason "workflow dispatch override"`. | +| Artifacts | `hotpath-warp-ab-` containing `target/hotpath-abba/` (14-day retention); the step summary renders `candidate_gate.md`, or the server-log tails when the rig failed before the gate. The `Enforce gate` step fails the job on a non-zero rig exit. | -## Diagnosing a failed run - -Each phase's server log and its startup environment are written under the run's -output dir (`target/hotpath-ab//server-logs/-sync-.{log,env}`) -and uploaded in the `hotpath-warp-ab-` artifact, so a failure is -diagnosable after the fact. On a health-check failure the rig also dumps the -last 50 log lines into the job log and the CI job writes the failing phase (or -the gate table) into the GitHub step summary. - -Readiness polling waits up to `--health-timeout` seconds (default **180**), -which must outlast the server's own startup-readiness budget -(`RUSTFS_STARTUP_READINESS_MAX_WAIT_SECS`, default 120s) — a shorter poll on a -slow shared runner misreports a slow cold start as a failure. In local mode the -rig also fails fast if the server process exits before becoming healthy instead -of polling out the full budget. - -## Scope note - -The gate logic is unit-validated across pass/warn/fail/exempt outcomes; the -orchestrator and workflow are shellcheck- and `--dry-run`-validated. The first -real warp measurement belongs on a Linux runner or the ansible cluster — there -is no warp/multi-disk rig in the repo's local checkout. - -This warp A/B gate is the **only** entry point for S3-face (PutObject / -GetObject / ListObjects) performance coverage. There is deliberately no -in-process criterion benchmark for those operations: a criterion harness that -stands up an embedded server measures the harness, not the S3 path, so it would -report a number without guarding anything. Micro-benchmarks stay at the -function level (EC encode, `xl.meta` parse, `rename_data`; perf-8). +The nightly detects a regression within a day of landing; it does not block a merge. For pre-merge evidence run the ABBA procedure above and attach the outputs to the PR. diff --git a/docs/operations/hotpath-warp-abba-runbook.md b/docs/operations/hotpath-warp-abba-runbook.md deleted file mode 100644 index 7b0e290cf..000000000 --- a/docs/operations/hotpath-warp-abba-runbook.md +++ /dev/null @@ -1,296 +0,0 @@ -# Hotpath warp ABBA validation runbook - -This runbook describes how to collect formal Linux or production-cluster -evidence for hotpath performance changes. Use it when a short local A/B smoke -run is too noisy to decide whether a regression is real. - -The ABBA runner executes each workload and drive-sync cell as: - -```text -A1 baseline -> B1 candidate -> B2 candidate -> A2 baseline -``` - -`B1` and `B2` are compared with `A1` to measure the candidate delta. `A2` is -also compared with `A1` to measure baseline drift. Treat a candidate regression -as actionable only when the `A2` drift is passing or materially smaller than -the `B1` and `B2` delta for the same workload. - -## Scope - -Use this runbook for hotpath profiling and performance validation of RustFS -object I/O changes, especially when CPU, memory allocation, lock/channel wait -time, request throughput, or tail latency is the review question. - -The script validates the same workload matrix as the hotpath warp A/B gate: - -| Workload | mode | size | -| --- | --- | --- | -| `put-4kib` | put | 4KiB | -| `put-4mib` | put | 4MiB | -| `get-4kib` | get | 4KiB | -| `get-4mib` | get | 4MiB | -| `get-10mib` | get | 10MiB | -| `mixed-256k` | mixed | 256KiB | - -Each workload runs with `RUSTFS_DRIVE_SYNC_ENABLE=true` and -`RUSTFS_DRIVE_SYNC_ENABLE=false`, so a full ABBA pass produces 48 measurement -cells: 6 workloads x 2 drive-sync modes x 4 ABBA legs. - -## Prerequisites - -Run the formal pass on Linux, not on a laptop smoke environment. - -Required tools on the bench host: - -- `bash`, `curl`, `git`, and core GNU userland. -- `warp` on `PATH`, or pass `--warp-bin`. -- Two RustFS Linux binaries: one baseline and one candidate. -- Enough isolated disks or directories for the local runner, or an externally - managed RustFS cluster for production-like validation. -- Stable host telemetry collection such as `pidstat`, `mpstat`, `iostat`, - `sar`, `perf`, `heaptrack`, or the platform's equivalent observability stack. - -Cluster-mode requirements: - -- A deploy hook that can replace the RustFS binary on every node. -- The hook must apply `RUSTFS_DRIVE_SYNC_ENABLE` for the current ABBA leg. -- The hook must restart RustFS and return only after the rollout command has - been accepted. The ABBA script performs the HTTP readiness wait. -- The benchmark client should run outside the RustFS nodes when possible. -- Do not run against a production data set unless the workload bucket and test - credentials are isolated and approved for destructive benchmark traffic. - -## Build the binaries - -Build the baseline from the comparison commit, usually `origin/main` or the -previous accepted release: - -```bash -git fetch origin main -git switch --detach origin/main -cargo build --release -p rustfs --bins -cp target/release/rustfs /tmp/rustfs-baseline -``` - -Build the candidate from the PR commit: - -```bash -git switch -cargo build --release -p rustfs --bins -cp target/release/rustfs /tmp/rustfs-candidate -``` - -For cross-compiled cluster binaries, keep both outputs on the bench host and -make the deploy hook copy the selected binary to the cluster. The ABBA runner -passes the selected binary path through `HOTPATH_ABBA_BINARY`. - -## Local Linux runner - -Use local mode for a dedicated Linux runner with disposable data paths. This is -not a substitute for a production-like cluster, but it is useful before spending -cluster time. - -```bash -scripts/run_hotpath_warp_abba.sh \ - --baseline-bin /tmp/rustfs-baseline \ - --candidate-bin /tmp/rustfs-candidate \ - --address 127.0.0.1:9000 \ - --data-root /var/tmp/rustfs-hotpath-abba \ - --disks 4 \ - --duration 120s \ - --rounds 3 \ - --cooldown 30 \ - --concurrency 16 \ - --out-dir target/hotpath-abba/linux-local -``` - -The script starts and stops RustFS for each ABBA leg. The data root is -throwaway and should not contain important data. - -## Production-like cluster runner - -Use external mode when RustFS lifecycle is managed by ansible, systemd, a -cluster scheduler, or a dedicated deployment harness. In this mode the ABBA -script does not start RustFS directly; it calls `--deploy-hook` before each leg -and then waits for `http://`. - -The deploy hook receives: - -| Environment variable | Value | -| --- | --- | -| `HOTPATH_ABBA_LEG` | `A1`, `B1`, `B2`, or `A2` | -| `HOTPATH_ABBA_PHASE` | `baseline` or `candidate` | -| `HOTPATH_ABBA_BINARY` | selected baseline or candidate binary path | -| `HOTPATH_ABBA_DRIVE_SYNC` | `true` or `false` | - -Example ansible-shaped command: - -```bash -scripts/run_hotpath_warp_abba.sh \ - --baseline-bin /srv/rustfs-binaries/rustfs-baseline \ - --candidate-bin /srv/rustfs-binaries/rustfs-candidate \ - --endpoint rustfs-bench.example.internal:9000 \ - --deploy-hook ' - set -euo pipefail - cd /srv/rustfs-ansible - cp "${HOTPATH_ABBA_BINARY:?}" roles/rustfs/files/rustfs - export RUSTFS_DRIVE_SYNC_ENABLE="${HOTPATH_ABBA_DRIVE_SYNC:?}" - ansible-playbook -f 4 -l bench rustfs-manage.yml --tags stop - ansible-playbook -f 4 -l bench rustfs-manage.yml --tags config - ansible-playbook -f 4 -l bench rustfs-manage.yml --tags binary-copy - ansible-playbook -f 4 -l bench rustfs-manage.yml --tags start - ' \ - --duration 180s \ - --rounds 5 \ - --cooldown 45 \ - --concurrency 32 \ - --out-dir target/hotpath-abba/cluster-pr-XXXX -``` - -For formal evidence, prefer `--rounds 5` or higher when the cluster budget -allows it. The script enforces `--rounds >= 3`. - -## CPU and memory evidence - -ABBA warp output answers whether the candidate changed throughput or latency. -Collect host telemetry at the same time to explain why. - -Recommended minimum: - -```bash -mkdir -p target/hotpath-abba/cluster-pr-XXXX/telemetry - -pidstat -durh 5 > target/hotpath-abba/cluster-pr-XXXX/telemetry/pidstat.txt & -PIDSTAT_PID=$! - -mpstat 5 > target/hotpath-abba/cluster-pr-XXXX/telemetry/mpstat.txt & -MPSTAT_PID=$! - -iostat -xz 5 > target/hotpath-abba/cluster-pr-XXXX/telemetry/iostat.txt & -IOSTAT_PID=$! -``` - -Stop the collectors after the ABBA script exits: - -```bash -kill "$PIDSTAT_PID" "$MPSTAT_PID" "$IOSTAT_PID" -``` - -For deeper CPU attribution, run `perf record` around one representative -workload after the ABBA gate identifies a candidate regression or improvement: - -```bash -perf record -F 99 -g -- sleep 180 -perf report --stdio > target/hotpath-abba/cluster-pr-XXXX/telemetry/perf-report.txt -``` - -When using samply against an already-running RustFS service, attach through the -bounded helper instead of calling `samply record -p` directly: - -```bash -scripts/run_samply_attach_window.sh \ - --pid "$RUSTFS_PID" \ - --duration-secs 180 \ - --output target/hotpath-abba/cluster-pr-XXXX/telemetry/samply-A1-get-4mib.json.gz -``` - -Run one attach window per ABBA leg or focused verification cell. After every -window, confirm that the `.json.gz` profile and `.syms.json` sidecar are -non-empty, that no `samply` process is still attached to the RustFS PID, and -that any temporary `perf_event_paranoid` change has been restored before the -next cell starts. - -For allocation profiling, build the candidate with: - -```bash -cargo build --release -p rustfs --bins --features hotpath-alloc -``` - -Then run the same ABBA command with that binary. Compare allocation-heavy -function sections only within the same build mode. Do not compare -`hotpath-alloc` binaries directly with default release binaries for throughput -acceptance, because allocation instrumentation intentionally changes what is -measured. - -For CPU hotpath sections emitted by hotpath, build with: - -```bash -cargo build --release -p rustfs --bins --features hotpath-cpu -``` - -Use the CPU-enabled report to explain hotspots after the default or plain -`hotpath` ABBA gate shows a real effect. - -## Output layout - -The ABBA runner writes: - -```text -/ - manifest.env - abba_schedule.csv - candidate_gate.md - baseline_drift_gate.md - summary.md - ///median_summary.csv - ///baseline_compare.csv -``` - -Attach or link at least these files in the issue or PR: - -- `summary.md` -- `candidate_gate.md` -- `baseline_drift_gate.md` -- `abba_schedule.csv` -- every `median_summary.csv` and `baseline_compare.csv` for a failed or - borderline workload -- host telemetry files used to explain CPU, memory, or disk saturation - -## Interpretation - -Use this decision table: - -| Candidate gate | A2 drift gate | Interpretation | -| --- | --- | --- | -| PASS | PASS | Candidate is acceptable for the measured matrix. | -| WARN | PASS | Candidate has a small measurable signal; inspect telemetry and decide if it is expected. | -| FAIL | PASS | Candidate likely regressed the affected workload; investigate before merge. | -| FAIL | FAIL on the same workload | Environment drift is high; rerun on a quieter runner or increase duration and rounds. | -| PASS | FAIL | Candidate did not exceed the budget, but the rig was unstable; avoid using the numbers as proof of improvement. | - -When `B1` and `B2` disagree, treat the result as inconclusive even if the gate -passes. Increase duration, rounds, cooldown, or runner isolation before drawing -a conclusion. - -## AI execution checklist - -When delegating the run to an AI agent or an automation runner, provide these -inputs explicitly: - -- repository checkout and candidate branch or commit; -- baseline commit or binary path; -- candidate binary path; -- runner type: local Linux or external cluster; -- endpoint, access key, secret key source, and region; -- deploy hook path or exact command for cluster mode; -- output directory; -- required duration, rounds, cooldown, concurrency, and fail/warn budgets; -- where to upload artifacts after the run. - -The AI agent should execute this sequence: - -1. Confirm `uname -a`, RustFS commits, binary SHA256 sums, `warp --version`, - CPU model, memory size, disk layout, and whether the run is local or cluster. -2. Run `scripts/run_hotpath_warp_abba.sh --dry-run` with the final arguments. -3. Run the real ABBA command with `--rounds >= 3`. -4. For samply CPU attribution, use `scripts/run_samply_attach_window.sh` for - each bounded attach window and reject the cell if the profile is empty or a - stale `samply` process remains. -5. Preserve the full output directory without editing generated CSV files. -6. Read `summary.md`, `candidate_gate.md`, and `baseline_drift_gate.md`. -7. Summarize only measured facts: candidate deltas, baseline drift, CPU or - memory saturation, and any failed workloads. -8. Post the summary and artifact location to the tracking issue or PR. - -Do not report a performance win or loss when the baseline drift gate failed on -the same workload and no rerun was collected. diff --git a/docs/operations/internode-grpc-benchmark-runbook.md b/docs/operations/internode-grpc-benchmark-runbook.md index 5615bc1b7..2c2fb6385 100644 --- a/docs/operations/internode-grpc-benchmark-runbook.md +++ b/docs/operations/internode-grpc-benchmark-runbook.md @@ -1,146 +1,94 @@ -# Internode gRPC Optimization — A/B Benchmark Runbook +# Internode gRPC A/B benchmark runbook -Reproducible procedure to collect **before/after** artifacts for each internode gRPC -optimization stage (grpc-optimization P0–P3). Every stage is env-gated, so "before" and -"after" are the *same binary* with different env — no rebuild between runs. +**Use this when:** collecting before/after evidence for an env-gated internode gRPC transport stage (P0 transport tuning, P1 channel isolation, P2 msgpack-only codec, P3 prewarm/offline bypass) on a real multi-node cluster. +**Source of truth:** `scripts/run_internode_grpc_ab_bench.sh` (stage/phase driver), `crates/config/src/constants/internode.rs` (`ENV_INTERNODE_*` / `DEFAULT_INTERNODE_*`), `crates/io-metrics/src/internode_metrics.rs` (metric names), [internode-msgpack-json-convergence-runbook.md](internode-msgpack-json-convergence-runbook.md) (the P2 gate). -> Live runs need a multi-node cluster (Docker or ≥2 rustfs endpoints), a load tool -> (`warp` or `s3bench`), and a Prometheus scrape of `/metrics`. They are not runnable in a -> single-process sandbox. Capture artifacts on a real cluster. +Every stage is env-gated, so "before" and "after" run the *same binary* with different server env; no rebuild between runs. Live runs need a multi-node cluster (Docker compose or two or more endpoints), a load tool (`warp` or `s3bench`), and a metrics sink; they are not runnable in a single-process sandbox. -## One-click driver +## Prerequisites -`scripts/run_internode_grpc_ab_bench.sh --stage --phase [-- ]` -wraps the env matrix below: it writes the stage/phase **server** env to -`/server-env.sh`, then runs the right underlying bench into -`target/bench/internode-transport/-/`. +| Requirement | Detail | +| --- | --- | +| RPC secret | Internode RPC fails closed: remote endpoints with default credentials and no `RUSTFS_RPC_SECRET` abort startup with `store init aborted: endpoints include remote nodes but ...` (`crates/ecstore/src/store/init.rs`). Set a non-default `RUSTFS_RPC_SECRET`, identical on every node. | +| systemd start timeout | `deploy/build/rustfs.service` is `Type=notify` and ships `TimeoutStartSec=120s`; READY fires only after quorum. If freshly purged disks need longer, raise it in a drop-in rather than lowering it. | +| Metrics export | RustFS has no Prometheus pull endpoint (`/admin/v3/metrics` is NDJSON, see `rustfs/src/admin/handlers/metrics.rs`); it pushes OTLP. Run an otel-collector (OTLP receiver → Prometheus exporter) and set `RUSTFS_OBS_ENDPOINT`, `RUSTFS_OBS_METRICS_EXPORT_ENABLED=true`, `RUSTFS_OBS_METER_INTERVAL=5`. For lock p99 also set `RUSTFS_OBJECT_LOCK_DIAG_ENABLE=true` (default off). | +| Server env | `RUSTFS_INTERNODE_*` are **server** env. For p0/p1/p2, source the emitted `server-env.sh` on every node and restart before the run; the driver cannot mutate a running server. | + +## Driver ```bash -# P1 A/B (restart the cluster with each phase's server-env.sh between the two runs): -scripts/run_internode_grpc_ab_bench.sh --stage p1 --phase before -- --access-key AK --secret-key SK --metrics-url http://node1:9000/metrics -scripts/run_internode_grpc_ab_bench.sh --stage p1 --phase after -- --access-key AK --secret-key SK --metrics-url http://node1:9000/metrics -# P3 failover A/B (docker four-node): -scripts/run_internode_grpc_ab_bench.sh --stage p3 --phase after -# P2 rollout gates (env preview for request-only rehearsal, canary, and rollback): -scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase request-only --dry-run -scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase canary --dry-run -scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase rollback --dry-run +scripts/run_internode_grpc_ab_bench.sh --stage --phase [--dry-run] [-- ] ``` -`RUSTFS_INTERNODE_*` are **server** env: for the load-driven stages (p0/p1/p2) source the emitted `server-env.sh` on every node and restart rustfs *before* the run — the driver cannot mutate an already-running server. The P2 `canary` phase is the exception: source it only on the selected canary node after the release-window counters and fleet-support checks pass, and keep the rest of the fleet on `before` or `request-only` while observing fallback/decode-error counters. Use `--dry-run` to preview the env and command. +The driver writes the stage/phase server env to `/server-env.sh` and runs the underlying bench into `target/bench/internode-transport/-/`. `p0/p1/p3` accept `before|after`; `p2` also accepts `request-only|canary|rollback`. `--dry-run` prints the env and command only. -## Harness +```bash +# P1 A/B: restart the cluster with each phase's server-env.sh between the two runs +scripts/run_internode_grpc_ab_bench.sh --stage p1 --phase before -- --access-key AK --secret-key SK --metrics-url http://node1:9000/metrics +scripts/run_internode_grpc_ab_bench.sh --stage p1 --phase after -- --access-key AK --secret-key SK --metrics-url http://node1:9000/metrics +# P3 failover A/B (docker four-node) +scripts/run_internode_grpc_ab_bench.sh --stage p3 --phase after +# P2 env previews +scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase canary --dry-run +``` -- Throughput / latency: `scripts/run_internode_transport_baseline.sh` (drives - `run_object_batch_bench.sh`; writes `target/bench/internode-transport-/`). Pass - `--metrics-url ` to also capture internode metric deltas. -- Failover / offline: `scripts/run_four_node_cluster_failover_bench.sh` (spins up a 4-node - compose cluster, kills `FAILOVER_NODE`, benchmarks; writes - `target/bench/four-node-failover-/`). - -The one-click driver writes each run to `target/bench/internode-transport/-/` -(e.g. `p0-before/`, `p0-after/`, `p1-before/`, `p1-after/`, `p3-before/`, `p3-after/`), each -containing the emitted `server-env.sh` plus the underlying bench artifacts. `target/` is -gitignored — attach the paired directories to the PR / issue. - -## Metrics to capture (Prometheus) - -| Metric | Stage signal | -|---|---| -| `rustfs_system_network_internode_operation_duration_ms{operation,backend}` | control-plane RTT (P0), lock/bulk latency | -| `rustfs_system_network_internode_operation_payload_bytes` | payload size distribution (P0/P1 sizing) | -| `rustfs_system_network_internode_operation_large_payloads_total` | large unary RPCs sharing the channel (P1 target) | -| `rustfs_system_network_internode_dial_avg_time_nanos`, `..._dial_errors_total` | connect cost (P3 prewarm) | -| `rustfs_system_network_internode_msgpack_json_decode_total{direction,message,codec}` | must be **>0** for each expected P2 series before zero fallback/error readings are meaningful | -| `rustfs_system_network_internode_msgpack_json_fallback_total{direction,message}` | must be **0** before enabling both msgpack-only gates (P2) | -| `rustfs_system_network_internode_msgpack_json_decode_error_total{direction,message,codec}` | must be **0** before enabling both msgpack-only gates (P2) | -| `rustfs_cluster_servers_offline_total` | offline detection correctness (P3 bypass) | -| lock p99 (lock metrics) | P1 head-of-line-blocking win | +Underlying benches: `scripts/run_internode_transport_baseline.sh` (throughput/latency via `scripts/run_object_batch_bench.sh`; `--metrics-url ` also captures internode metric deltas) and `scripts/run_four_node_cluster_failover_bench.sh` (4-node compose cluster, kills `FAILOVER_NODE`). `target/` is gitignored; attach the paired directories to the PR or issue. ## Per-stage env matrix -Run **before** with the stage's env at its baseline column, **after** with the enabled -column, everything else at defaults. Roll a restart between runs. +Run **before** at the baseline column and **after** at the enabled column, everything else at defaults, with a restart between. Defaults are the `DEFAULT_INTERNODE_*` constants in `crates/config/src/constants/internode.rs`. -| Stage | Env | before (baseline) | after (enabled) | -|---|---|---|---| -| P0 nodelay | `RUSTFS_INTERNODE_RPC_TCP_NODELAY` | `false` | `true` (default) | -| P0 stream window | `RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE` | `0` | unset (1 MiB) | -| P0 conn window | `RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE` | `0` | unset (2 MiB) | -| P0 msg limit | `RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE` | `4194304` | unset (100 MiB) | +| Stage | Env | before | after | +| --- | --- | --- | --- | +| P0 nodelay | `RUSTFS_INTERNODE_RPC_TCP_NODELAY` | `false` | unset (`DEFAULT_INTERNODE_RPC_TCP_NODELAY`) | +| P0 stream window | `RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE` | `0` | unset (`DEFAULT_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE`) | +| P0 conn window | `RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE` | `0` | unset (`DEFAULT_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE`) | +| P0 msg limit | `RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE` | `4194304` (tonic default) | unset (RustFS default, see `rustfs/src/server/http.rs`) | | P1 isolation | `RUSTFS_INTERNODE_CHANNEL_ISOLATION` | `false` (default) | `true` | -| P1 bulk pool | `RUSTFS_INTERNODE_BULK_CHANNELS` | `1` | `2`–`4` | -| P2 msgpack-only request | `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY` | `false` (default) | `true` (only after fallback counter = 0 across a window) | -| P2 fleet confirmation | `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED` | `false` (default) | `true` (only after mixed-version, fallback-zero, soak, and rollback gates pass) | +| P1 bulk pool | `RUSTFS_INTERNODE_BULK_CHANNELS` | `1` | unset (`DEFAULT_INTERNODE_BULK_CHANNELS`) or higher | +| P2 msgpack-only | `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY` + `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED` | both `false` (default) | per the convergence runbook | | P3 prewarm | `RUSTFS_INTERNODE_PREWARM` | `false` (default) | `true` | | P3 offline bypass | `RUSTFS_INTERNODE_OFFLINE_BYPASS` | `false` (default) | `true` | -| P3 reprobe / threshold | `RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS` / `RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD` | defaults | `5` / `3` | +| P3 reprobe / threshold | `RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS` / `RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD` | defaults | defaults (`DEFAULT_INTERNODE_OFFLINE_REPROBE_SECS`, `DEFAULT_INTERNODE_OFFLINE_FAILURE_THRESHOLD`) | -## Procedure per stage +## Metrics to capture -1. **Baseline**: start the cluster with the stage's env at the *before* column. Run the - relevant bench; save to `.../baseline/` (or `.../after-P{n-1}/` when chaining stages). -2. **After**: restart with the *after* column; re-run the identical bench; save to - `.../after-P{n}/`. -3. Diff the object-bench summaries and the metric deltas. +All names are defined in `crates/io-metrics/src/internode_metrics.rs`. -- **P0** — `run_internode_transport_baseline.sh` with `--sizes 4KiB,1MiB,16MiB,128MiB` and - `--concurrencies 1,16,64`. Expect: small-RPC `duration_ms` (DiskInfo/Ping) down (nodelay), - large-metadata (ReadMultiple/BatchReadVersion) throughput up (windows). Functional: a - `>4 MiB` multi-version `xl.meta` no longer fails `out_of_range`. -- **P1** — mixed workload (large `ReadAll` + high-frequency `Refresh`). Acceptance gate from - the design doc: **lock p99 down ≥ 20%** with `RUSTFS_INTERNODE_CHANNEL_ISOLATION=true`. -- **P2** — observe `msgpack_json_decode_total`, `msgpack_json_fallback_total`, and `msgpack_json_decode_error_total` across a release window; every expected `codec="msgpack"` decode series must be **>0**, while fallback and decode-error series must stay **0** before flipping both `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true` and `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=true` (see the msgpack convergence runbook). Codec allocation via a `dhat`/`heaptrack` micro-run. -- **P3** — cold-start: first cross-node op latency should drop ~one connect RTT with prewarm. - Failover: `run_four_node_cluster_failover_bench.sh`, kill a node with - `RUSTFS_INTERNODE_OFFLINE_BYPASS=true`; expect faster failover and a correct - `rustfs_cluster_servers_offline_total` (1 while the node is down, back to 0 after recovery). +| Metric | Stage signal | +| --- | --- | +| `rustfs_system_network_internode_operation_duration_ms{operation,backend}` | control-plane RTT (P0), lock/bulk latency (P1), first-op latency (P3) | +| `rustfs_system_network_internode_operation_payload_bytes` | payload size distribution (P0/P1 sizing) | +| `rustfs_system_network_internode_operation_large_payloads_total` | large unary RPCs sharing a channel (P1 target) | +| `rustfs_system_network_internode_dial_avg_time_nanos`, `rustfs_system_network_internode_dial_errors_total` | connect cost and failures (P3) | +| `rustfs_system_network_internode_msgpack_json_decode_total{direction,message,codec}`, `..._msgpack_json_fallback_total`, `..._msgpack_json_decode_error_total` | P2 gate inputs | +| `rustfs_cluster_servers_offline_total` | offline detection correctness (P3 bypass) | +| lock p99 (lock metrics, needs `RUSTFS_OBJECT_LOCK_DIAG_ENABLE=true`) | P1 head-of-line-blocking win | -## Acceptance gates & artifact layout +## Acceptance gates -Each stage's paired run must satisfy an explicit gate before its numbers are accepted. Record -the gate verdict (pass/fail + measured delta) in the paired directory's `summary` and attach it. +Record the verdict (pass/fail plus measured delta) in the paired directory's summary. -| Stage | Bench | Acceptance gate | Primary metric(s) | -|---|---|---|---| -| **P0** | `run_internode_transport_baseline.sh` | small-RPC `duration_ms` (DiskInfo/Ping) **down**; large-metadata (ReadMultiple/BatchReadVersion) throughput **up**; a `>4 MiB` multi-version `xl.meta` no longer fails `out_of_range` (functional). | `..._operation_duration_ms{operation}`, `..._operation_payload_bytes`, object-bench throughput | -| **P1** | `run_internode_transport_baseline.sh` (mixed: large `ReadAll` + high-frequency `Refresh`) | **lock p99 down ≥ 20%** with `RUSTFS_INTERNODE_CHANNEL_ISOLATION=true` vs baseline. | lock p99 (lock metrics), `..._operation_large_payloads_total` | -| **P3 cold-start** | `run_internode_transport_baseline.sh` (fresh cluster, first cross-node op) | first cross-node op latency **drops ~one connect RTT** with `RUSTFS_INTERNODE_PREWARM=true`. | `..._dial_avg_time_nanos`, first-op `..._operation_duration_ms` | -| **P3 offline** | dedicated *sustained-offline + survivor cross-node access* experiment (the standard four-node failover bench is **not** sensitive to the bypass — quorum holds, `recovery_seconds=0`) | with `RUSTFS_INTERNODE_OFFLINE_BYPASS=true`, survivor cross-node op latency to the downed peer **fast-fails** instead of hanging the dial timeout; `rustfs_cluster_servers_offline_total` = **1** while down, back to **0** after recovery. | `rustfs_cluster_servers_offline_total`, survivor cross-node `..._operation_duration_ms`, `..._dial_errors_total` | +| Stage | Bench | Gate | Primary metrics | +| --- | --- | --- | --- | +| P0 | `run_internode_transport_baseline.sh --sizes 4KiB,1MiB,16MiB,128MiB --concurrencies 1,16,64` | small-RPC `duration_ms` (DiskInfo/Ping) down; large-metadata (ReadMultiple/BatchReadVersion) throughput up; a `>4 MiB` multi-version `xl.meta` no longer fails `out_of_range` | `..._operation_duration_ms{operation}`, `..._operation_payload_bytes`, object-bench throughput | +| P1 | `run_internode_transport_baseline.sh` with a mixed workload (large `ReadAll` plus high-frequency `Refresh`) | lock p99 down by at least 20% with `RUSTFS_INTERNODE_CHANNEL_ISOLATION=true` | lock p99, `..._operation_large_payloads_total` | +| P2 | none (not a throughput gate) | operational gate defined once in [internode-msgpack-json-convergence-runbook.md](internode-msgpack-json-convergence-runbook.md): expected `codec="msgpack"` decode series non-zero, fallback and decode-error series zero across a full observation window before both flags are enabled. Optionally a `dhat`/`heaptrack` micro-run for codec allocation. | the three `msgpack_json_*` counters | +| P3 cold-start | `run_internode_transport_baseline.sh` on a fresh cluster, first cross-node op | first cross-node op latency drops by about one connect RTT with prewarm | `..._dial_avg_time_nanos`, first-op `..._operation_duration_ms` | +| P3 offline | sustained-offline plus survivor cross-node access (below) | with bypass on, survivor ops to the downed peer fast-fail instead of hanging for the dial timeout; `rustfs_cluster_servers_offline_total` is `1` while down and `0` after recovery | `rustfs_cluster_servers_offline_total`, survivor `..._operation_duration_ms`, `..._dial_errors_total` | -> **P2 is not a throughput gate.** Its acceptance is operational: every expected `msgpack_json_decode_total{codec="msgpack"}` series must have non-zero traffic, while `msgpack_json_fallback_total` and `msgpack_json_decode_error_total` must read **0** across a full release window before both msgpack-only env gates are enabled (see the msgpack convergence runbook). Do not benchmark P2 as before/after throughput. +P3 offline method: the standard four-node failover bench is not sensitive to the bypass (quorum holds, `recovery_seconds=0`). Instead: all nodes up → stop one node and keep it down → warm up until offline detection trips → drive warp against the survivors only (`--host` excludes the dead node) → compare survivor op p99 and the offline gauge with `RUSTFS_INTERNODE_OFFLINE_BYPASS` off and on. -Artifact layout per stage (attach both halves + the diff): +Artifact layout: -``` +```text target/bench/internode-transport/ - p0-before/ p0-after/ # server-env.sh + object-bench summaries + metric deltas - p1-before/ p1-after/ # + lock p99 delta (the ≥20% gate) - p2-before/ p2-request-only/ p2-canary/ p2-after/ p2-rollback/ # env gate artifacts + fallback/decode-error observations - p3-before/ p3-after/ # cold-start + sustained-offline experiment + offline gauge trace + p0-before/ p0-after/ # server-env.sh + object-bench summaries + metric deltas + p1-before/ p1-after/ # + lock p99 delta + p2-before/ p2-request-only/ p2-canary/ p2-after/ p2-rollback/ # env previews + counter observations + p3-before/ p3-after/ # cold-start + sustained-offline + offline gauge trace ``` -## Bench-host prerequisites (ansible bare-metal) - -Captured while running the first real A/B on a 4-node ansible cluster; needed before any live run: - -- **RPC secret is mandatory on current `main`.** Internode RPC fails closed: default creds - (`RUSTFS_SECRET_KEY=rustfsadmin`) with no `RUSTFS_RPC_SECRET` → the node aborts startup immediately - with `store init aborted: endpoints include remote nodes but RPC authentication secret is not - configured` (builds before the preflight instead showed `No valid auth token` and never reached - `storage_quorum`). Set a **non-default** `RUSTFS_RPC_SECRET`, identical on every node. -- **systemd start timeout.** The install unit is `Type=notify`; READY only fires after quorum, which on - freshly-purged disks exceeds a 30 s `TimeoutStartSec` → crash loop. Use a drop-in `TimeoutStartSec=infinity`. -- **Server-side metrics need OTLP.** RustFS has no Prometheus pull endpoint (`/admin/v3/metrics` is NDJSON, - not exposition); it only pushes via OTLP. To capture lock/offline/internode metrics, run an - otel-collector (OTLP receiver → Prometheus exporter) and set `RUSTFS_OBS_ENDPOINT`, - `RUSTFS_OBS_METRICS_EXPORT_ENABLED=true`, `RUSTFS_OBS_METER_INTERVAL=5`. For lock p99 also set - `RUSTFS_OBJECT_LOCK_DIAG_ENABLE=true` (default off). -- **P3 offline method.** Standard failover is quorum-insensitive to the bypass. Use a *sustained-offline + - survivor cross-node access* run instead: all nodes up → stop one node (sustained) → warm up to trip - offline detection → drive warp on the survivors only (`--host` excludes the dead node) → compare - survivor op p99 and `rustfs_cluster_servers_offline_total` for `RUSTFS_INTERNODE_OFFLINE_BYPASS` off/on. - ## Rollback -Every stage rolls back by setting the env back to its baseline column and restarting. P2 rolls back by unsetting either msgpack-only gate, or by setting both `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=false` and `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=false`; no wire format is broken because the JSON fields, `_bin` fields, and proto field numbers remain additive. Do not remove or reuse JSON proto fields as part of this benchmark stage. +Every stage rolls back by restoring the baseline column and restarting. P2 rollback and its wire-format guarantees are in the [convergence runbook's rollback matrix](internode-msgpack-json-convergence-runbook.md#rollback-matrix); do not remove or reuse JSON proto fields as part of any benchmark stage. diff --git a/docs/operations/internode-msgpack-json-convergence-runbook.md b/docs/operations/internode-msgpack-json-convergence-runbook.md index 927475b23..4502d02bf 100644 --- a/docs/operations/internode-msgpack-json-convergence-runbook.md +++ b/docs/operations/internode-msgpack-json-convergence-runbook.md @@ -1,75 +1,37 @@ -# Internode msgpack/JSON Convergence Runbook +# Internode msgpack/JSON convergence runbook -Operational runbook for retiring the redundant JSON compatibility fields on internode -gRPC metadata RPCs (grpc-optimization **P2-1**). This is a **cross-version** change: it -proceeds strictly by observation-gated stages, never in one step. +**Use this when:** operating or changing the staged retirement of the JSON compatibility fields on internode gRPC metadata RPCs, deciding whether a fleet may enable `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY`, or adding a new `*_bin` proto field. +**Source of truth:** `crates/config/src/constants/internode.rs` (`ENV_INTERNODE_RPC_MSGPACK_ONLY`, `ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED` and their compile-time default-off asserts), `crates/protos/src/node.proto` (`*_bin` fields), `decode_msgpack_or_json` in `rustfs/src/storage/rpc/node_service/disk.rs` (server side) and `crates/ecstore/src/cluster/rpc/remote_disk.rs` (client side), `crates/io-metrics/src/internode_metrics.rs` (counters). + +This is a cross-version change and proceeds by observation-gated stages, never in one step. ## Background -Internode RPCs dual-encode each metadata value as **both**: +Internode RPCs dual-encode each metadata value as a msgpack binary field (`*_bin`, for example `file_info_bin`) and a JSON compatibility string (for example `file_info`). Decoders prefer `_bin` and fall back to JSON only when `_bin` is empty. The dual-write costs bandwidth and CPU; before the JSON fields can be dropped, the fallback branch must be proven unused across the fleet, otherwise a mixed-version rolling upgrade could read an emptied field. -- a msgpack binary field (`*_bin`, e.g. `file_info_bin`), and -- a JSON compatibility string (e.g. `file_info`). +## Observation counters -Decoders prefer the `_bin` payload and fall back to the JSON string only when `_bin` is -empty (`decode_msgpack_or_json`). The dual-write costs bandwidth and CPU. Before the JSON -fields can be dropped, the fallback branch must be proven **unused** in production — -otherwise a rolling upgrade with mixed node versions could read an emptied field. +| Counter | Labels | Increments when | +| --- | --- | --- | +| `rustfs_system_network_internode_msgpack_json_decode_total` | `direction`, `message`, `codec` | a msgpack or JSON decode succeeded | +| `rustfs_system_network_internode_msgpack_json_fallback_total` | `direction`, `message` | a decode fell back to JSON because `_bin` was empty | +| `rustfs_system_network_internode_msgpack_json_decode_error_total` | `direction`, `message`, `codec` | either codec failed to decode (`codec` = the failed codec) | -## The observation metric (already shipped) +`direction="request"` is a server decoding a peer's request (`node_service/disk.rs`); `direction="response"` is a client decoding a peer's response (`cluster/rpc/remote_disk.rs`), including the list-level `ReadMultiple` / `BatchReadVersion` fallbacks. `message` is the value name (`FileInfo`, `RawFileInfo`, `ReadMultipleResp`, ...). -``` -rustfs_system_network_internode_msgpack_json_fallback_total{direction, message} -rustfs_system_network_internode_msgpack_json_decode_total{direction, message, codec} -rustfs_system_network_internode_msgpack_json_decode_error_total{direction, message, codec} -``` - -The decode counter increments after a successful msgpack or JSON compatibility decode. The fallback counter increments whenever a decode falls back to the JSON field because the msgpack payload was absent. The decode-error counter increments whenever either codec fails to decode. - -- `direction="request"` — a server decoding a peer's request (`node_service/disk.rs`). -- `direction="response"` — a client decoding a peer's response (`cluster/rpc/remote_disk.rs`), - including the list-level `ReadMultiple` / `BatchReadVersion` fallbacks. -- `message` — the value name, e.g. `FileInfo`, `RawFileInfo`, `ReadMultipleResp`. -- `codec` — the failed codec for decode errors: `msgpack` for corrupt non-empty `_bin`, or `json` for corrupt legacy fallback JSON. - -## Stage 0 — Observe (current stage) - -Ship the current release (which contains these counters) and let it run for **at least one -full release window** across the whole fleet. The fallback and decode-error counters must stay at **zero**. - -First confirm each expected message/direction has real traffic in the observation window: +Gate query template, run over the whole observation window (adjust `[30d]`): ```promql -sum by (direction, message, codec) ( - increase(rustfs_system_network_internode_msgpack_json_decode_total[30d]) -) +sum by (direction, message, codec) (increase([30d])) ``` -For every convergence-ready message/direction below, the `codec="msgpack"` series must be non-zero before a zero fallback result is meaningful. Missing series, zero traffic, counter reset, or scrape gaps make the gate inconclusive rather than passed. +| Counter | Required reading | Meaning of a violation | +| --- | --- | --- | +| `..._decode_total` | every convergence-ready `{direction, message}` has a non-zero `codec="msgpack"` series | no traffic, a counter reset, or scrape gaps make the gate inconclusive, not passed | +| `..._fallback_total` | `0` for every series | some peer still sends an empty `_bin` (old node, or a sender that does not fill `_bin`); investigate the labels | +| `..._decode_error_total` | `0` for every series | `codec="msgpack"`: corrupt or incompatible `_bin` bytes; `codec="json"`: corrupt legacy fallback. Either blocks convergence and rollback confidence | -Confirm zero across the observation window (adjust `[30d]` to the window length): - -```promql -sum by (direction, message) ( - increase(rustfs_system_network_internode_msgpack_json_fallback_total[30d]) -) -``` - -Every series must be `0`. A non-zero value means some peer is still emitting an empty -`_bin` (an old node, or a message whose sender does not fill `_bin`) — investigate the -`{direction, message}` label before proceeding. - -Decode errors must also stay at zero across the observation window: - -```promql -sum by (direction, message, codec) ( - increase(rustfs_system_network_internode_msgpack_json_decode_error_total[30d]) -) -``` - -A non-zero `codec="msgpack"` series means a peer sent corrupt or incompatible `_bin` bytes; it must fail closed and block convergence. A non-zero `codec="json"` series means the legacy fallback field was corrupt or semantically incompatible; it also blocks convergence and rollback confidence. - -Standing alert (keep enabled through all stages): +Standing alerts (keep enabled through every stage): ```yaml - alert: InternodeMsgpackJsonFallback @@ -90,96 +52,70 @@ Standing alert (keep enabled through all stages): ## Field → peer-decoder audit -The send-side change (Stage 1) may only empty a JSON field whose **peer decodes `_bin` -first**. The following mapping is verified against the current code. +Stage 1 may only empty a JSON field whose peer decodes `_bin` first. Any `*_bin` field not listed here must be mapped to a confirmed `_bin`-first peer decoder before it joins the convergence set. -### Convergence-ready (peer decodes `_bin` first) +Convergence-ready: | Direction | Message / field | Peer decoder | -|---|---|---| -| request | `WriteMetadata.file_info` | `FileInfo` (node_service/disk.rs) | +| --- | --- | --- | +| request | `WriteMetadata.file_info` | `FileInfo` (`node_service/disk.rs`) | | request | `UpdateMetadata.file_info` | `FileInfo` | | request | `UpdateMetadata.opts` | `UpdateMetadataOpts` | | request | `RenameData.file_info` | `FileInfo` | | request | `ReadMultiple.read_multiple_req` | `ReadMultipleReq` | | request | `BatchReadVersion.batch_read_version_req` | `BatchReadVersionReq` | | request | `Read*.opts` | `ReadOptions` | -| response | `ReadVersion.file_info` | `FileInfo` (cluster/rpc/remote_disk.rs) | +| response | `ReadVersion.file_info` | `FileInfo` (`cluster/rpc/remote_disk.rs`) | | response | `ReadXL.raw_file_info` | `RawFileInfo` | | response | `RenameData.rename_data_resp` | `RenameDataResp` | -| response | `ReadMultiple` resp list | per-item + list fallback | -| response | `BatchReadVersion` resp list | per-item + list fallback | +| response | `ReadMultiple` response list | per-item plus list fallback | +| response | `BatchReadVersion` response list | per-item plus list fallback | -### `_bin` support added THIS release — converge after their own window - -The `DeleteVersion`/`DeleteVersions` protos had **no `_bin` fields**. They gained additive -`*_bin` fields plus bin-first server decoders in this release, and the client now dual-writes -them. They are **kept out** of the msgpack-only set (always dual-write) until their own -fallback counter has read zero across a window with the new decoders fully deployed. +Delete messages, kept on dual-write until their own window reads zero with `_bin`-first decoders deployed fleet-wide (the client always dual-writes these regardless of the flags): | Direction | Message / field | Status | -|---|---|---| -| request | `DeleteVersion.file_info` (`FileInfo`) | `_bin` added; dual-write; converge after window | -| request | `DeleteVersion.opts` (`DeleteOptions`) | `_bin` added; dual-write; converge after window | -| request | `DeleteVersions.versions` (`FileInfoVersions`) | `_bin` added; dual-write; converge after window | -| request | `DeleteVersions.opts` (`DeleteOptions`) | `_bin` added; dual-write; converge after window | +| --- | --- | --- | +| request | `DeleteVersion.file_info` (`FileInfo`) | `_bin` present; dual-write; converge after its own window | +| request | `DeleteVersion.opts` (`DeleteOptions`) | same | +| request | `DeleteVersions.versions` (`FileInfoVersions`) | same | +| request | `DeleteVersions.opts` (`DeleteOptions`) | same | -> Any `*_bin` proto field not in the tables above must be mapped to a confirmed `_bin`-first -> peer decoder before it is added to the convergence set. - -### Still JSON-only (no `_bin` field) +Still JSON-only: | Direction | Message / field | Note | -|---|---|---| -| response | `DeleteVersion.raw_file_info` | proto has no `_bin`; needs an additive proto field before it can converge. | +| --- | --- | --- | +| response | `DeleteVersion.raw_file_info` | proto has no `_bin`; needs an additive proto field before it can converge | -## Stage 1 — Stop writing JSON (env-gated, after Stage 0 reads zero) +## Stage 0 — Observe -The send-side lever is **implemented** and requires two default-off env flags: +Run the fleet with both flags at their default (`false`) for at least one full observation window and evaluate the three gate rows above. All three must hold before Stage 1. -- `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true` -- `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=true` +## Stage 1 — Stop writing JSON (env-gated) -The first flag only requests msgpack-only. The second flag is the explicit proof gate that the fleet has passed the release-window fallback, capability, and rollback checks. If only `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true` is set, RustFS keeps dual-writing JSON compatibility fields so old JSON-only peers remain compatible. When both flags are enabled, the convergence-ready fields above send only `_bin` and leave the JSON string empty; the `_bin` payload is always sent and decoders keep the JSON read fallback unchanged. The delete fields are excluded (dual-write) per the section above. +Flag semantics are defined on `ENV_INTERNODE_RPC_MSGPACK_ONLY` and `ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED` in `crates/config/src/constants/internode.rs`; both default off and only their conjunction stops the JSON write for convergence-ready fields. Decoders keep the JSON fallback in every state. -Only enable it **after** Stage 0 has read zero for a full window across the fleet: +1. Rehearse `request-only`: `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true`, `..._FLEET_CONFIRMED=false`. Behaviour is unchanged (still dual-write) and safe with old peers; any fallback or decode-error increment still blocks. +2. Canary: set both flags `true` on one node, restart that node only, and watch the fallback and decode-error counters for a soak period with real internode traffic. +3. Fleet: if the counters stay zero, enable both flags fleet-wide with a rolling restart. +4. Rollback: set either flag to `false` (or unset it) and restart; no wire format changed, so rollback is immediate. -1. Ship with the flag **off** (no behavior change). -2. Rehearse the `request-only` gate with `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true` and `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=false`. This must keep writing JSON compatibility fields and is safe with unsupported/old peers; any fallback or decode-error increment still blocks convergence. -3. Enable it on one canary node with both `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true` and `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=true`, then restart that node only and watch the fallback and decode-error counters for a soak period while real internode traffic is present. If they stay zero, enable fleet-wide. -4. **Rollback:** set either `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=false` or `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=false` (or unset either flag) and restart. No - wire-format was broken in this stage, so rollback is immediate and safe. +The benchmark driver pins these operator states as dry-run phases (`before`, `request-only`, `canary`, `after`, `rollback`); see [internode-grpc-benchmark-runbook.md](internode-grpc-benchmark-runbook.md). -The benchmark helper pins these operator states as dry-run phases: +## Stage 2 — Remove the proto JSON fields -```bash -scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase before --dry-run -scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase request-only --dry-run -scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase canary --dry-run -scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase after --dry-run -scripts/run_internode_grpc_ab_bench.sh --stage p2 --phase rollback --dry-run -``` +Only after Stage 1 has been stable fleet-wide for a full window with the counters still zero. -## Stage 2 — Remove the proto JSON fields (next release, N+1) - -Only after Stage 1 has been stable with the flag on for a full window and the counter is -still zero. - -1. Mark the retired text fields `reserved` in `crates/protos/src/node.proto` (never reuse - the field numbers) and delete the JSON read-fallback branches; codec becomes msgpack-only. -2. This is a hard wire-format change — it requires the mixed-version upgrade rehearsal - (four-node scripts) to pass, and it cannot be rolled back by env alone. +1. Mark the retired text fields `reserved` in `crates/protos/src/node.proto` (never reuse field numbers) and delete the JSON read-fallback branches. +2. This is a hard wire-format change: it requires the mixed-version upgrade rehearsal (four-node scripts) to pass and cannot be rolled back by env alone. ## Rollback matrix -| Stage | Wire-format broken? | Rollback | -|---|---|---| -| 0 Observe | no | n/a (metric only) | -| 1 msgpack-only send | no | unset either msgpack-only env flag + restart | -| 2 remove fields | yes | redeploy prior release; field numbers stay `reserved` | +| Stage | Wire format broken? | Rollback | +| --- | --- | --- | +| 0 Observe | no | n/a (metrics only) | +| 1 msgpack-only send | no | unset either flag and restart | +| 2 remove fields | yes | redeploy the prior release; field numbers stay `reserved` | ## Related -- Codec + counter implementation: commit `feat(internode): P2 msgpack/JSON codec observability + encode buffer presizing`. -- Decoders: `decode_msgpack_or_json` in `crates/ecstore/src/cluster/rpc/remote_disk.rs` (client) - and `rustfs/src/storage/rpc/node_service/disk.rs` (server). +- Transport and codec observability landed in `feat(internode): optimize gRPC transport (#4337)`. diff --git a/docs/operations/keycloak-oidc-integration.md b/docs/operations/keycloak-oidc-integration.md deleted file mode 100644 index 829ec3f57..000000000 --- a/docs/operations/keycloak-oidc-integration.md +++ /dev/null @@ -1,305 +0,0 @@ -# Keycloak OIDC Integration Runbook - -This runbook describes how to connect the RustFS Console to Keycloak by using OpenID Connect Authorization Code Flow. The examples use the default RustFS provider id, `default`. - -## 1. Integration Model - -RustFS supports standard OpenID Connect for Console login: - -- RustFS sends an authorization-code request with PKCE S256. -- Keycloak redirects back with `code` and `state`. -- RustFS exchanges the code at the token endpoint. -- RustFS requires an `id_token` and verifies signature, issuer, audience, expiry, and nonce. -- RustFS maps ID token claim values to local RustFS IAM policies. - -RustFS does not call Keycloak Authorization Services for object or admin authorization. Authorization is handled by RustFS policies after claims are mapped. - -## 2. Example Values - -Replace these values for your environment: - -| Value | Example | Notes | -| --- | --- | --- | -| Keycloak base URL | `https://keycloak.example.com` | Public Keycloak URL. | -| Realm | `rustfs` | Keycloak realm name. | -| Keycloak issuer | `https://keycloak.example.com/realms/rustfs` | RustFS `config_url`. | -| Discovery URL | `https://keycloak.example.com/realms/rustfs/.well-known/openid-configuration` | Used to validate metadata. | -| Public RustFS browser origin | `https://rustfs.example.com` | The scheme and authority users open in the browser. | -| Provider id | `default` | This runbook uses the default provider. | -| RustFS callback URL | `https://rustfs.example.com/rustfs/admin/v3/oidc/callback/default` | Register this exact URL in Keycloak. | -| Keycloak client id | `rustfs-console` | OIDC client used by RustFS. | -| Keycloak client secret | `` | Confidential client secret. | -| RustFS scopes | `openid,profile,email` | Add custom scopes if they emit authorization claims. | -| RustFS groups claim | `groups` | Recommended flat array claim. | -| RustFS roles claim | `roles` | Optional flat array claim. | - -## 3. Keycloak Configuration - -### 3.1 Create or Select the Realm - -1. Open the Keycloak Admin Console. -2. Create or select the `rustfs` realm. -3. Verify discovery: - -```bash -curl -fsS "https://keycloak.example.com/realms/rustfs/.well-known/openid-configuration" \ - | jq '.issuer,.authorization_endpoint,.token_endpoint,.jwks_uri' -``` - -The `issuer` should be: - -```text -https://keycloak.example.com/realms/rustfs -``` - -### 3.2 Create the RustFS Client - -In the Keycloak Admin Console: - -1. Open `Clients` and create a client. -2. Set `Client type` to `OpenID Connect`. -3. Set `Client ID` to `rustfs-console`. -4. Enable `Client authentication`. -5. Enable `Standard flow`. -6. Disable unused flows such as `Implicit flow`, `Direct access grants`, and `Service accounts roles`. -7. Set `Valid redirect URIs` to: - -```text -https://rustfs.example.com/rustfs/admin/v3/oidc/callback/default -``` - -8. Set `Web origins` to: - -```text -https://rustfs.example.com -``` - -9. Set `Proof Key for Code Exchange Code Challenge Method` to `S256`. -10. Save and copy the client secret from `Credentials`. - -RustFS submits the client secret in the token request body. Do not use a client policy that disables `client_secret_post`. - -### 3.3 Map Groups or Roles to RustFS Policies - -RustFS policy names are the final authorization source. Common built-in policies are: - -| Policy | Purpose | -| --- | --- | -| `consoleAdmin` | Full Console, admin, KMS, and S3 access. | -| `readwrite` | S3 read/write access. | -| `readonly` | S3 read-only access. | -| `writeonly` | S3 write-only access. | -| `diagnostics` | Diagnostic admin access. | - -Recommended production setup: - -1. Create Keycloak groups such as `consoleAdmin` and `readonly`. -2. Add users to the groups. -3. Add a group membership mapper for the `rustfs-console` client. -4. Emit a flat top-level ID token claim named `groups`. -5. Keep group values equal to RustFS policy names. - -### 3.4 Group Claim Mapper - -Create a `Group Membership` mapper in the dedicated client scope: - -| Mapper field | Value | -| --- | --- | -| Name | `rustfs-groups` | -| Token Claim Name | `groups` | -| Full group path | `Off` | -| Add to ID token | `On` | -| Add to access token | `On` | -| Add to userinfo | `On` | -| Multivalued | `On` | - -Keep `Full group path` disabled. RustFS policy names cannot contain `/`, so `/consoleAdmin` will not map to the `consoleAdmin` policy. - -### 3.5 Optional Role Claim Mapper - -If the deployment uses Keycloak roles: - -1. Assign realm or client roles such as `consoleAdmin`. -2. Add a `User Realm Role` or `User Client Role` mapper. -3. Emit a flat top-level claim named `roles`. -4. Set `RUSTFS_IDENTITY_OPENID_ROLES_CLAIM=roles`. - -RustFS does not parse Keycloak's default nested `realm_access.roles` claim. Emit a flat `roles` array when role mapping is required. - -## 4. RustFS Configuration - -### 4.1 Environment Variables - -Configure the provider and the public browser origin: - -```bash -export RUSTFS_BROWSER_REDIRECT_URL="https://rustfs.example.com" - -export RUSTFS_IDENTITY_OPENID_ENABLE=on -export RUSTFS_IDENTITY_OPENID_CONFIG_URL="https://keycloak.example.com/realms/rustfs" -export RUSTFS_IDENTITY_OPENID_CLIENT_ID="rustfs-console" -export RUSTFS_IDENTITY_OPENID_CLIENT_SECRET="" -export RUSTFS_IDENTITY_OPENID_SCOPES="openid,profile,email" -export RUSTFS_IDENTITY_OPENID_REDIRECT_URI="https://rustfs.example.com/rustfs/admin/v3/oidc/callback/default" -export RUSTFS_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC=off -export RUSTFS_IDENTITY_OPENID_DISPLAY_NAME="Keycloak" -export RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM="groups" -export RUSTFS_IDENTITY_OPENID_ROLES_CLAIM="roles" -export RUSTFS_IDENTITY_OPENID_EMAIL_CLAIM="email" -export RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM="preferred_username" -``` - -If RustFS reaches Keycloak through an internal URL while tokens use a public issuer, configure both values: - -```bash -export RUSTFS_IDENTITY_OPENID_CONFIG_URL="http://keycloak.keycloak.svc.cluster.local:8080/realms/rustfs/.well-known/openid-configuration" -export RUSTFS_IDENTITY_OPENID_ISSUER="https://keycloak.example.com/realms/rustfs" -export RUSTFS_OUTBOUND_ALLOW_ORIGINS="http://keycloak.keycloak.svc.cluster.local:8080" -``` - -Discovery and issuer-relative JWKS requests use the internal `CONFIG_URL` base. ID token issuer validation still uses `ISSUER`. -The outbound allowlist entry is the exact internal origin only; do not include the realm or discovery path. RustFS reads this process setting at startup, so restart every RustFS node after changing it. -Use HTTPS with a trusted CA for the internal URL whenever possible. Discovery and JWKS define the token-signing trust root; use HTTP only on a network where DNS and traffic cannot be tampered with, because a compromised response can authorize forged tokens. - -For short-lived connectivity testing only, you may temporarily add: - -```bash -export RUSTFS_IDENTITY_OPENID_ROLE_POLICY="consoleAdmin" -``` - -Do not use the temporary `role_policy` shortcut as a permanent production authorization model. - -Restart RustFS after changing OIDC settings. - -### 4.2 Admin Config - -If the deployment uses compatible admin configuration commands: - -```bash -mc admin config set rustfs identity_openid \ - enable=on \ - config_url="https://keycloak.example.com/realms/rustfs" \ - client_id="rustfs-console" \ - client_secret="" \ - scopes="openid,profile,email" \ - redirect_uri="https://rustfs.example.com/rustfs/admin/v3/oidc/callback/default" \ - redirect_uri_dynamic=off \ - display_name="Keycloak" \ - groups_claim="groups" \ - roles_claim="roles" \ - email_claim="email" \ - username_claim="preferred_username" - -mc admin service restart rustfs -``` - -`RUSTFS_BROWSER_REDIRECT_URL` is a process environment variable, not an `identity_openid` provider key. Configure it in the RustFS service environment even when the provider is stored through admin config. - -### 4.3 Named Provider - -To use a provider id such as `keycloak`, register this callback URL in Keycloak: - -```text -https://rustfs.example.com/rustfs/admin/v3/oidc/callback/keycloak -``` - -Then suffix the provider-specific environment variables: - -```bash -export RUSTFS_IDENTITY_OPENID_ENABLE_keycloak=on -export RUSTFS_IDENTITY_OPENID_CONFIG_URL_keycloak="https://keycloak.example.com/realms/rustfs" -export RUSTFS_IDENTITY_OPENID_CLIENT_ID_keycloak="rustfs-console" -export RUSTFS_IDENTITY_OPENID_CLIENT_SECRET_keycloak="" -export RUSTFS_IDENTITY_OPENID_SCOPES_keycloak="openid,profile,email" -export RUSTFS_IDENTITY_OPENID_REDIRECT_URI_keycloak="https://rustfs.example.com/rustfs/admin/v3/oidc/callback/keycloak" -export RUSTFS_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC_keycloak=off -export RUSTFS_IDENTITY_OPENID_DISPLAY_NAME_keycloak="Keycloak" -export RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM_keycloak="groups" -``` - -`RUSTFS_BROWSER_REDIRECT_URL` remains global and is not suffixed per provider. - -### 4.4 Redirect URL Priority - -RustFS builds browser-facing URLs with this priority: - -1. Provider `redirect_uri`, when configured, is used for the OIDC callback URL sent to Keycloak. -2. `RUSTFS_BROWSER_REDIRECT_URL`, when configured, is used as the public origin for OIDC callback generation when no provider `redirect_uri` exists, and for Console success redirects and logout fallback redirects. -3. Request headers are used only when provider dynamic redirects are enabled and no browser redirect URL is configured. - -For reverse-proxy or load-balancer deployments, set `RUSTFS_BROWSER_REDIRECT_URL` to avoid depending on `Host` and `X-Forwarded-Proto` for Console redirects. OIDC authorize and callback requests must still reach the same RustFS node because in-flight OIDC `state` is local to the node. - -## 5. Validation - -### 5.1 Validate Discovery - -```bash -curl -fsS "https://keycloak.example.com/realms/rustfs/.well-known/openid-configuration" | jq '{ - issuer, - authorization_endpoint, - token_endpoint, - jwks_uri, - code_challenge_methods_supported, - token_endpoint_auth_methods_supported -}' -``` - -Check that: - -- `issuer` equals `RUSTFS_IDENTITY_OPENID_ISSUER` when set; otherwise it matches the issuer derived from `RUSTFS_IDENTITY_OPENID_CONFIG_URL` -- `authorization_endpoint`, `token_endpoint`, and `jwks_uri` are present -- `code_challenge_methods_supported` includes `S256` -- the token endpoint accepts client secret authentication compatible with request-body submission - -### 5.2 Validate ID Token Claims - -After a test login, decode the ID token and confirm: - -- `iss` matches the Keycloak issuer -- `aud` includes the RustFS client id -- `email` and `preferred_username` are present when configured -- `groups` or `roles` contains RustFS policy names if fine-grained authorization is enabled - -### 5.3 Test Browser Login - -Open: - -```text -https://rustfs.example.com/rustfs/admin/v3/oidc/authorize/default -``` - -Expected flow: - -1. Browser redirects to Keycloak. -2. The user signs in. -3. Keycloak redirects to `/rustfs/admin/v3/oidc/callback/default?code=...&state=...`. -4. RustFS validates the ID token and issues STS credentials. -5. The browser lands on the RustFS Console and can use the expected permissions. - -## 6. Troubleshooting - -| Symptom | Common cause | Fix | -| --- | --- | --- | -| Keycloak reports `invalid redirect_uri` | Valid Redirect URIs does not match the RustFS callback URL | Use the exact callback URL and provider id. | -| Callback reports missing `code` or `state` | Proxy dropped the query string | Preserve the full callback URL and query string. | -| Token exchange fails | Client secret or client authentication policy mismatch | Confirm the client is confidential and accepts request-body secret auth. | -| RustFS reports no `id_token` | Missing `openid` scope or disabled Standard Flow | Include `openid` and enable Standard Flow. | -| ID token verification fails | Issuer, client id, audience, or JWKS mismatch | Compare discovery metadata and client settings. | -| Login succeeds but access is denied | No RustFS policy claim was mapped | Emit `groups` or `roles` as a flat ID token claim matching RustFS policy names. | -| Groups appear as `/consoleAdmin` | Keycloak `Full group path` is enabled | Disable `Full group path`. | -| Console redirects to an internal host | Missing `RUSTFS_BROWSER_REDIRECT_URL` or incorrect proxy headers | Set `RUSTFS_BROWSER_REDIRECT_URL` to the public browser origin. | -| Invalid or expired OIDC state | Callback reached a different RustFS node | Configure load-balancer session affinity for authorize and callback requests. | -| OIDC provider or login button is missing after upgrading to beta.12+ | The internal Keycloak origin is blocked by the outbound policy | Add the exact `scheme://host:port` origin to `RUSTFS_OUTBOUND_ALLOW_ORIGINS` and restart every RustFS node. | - -## 7. Production Checklist - -- [ ] Keycloak and RustFS use HTTPS. -- [ ] Keycloak Valid Redirect URIs uses exact callback URLs. -- [ ] `RUSTFS_BROWSER_REDIRECT_URL` is set to the public RustFS browser origin. -- [ ] `RUSTFS_IDENTITY_OPENID_REDIRECT_URI` matches the registered Keycloak callback URL. -- [ ] PKCE S256 is enabled or required. -- [ ] Users receive `groups` or `roles` claims that match RustFS policy names. -- [ ] `role_policy=consoleAdmin` is not used as a permanent production shortcut. -- [ ] The load balancer preserves query strings. -- [ ] OIDC authorize and callback requests have session affinity to the same RustFS node. -- [ ] Internal Keycloak origins are listed exactly in `RUSTFS_OUTBOUND_ALLOW_ORIGINS` on every RustFS node. diff --git a/docs/operations/kms-admin-contract.md b/docs/operations/kms-admin-contract.md index 25539ffde..fb334e175 100644 --- a/docs/operations/kms-admin-contract.md +++ b/docs/operations/kms-admin-contract.md @@ -1,68 +1,56 @@ -# KMS admin API contract and client handoff +# KMS admin API contract -This page is the server-side handoff for rustfs/backlog#1639. Response-shape snapshots for the key and metadata handlers live beside the producers under `rustfs/src/admin/handlers/snapshots/` (PR #5626). This matrix records the remaining route-level handoff contract without duplicating those snapshots. The management `KmsStatusResponse` shape remains an identified gap and will be pinned after the status-handler changes in #1636 land. +**Use this when:** you are wiring a client (CLI, console, automation) to the KMS admin endpoints and need the IAM action, risk class, per-key scope, and key-listing paging rules for each route. +**Source of truth:** `rustfs/src/admin/route_policy.rs` (action and risk per route, asserted by `rustfs/src/admin/route_registration_test.rs`); `crates/kms/src/backends/mod.rs` (`DEFAULT_LIST_KEYS_PAGE_SIZE`, `MAX_LIST_KEYS_PAGE_SIZE`, `list_keys_page_size`); response shapes pinned by snapshots under `crates/kms/src/snapshots/` and `rustfs/src/admin/handlers/snapshots/`. -The wire prefix is `/rustfs/admin/v3`. Request and response field names for the producer snapshots are pinned in #5626; fields in this matrix are the client handoff reference. `GET /kms/status` and `GET /kms/service-status` intentionally use different response types; `capabilities` on `/kms/status` is additive and optional. +The wire prefix is `/rustfs/admin/v3`. `GET /kms/status` and `GET /kms/service-status` return different response types; `capabilities` on `/kms/status` is additive and optional. The **Per-key** column says whether the route authorizes against the key it names (see [Per-key KMS authorization](kms-per-key-authorization.md)); `no` means the route matches any KMS resource in the caller's policy. -| Method and endpoint | Action / risk | Per-key | rc | console | Handoff | -| --- | --- | ---: | --- | --- | --- | -| `POST /kms/configure` | `kms:Configure` / high | no | supported | supported | none | -| `POST /kms/reconfigure` | `kms:Configure` / high | no | supported | supported | none | -| `POST /kms/start` | `kms:ServiceControl` / high | no | supported | supported | none | -| `POST /kms/stop` | `kms:ServiceControl` / high | no | supported | supported | none | -| `POST /kms/reload` | `kms:ServiceControl` / high | no | pending | pending | Re-reads the cluster-persisted configuration without resubmitting secrets; response reuses the configure shape. | -| `GET /kms/config` | `kms:Configure` / sensitive | no | no | supported | Redact operational paths before display. | -| `POST /kms/clear-cache` | `kms:ClearCache` / high | no | no | supported | Keep the current `{status,message}` response stable. | -| `POST /kms/keys` | `kms:Configure` / high | no | supported | supported | none | -| `GET /kms/keys` | `kms:ListKeys` / sensitive | no | supported | supported | none | -| `GET /kms/keys/{key_id}` | `kms:DescribeKey` / sensitive | yes | supported | supported | none | -| `DELETE /kms/keys/delete` | `kms:DeleteKey` / critical | yes | supported | supported | Preserve immediate-delete confirmations. | -| `POST /kms/keys/cancel-deletion` | `kms:DeleteKey` / high | yes | supported | supported | none | -| `POST /kms/create-key` | `kms:Configure` / high | no | no | no | Legacy `mc` alias; do not add a second client command. | -| `POST /kms/key/create` | `kms:Configure` / high | no | no | no | Legacy `mc` alias; do not add a second client command. | -| `GET /kms/describe-key` | `kms:DescribeKey` / sensitive | yes | no | no | Legacy `mc` alias. | -| `GET /kms/key/status` | `kms:DescribeKey` / sensitive | yes | supported | no | `rc key status` uses this legacy-compatible shape. | -| `GET /kms/list-keys` | `kms:ListKeys` / sensitive | no | supported | no | `rc key list` uses this legacy-compatible shape. | -| `POST /kms/generate-data-key` | `kms:GenerateDataKey` / high | yes | do not expose | do not expose | Programmatic primitive; never print plaintext key material. | -| `GET /kms/status` | `kms:ServiceControl` / sensitive | no | supported | supported | Keep `capabilities` optional for old servers. | -| `POST /kms/status` | `kms:ServiceControl` / high | no | no | no | Internal compatibility route; not a client command. | -| `GET /kms/service-status` | `kms:ServiceControl` / sensitive | no | no | supported | Do not conflate this type with `/kms/status`. | -| `POST /kms/keys/enable` | `kms:EnableKey` / high | yes | pending | pending | Add an explicit lifecycle command/UI action. | -| `POST /kms/keys/disable` | `kms:DisableKey` / high | yes | pending | pending | Add an explicit lifecycle command/UI action. | -| `POST /kms/keys/rotate` | `kms:RotateKey` / high | yes | pending | pending | Add an explicit lifecycle command/UI action. | -| `POST /kms/keys/update-description` | `kms:UpdateKeyDescription` / high | yes | pending | pending | Add a metadata mutation command/UI action. | -| `POST /kms/keys/tag` | `kms:TagResource` / high | yes | pending | pending | Add a metadata mutation command/UI action. | -| `POST /kms/keys/untag` | `kms:UntagResource` / high | yes | pending | pending | Add a metadata mutation command/UI action. | -| `GET /kms/backup` | `kms:Backup` / sensitive | no | pending | pending | Status/readiness only; never expose KEK material. | -| `POST /kms/backup` | `kms:Backup` / high | no | pending | pending | Preserve `backup_id` and metadata-only response. | -| `POST /kms/restore/dry-run` | `kms:Restore` / sensitive | no | pending | pending | Dry-run must be the default and show differences. | -| `POST /kms/restore` | `kms:Restore` / high | no | pending | pending | Require `confirm_backup_id` and `confirm_conflict_policy`; no blanket `--yes`. | -| `POST /kms/restore/abort` | `kms:Restore` / high | no | pending | pending | Require `confirm_target_key_dir`. | +## Endpoint matrix + +| Method and endpoint | IAM action | Risk | Per-key | Notes | +| --- | --- | --- | --- | --- | +| `POST /kms/configure` | `kms:Configure` | high | no | Persists to cluster storage, switches the local node, broadcasts a best-effort peer reload | +| `POST /kms/reconfigure` | `kms:Configure` | high | no | Same contract as configure | +| `POST /kms/start` | `kms:ServiceControl` | high | no | | +| `POST /kms/stop` | `kms:ServiceControl` | high | no | | +| `POST /kms/reload` | `kms:ServiceControl` | high | no | Re-reads the persisted configuration without resubmitting secrets; reuses the configure response shape | +| `GET /kms/status` | `kms:ServiceControl` | sensitive | no | Backend type plus capability matrix | +| `POST /kms/status` | `kms:ServiceControl` | high | no | Compatibility route; not a client command | +| `GET /kms/service-status` | `kms:ServiceControl` | sensitive | no | Carries `cluster_config` fingerprints and the `consistent` flag | +| `GET /kms/config` | `kms:Configure` | sensitive | no | Contains operational paths; redact before display | +| `POST /kms/clear-cache` | `kms:ClearCache` | high | no | `KmsClearCacheResponse` (`{status,message}`) | +| `POST /kms/keys` | `kms:Configure` | high | no | Key creation shares the configure action | +| `GET /kms/keys` | `kms:ListKeys` | sensitive | no | See the key listing contract below | +| `GET /kms/keys/{key_id}` | `kms:DescribeKey` | sensitive | yes | `?impact=true` opts into the configuration-reference report | +| `DELETE /kms/keys/delete` | `kms:DeleteKey` | critical | yes | JSON body; `force_immediate` also requires `confirm_key_id` and the server-side `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION` gate | +| `POST /kms/keys/cancel-deletion` | `kms:DeleteKey` | high | yes | | +| `POST /kms/keys/enable` | `kms:EnableKey` | high | yes | | +| `POST /kms/keys/disable` | `kms:DisableKey` | high | yes | | +| `POST /kms/keys/rotate` | `kms:RotateKey` | high | yes | Subject to the rotation constraints in [KMS backend security properties](kms-backend-security.md#master-key-rotation-retention-destruction-and-upgrade-ordering) | +| `POST /kms/keys/rekey` | `kms:Rekey` | high | no | Bulk DEK rekey sweep; cluster-scoped, see [`kms-bulk-rekey-contract.md`](../architecture/kms-bulk-rekey-contract.md) | +| `GET /kms/keys/rekey/status` | `kms:Rekey` | sensitive | no | | +| `POST /kms/keys/rekey/cancel` | `kms:Rekey` | high | no | | +| `POST /kms/keys/update-description` | `kms:UpdateKeyDescription` | high | yes | | +| `POST /kms/keys/tag` | `kms:TagResource` | high | yes | | +| `POST /kms/keys/untag` | `kms:UntagResource` | high | yes | | +| `POST /kms/generate-data-key` | `kms:GenerateDataKey` | high | yes | Response carries a base64 plaintext data key; never surface it in a UI or CLI | +| `GET /kms/backup` | `kms:Backup` | sensitive | no | Status and readiness only; no KEK material | +| `POST /kms/backup` | `kms:Backup` | high | no | Returns `backup_id` and metadata only | +| `POST /kms/restore/dry-run` | `kms:Restore` | sensitive | no | Preflight; writes nothing | +| `POST /kms/restore` | `kms:Restore` | high | no | Requires `confirm_backup_id` and `confirm_conflict_policy` | +| `POST /kms/restore/abort` | `kms:Restore` | high | no | Requires `confirm_target_key_dir` | +| `POST /kms/create-key`, `POST /kms/key/create` | `kms:Configure` | high | no | Legacy `mc` aliases of `POST /kms/keys` | +| `GET /kms/describe-key`, `GET /kms/key/status` | `kms:DescribeKey` | sensitive | yes | Legacy aliases of `GET /kms/keys/{key_id}` | +| `GET /kms/list-keys` | `kms:ListKeys` | sensitive | no | Legacy alias of `GET /kms/keys`; same listing contract | ## Key listing contract Both listing routes (`GET /kms/keys` and the legacy `GET /kms/list-keys`) share one contract. -`limit` is optional. When it is absent the server applies its own default page size of 100. When it is present it must parse as a non-negative integer: `limit=abc`, `limit=-1` and a value-less `limit` are refused with `400`, not silently read as "use the default". `limit=0` is a well-formed request for an empty page. Any page size above 1000 is served as 1000 — the response is `truncated` with a usable `next_marker`, so a client that pages until `truncated` is false still reaches every key. Clients must not assume a page is the size they asked for. +`limit` is optional. When it is absent the server applies `DEFAULT_LIST_KEYS_PAGE_SIZE` (100). When it is present it must parse as a non-negative integer: `limit=abc`, `limit=-1` and a value-less `limit` are refused with `400`, not silently read as "use the default". `limit=0` is a well-formed request for an empty page. Any page size above `MAX_LIST_KEYS_PAGE_SIZE` (1000) is served as 1000 — the response is `truncated` with a usable `next_marker`, so a client that pages until `truncated` is false still reaches every key. Clients must not assume a page is the size they asked for. `marker` is opaque to the client: treat it as a cursor to hand back unchanged, never as a value to construct. On the Local, Vault KV2, Vault Transit and Static backends it happens to be an exclusive lower bound on the key identifier, which is what makes paging survive keys being created or destroyed mid-listing; on the AWS backend it is AWS's own pagination token, and sending a key id there is rejected. An empty `marker` means the same thing as no marker at all. Filters are applied after the page is cut, so a filtered page can be short — even empty — while more keys remain. Page until `truncated` is false, never until a page comes back short. `unreadable_key_ids` is present only when the server listed a key whose record it could not describe — a record written by a newer build, or damaged material. The identifiers are reported rather than omitted, so a listing never quietly understates the key set; a client displaying an inventory should surface them as damaged rather than dropping them, and paging always advances past a damaged key. A failure that says nothing about a specific key (timeout, `5xx`, permission denied) still fails the whole listing instead of appearing here. One case is deliberately an error rather than a report: a listing that covered the entire key set — no `marker`, and not `truncated` — in which nothing was readable. An empty `keys` array there would be indistinguishable, to any client written before this field existed, from a deployment that has no keys, and the usual response to that is to provision a new one. Such a listing returns `500` instead, naming the first failure; the individual identifiers are in the server log. A truncated page, or one resumed from a marker, always reports rather than failing, so a damaged key can never strand the keys behind it. - -## Server-side snapshot coverage - -The merged #5626 producer snapshots cover the nine modern/legacy key response types and the metadata response type served by `kms_keys.rs` and `kms_key_metadata.rs`: create, describe, list, generate-data-key, delete, cancel-deletion, update-description, tag, and untag. The four dynamic responses served verbatim by `kms_dynamic.rs` are covered in `crates/kms/src/snapshots/`: configure, start, stop, and the `service-status` response. `POST /kms/reload` serves the same `ConfigureKmsResponse` type the configure snapshot pins; it adds no new wire shape. - -`POST /kms/clear-cache` now has a named `KmsClearCacheResponse` and a producer snapshot beside the others; its serialized bytes are unchanged from the inline JSON it replaced. - -The remaining wire-shape gaps are intentionally documented rather than duplicated here: the management `KmsStatusResponse` (`GET|POST /kms/status`, pending #1636), `KmsConfigResponse`, all three lifecycle responses, and the backup/restore response family. Adding producer snapshots for those gaps is a separate server test task; it must not be inferred from the client matrix. - -## Client handoff gaps - -The `rc` client currently has status, key list/status/create/delete/cancel-deletion, configure/reconfigure/start/restart/stop, and diagnostic/roundtrip entry points. It has no lifecycle enable/disable/rotate, key metadata, backup/restore, or reload commands; `POST /kms/reload` is the recovery path when a restarted server reports not-configured while a persisted configuration exists, so it is a client delivery item alongside the lifecycle gaps. The console currently calls service-status, configure/reconfigure/start/stop/config, clear-cache, status, and the modern key CRUD routes. It has no lifecycle, metadata, or backup/restore UI. These pending cells are delivery items for `rustfs/cli` and `rustfs/console`; they are not implemented in this repository. A read-only issue search on 2026-08-02 found no matching KMS issue in either client repository, so the client handoff still needs issue creation there. - -`POST /kms/generate-data-key` is deliberately marked “do not expose” for both clients: its response contains a base64 plaintext data key. `GET /kms/config` and backup status/restore responses contain operational paths and identifiers, not key material, but still require UI/CLI redaction and confirmation handling. - -The producer response snapshots in #5626 and this matrix do not imply that rustfs/backlog#1639 is complete. The pending client cells must be closed in their respective repositories before the parent delivery item can be marked complete. diff --git a/docs/operations/kms-backend-security.md b/docs/operations/kms-backend-security.md index 9d3c2307b..d0bf7ee7b 100644 --- a/docs/operations/kms-backend-security.md +++ b/docs/operations/kms-backend-security.md @@ -1,56 +1,38 @@ # KMS backend security properties -RustFS ships several KMS backends. They differ not only in deployment effort but in **where master key material lives and who can read it**. Pick a backend based on the confidentiality boundary you need, not on the name alone. +**Use this when:** choosing a KMS backend, scheduling or debugging master key rotation, planning a rolling upgrade of a cluster with KMS enabled, or auditing where master key material lives and who can read it. +**Source of truth:** `crates/kms/src/config.rs` (`KmsBackend`, `ENV_KMS_*` constants), `crates/kms/src/backends/{local,vault,vault_transit,aws}.rs`, `crates/kms/src/encryption/dek.rs` (`DataKeyEnvelope`), `rustfs/src/admin/route_policy.rs` (KMS route actions). -For how the Vault backends authenticate (static token, AppRole, Kubernetes, Vault Agent token file) and how credential refresh and the fail-closed window behave, see the [Vault KMS authentication runbook](vault-kms-authentication.md). For what may be claimed about the cryptographic implementations themselves, see [Cryptographic compliance positioning](kms-cryptographic-compliance.md). For which RustFS identities may manage or use a given key, see [Per-key KMS authorization](kms-per-key-authorization.md). If you are migrating from MinIO, read [Migrating from MinIO: encrypted objects do not carry over](#migrating-from-minio-encrypted-objects-do-not-carry-over) first. +RustFS ships several KMS backends. They differ not only in deployment effort but in **where master key material lives and who can read it**. Pick a backend based on the confidentiality boundary you need, not on the name alone. Related: [Vault KMS authentication runbook](vault-kms-authentication.md) (credential sources, refresh, fail-closed window), [Cryptographic compliance positioning](kms-cryptographic-compliance.md), [Per-key KMS authorization](kms-per-key-authorization.md), [KMS admin API contract](kms-admin-contract.md), [KMS observability runbook](kms-observability-runbook.md). ## Backend comparison | Backend | Config tag | Master key material location | At-rest protection of key material | Durability | Rotation | Intended use | | --- | --- | --- | --- | --- | --- | --- | -| Local | `Local` | Files under `key_dir`, encrypted with the configured local master key | Local master key (AES-GCM) + file permissions | Crash-durable commits on local filesystems only; see [Local backend durability and deployment support matrix](#local-backend-durability-and-deployment-support-matrix) | Rejected by design (single material, development backend) | Development, testing and demos only; not supported for production | +| Local | `Local` | Files under `key_dir`, encrypted with the configured local master key | Local master key (AES-GCM) + file permissions | Crash-durable commits on local filesystems only; see [Local backend durability and deployment support matrix](#local-backend-durability-and-deployment-support-matrix) | Rejected by design (single material) | Development, testing and demos only; not supported for production | | Static | `Static` | Provided out-of-band via environment/file; never persisted by RustFS | Operator-managed secret distribution | No state persisted by RustFS | Rejected (read-only backend) | Development and testing with an externally supplied key; not supported for production | | Vault KV2 | `VaultKV2` (legacy alias `Vault`) | Stored **directly** in Vault KV v2 (Base64-encoded plaintext) | Vault ACLs + KV v2 at-rest encryption + TLS only | Delegated to Vault storage | Versioned retention (immutable per-version records + current pointer) | Deployments that accept Vault KV ACLs as the sole confidentiality boundary | | Vault Transit | `VaultTransit` | Key-encryption keys never leave Vault; only Transit ciphertext is visible outside | Vault Transit engine (cryptographic isolation) | Delegated to Vault storage | Via Vault Transit key versioning | Deployments that need key material to be unreadable through storage APIs | -| AWS KMS | `AWS` (alias `AwsKms`) | Key material never leaves AWS KMS; RustFS mirrors no key state | AWS KMS (cryptographic isolation) + IAM | Delegated to AWS | On-demand `RotateKeyOnDemand`; prior backing keys stay usable for decryption | Deployments already rooted in AWS IAM that want AWS as the cryptographic root — read [AWS KMS: deviations from the shared backend contract](#aws-kms-deviations-from-the-shared-backend-contract) first | +| AWS KMS | `AWS` (alias `AwsKms`) | Key material never leaves AWS KMS; RustFS mirrors no key state | AWS KMS (cryptographic isolation) + IAM | Delegated to AWS | On-demand `RotateKeyOnDemand`; prior backing keys stay usable for decryption | Deployments rooted in AWS IAM — read [AWS KMS: deviations from the shared backend contract](#aws-kms-deviations-from-the-shared-backend-contract) first | ## Migrating from MinIO: encrypted objects do not carry over -> **Warning: RustFS does not currently support reading objects that MinIO encrypted.** -> This applies to SSE-S3, SSE-KMS, and SSE-C, in every released binary and container image, and it holds regardless of which KMS backend you configure. Configuring the `Static` backend with the same key material MinIO used does **not** make those objects readable — MinIO wraps data keys in a different envelope format that no RustFS backend produces or accepts (`crates/kms/src/config.rs:304-308`). Plan for this **before** moving data. Tracked in rustfs/backlog#1638. +> **Warning: default RustFS builds fail closed on objects that MinIO encrypted.** This applies to SSE-S3, SSE-KMS, and SSE-C, whichever KMS backend you configure; configuring `Static` with MinIO's key material does not make them readable. Such objects list and HEAD normally (their `xl.meta` parses), and only the payload read fails — with S3 `InvalidObjectState`, never plaintext. Read a sample of encrypted objects, not just their listings, before decommissioning the MinIO deployment. -The read does fail closed — ciphertext is never served as plaintext. MinIO's internal encryption headers mark the object as encrypted (`crates/utils/src/http/header_compat.rs:50-67`), so the read path demands encryption material and refuses when none resolves (`crates/ecstore/src/object_api/readers.rs:559-568`). Two properties still make the problem easy to discover late: - -- **The error does not say what happened.** It surfaces as a 500 `InternalError`, which reads as a RustFS fault rather than "another implementation encrypted this object". -- **Surrounding metadata migrates fine.** The object's `xl.meta` parses, so encrypted objects list and HEAD normally and report plausible sizes. The failure appears only when something reads the payload. - -Read a sample of encrypted objects, not just their listings, before decommissioning the MinIO deployment. - -Current options for a migration whose source contains encrypted objects: - -- Decrypt on the MinIO side first, migrate plaintext, then let RustFS re-encrypt with its own KMS. -- Copy through the S3 API rather than moving drives — MinIO decrypts on read, and RustFS encrypts on write. This re-encrypts rather than preserving ciphertext and costs a full data transfer. -- Leave encrypted objects on MinIO and migrate only unencrypted data. - -Inventory the source before choosing: bucket default-encryption settings mean objects can be encrypted without any client having sent SSE headers. - -The same limitation applies in reverse — objects RustFS encrypts are not readable by MinIO. For the code-level breakdown of which seams block each SSE mode, see [MinIO file-format interoperability, Part C](../architecture/minio-file-format-compat.md#part-c--server-side-encryption-sse). - -The migration warning is not a wire-protocol promise: the **AWS KMS wire protocol** and **MinIO KES wire protocol** are explicit non-targets for this document. The AWS backend uses the AWS SDK client path (`crates/kms/src/backends/aws.rs:830`), and KES remains outside the MinIO on-disk interop scope. Track those ecosystem evaluations and the MinIO/RustFS SSE compatibility matrix in the [#1562 Production Ready exit gate](https://github.com/rustfs/backlog/issues/1562); #1638 alone does not satisfy that gate. +The read path exists behind the `rio-v2` feature as a migration-only build, the reverse direction (RustFS-written SSE objects read by MinIO) is unsupported, and the migration options are enumerated in [MinIO file-format interoperability, Part C](../architecture/minio-file-format-compat.md#part-c--server-side-encryption-sse). The AWS KMS and MinIO KES wire protocols are non-targets of that document and of this one. ## Vault KV2: what the backend does and does not do The Vault KV2 backend uses Vault purely as a **secure storage** service: - Master key material is generated by RustFS and written to KV v2 as a Base64-encoded value (`encrypted_key_material` is an encoding, not a ciphertext). -- The backend never calls the Vault Transit engine. The `mount_path` configuration field and the `RUSTFS_KMS_VAULT_MOUNT_PATH` environment variable are deprecated leftovers: they are accepted for compatibility and ignored. +- The backend never calls the Vault Transit engine. The `mount_path` configuration field and the `RUSTFS_KMS_VAULT_MOUNT_PATH` environment variable are deprecated leftovers: accepted for compatibility and ignored. - Data-encryption keys (DEKs) handed to the object-encryption path are still wrapped with AES-256-GCM under the master key; the statement above concerns the master key's storage in Vault, not the DEK envelope. -- No runtime interface reports this boundary. `GET /rustfs/admin/v3/kms/status` names the active backend (`backend_type: vault-kv2`) and returns a `capabilities` matrix, but that matrix enumerates only the operations the backend supports — nothing in it describes where master key material lives or who can read it. Determining which confidentiality boundary is in force means reading `backend_type` and applying the comparison table above; this document is the only statement of the boundary an operator can consult. -- The `at_rest_protection: storage-only` field carried by a KMS backup manifest is a different thing: it declares the protection state of key material inside a backup bundle, not a property the running backend reports about itself. -- Key rotation retains every historical master key version as an immutable record under `{prefix}/{key_id}/versions/{N}` and only then moves the current-version pointer; see [Master key rotation](#master-key-rotation-retention-destruction-and-upgrade-ordering) for the retention preconditions and the cluster-upgrade ordering constraint. +- No runtime interface reports this boundary. `GET /rustfs/admin/v3/kms/status` names the active backend (`backend_type: vault-kv2`) and returns a `capabilities` matrix, but that matrix enumerates only supported operations. Determining the confidentiality boundary in force means reading `backend_type` and applying the comparison table above. +- The `at_rest_protection: storage-only` field carried by a KMS backup manifest declares the protection state of key material inside a backup bundle, not a property the running backend reports about itself. +- Key rotation retains every historical master key version as an immutable record under `{prefix}/{key_id}/versions/{N}` and only then moves the current-version pointer; see [Master key rotation](#master-key-rotation-retention-destruction-and-upgrade-ordering). -> **Warning: KV read access is equivalent to holding the master keys.** -> Any Vault identity (token, AppRole, or policy) that can `read` the RustFS key path in KV v2 can recover the plaintext master key material and decrypt every object protected by those keys. Treat KV read grants on that path with the same care as handing out the keys themselves. If this is not acceptable, use the Vault Transit backend instead. +> **Warning: KV read access is equivalent to holding the master keys.** Any Vault identity (token, AppRole, or policy) that can `read` the RustFS key path in KV v2 can recover the plaintext master key material and decrypt every object protected by those keys. If this is not acceptable, use the Vault Transit backend instead. ## Minimal Vault policy for the KV2 backend @@ -67,63 +49,49 @@ path "secret/metadata/rustfs/kms/keys/*" { } ``` -Notes: - -- The trailing wildcards also cover the per-version material records that rotation creates under `.../keys/{key_id}/versions/{N}`; no extra policy paths are needed. -- `delete` on the metadata path is required for permanent key deletion (`force_immediate`); drop it if you never hard-delete keys. RustFS refuses `force_immediate` unless the server sets `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true`, so leaving that gate off keeps the capability unreachable no matter what the Vault policy allows. -- Do not attach `sudo`, wildcard mounts, or Transit paths to this policy; the KV2 backend does not use them. -- Auditing KV reads on the key prefix is strongly recommended: every read event is a potential master-key disclosure. +- The trailing wildcards also cover the per-version material records under `.../keys/{key_id}/versions/{N}`; no extra policy paths are needed. +- `delete` on the metadata path is required only for permanent key deletion (`force_immediate`); drop it if you never hard-delete keys. RustFS refuses `force_immediate` unless the server sets `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true`, so leaving that gate off keeps the capability unreachable whatever the Vault policy allows. +- Do not attach `sudo`, wildcard mounts, or Transit paths; the KV2 backend does not use them. +- Audit KV reads on the key prefix: every read event is a potential master-key disclosure. ## Master key rotation: retention, destruction, and upgrade ordering -Rotation support differs per backend. Local and Static advertise no `rotate` capability — `capabilities.rotate` is false in the `kms/status` response — and reject rotation with `UnsupportedCapability`; their single key material is never overwritten. Vault Transit delegates rotation to the Transit engine's own key versioning (ciphertext is version-prefixed, e.g. `vault:v1:...`). Vault KV2 rotates by retaining every historical version, as described below. Rotation is reachable through the admin API as `POST /rustfs/admin/v3/kms/keys/rotate`, which the route policy classifies as high risk and gates behind `kms:RotateKey`; it is not exposed through the S3 surface. The upgrade ordering constraint below therefore applies to an operator action, not only to a call from inside the process. +Rotation is reachable through `POST /rustfs/admin/v3/kms/keys/rotate` (`kms:RotateKey`, high risk); it is not exposed on the S3 surface. Local and Static advertise no `rotate` capability (`capabilities.rotate` is false in the `kms/status` response) and reject rotation with `UnsupportedCapability`. Vault Transit delegates rotation to the Transit engine's own versioning (ciphertext is version-prefixed, e.g. `vault:v1:...`). Vault KV2 rotates by retaining every historical version, as described below. ### Rotation drivers and scheduling, per backend -The rotate endpoint is one API over three very different mechanisms, and which component actually performs the rotation decides how periodic rotation must be scheduled — on two backends it cannot be scheduled at all. +The rotate endpoint is one API over three different mechanisms, and which component performs the rotation decides how periodic rotation must be scheduled. -| Backend | Can rotate | Who performs the rotation | How to schedule periodic rotation | -| --- | --- | --- | --- | -| Local | No | Nobody — the backend advertises no `rotate` capability and the rotate endpoint is refused with `UnsupportedCapability` | Cannot be scheduled. Migrating to a rotating backend is the only path to rotation | -| Static | No | Nobody — same refusal as Local; the material is supplied out-of-band and read-only | Cannot be scheduled. Migrate to a rotating backend | -| Vault KV2 | Yes | **RustFS** owns the whole rotation protocol: freeze the outgoing material as an immutable version record, persist the new version's material, then move the current pointer with a check-and-set write | An **external scheduler** (cron, Kubernetes CronJob, your automation platform) calling `POST /rustfs/admin/v3/kms/keys/rotate`. RustFS deliberately ships no built-in rotation timer — see below | -| Vault Transit | Yes | **Vault's Transit engine** — RustFS only forwards the call to Transit's rotate endpoint and records the version bump in its own metadata | Vault's native `auto_rotate_period` on the Transit key. Do **not** additionally point an external scheduler at the RustFS rotate endpoint — see below | -| AWS KMS | Yes | **AWS** — the RustFS rotate endpoint maps to `RotateKeyOnDemand` | AWS's native automatic rotation, configured on the AWS side. Do **not** drive periodic rotation through the RustFS endpoint — see below | +| Backend | Can rotate | Who performs the rotation | How to schedule periodic rotation | Wrap ceiling | +| --- | --- | --- | --- | --- | +| Local | No | Nobody — the rotate endpoint is refused with `UnsupportedCapability` | Cannot be scheduled; migrating to a rotating backend is the only path | Unmitigable | +| Static | No | Nobody — same refusal; the material is supplied out-of-band and read-only | Cannot be scheduled; migrate | Unmitigable | +| Vault KV2 | Yes | **RustFS** owns the protocol: freeze the outgoing material as an immutable version record, persist the new material, move the current pointer with a check-and-set write | Exactly **one external scheduler** (cron, Kubernetes CronJob) calling the rotate endpoint with credentials scoped to `kms:RotateKey` | Reset by each rotation | +| Vault Transit | Yes | **Vault's Transit engine**; RustFS forwards the call and records the version bump | Vault's native `auto_rotate_period` on the Transit key. Do **not** also drive the RustFS endpoint: two owners of the version cadence means neither configured period holds. The reported key version advances only through RustFS, so on an auto-rotating key treat it as a floor | Not applicable (wraps inside Vault) | +| AWS KMS | Yes | **AWS** — the endpoint maps to `RotateKeyOnDemand` | AWS's native automatic rotation. Do **not** drive periodic rotation through the RustFS endpoint: AWS caps lifetime on-demand rotations, so a scheduler exhausts the quota and then fails forever. Keep the endpoint for incidents. RustFS neither enables nor observes AWS automatic rotation and records no rotation timestamp, so its rotation-age signals measure key age on this backend | Not applicable (wraps inside AWS) | -**Local and Static: the wrap ceiling is unmitigable.** These backends wrap every DEK with AES-256-GCM under their single master key using a random 96-bit nonce, and NIST SP 800-38D caps AES-GCM at 2^32 invocations per key when nonces are chosen at random. Each encrypted object write wraps a DEK, so the invocation count tracks the number of encrypted-object writes over the deployment's lifetime. On a rotating backend that count restarts whenever new master key material takes over; on Local and Static it can never restart, because there is no rotation to restart it. The only mitigation is migrating to a backend that rotates. The same 2^32 bound applies to the KV2 backend's wrapping — RustFS wraps DEKs locally there too — but there each rotation mints fresh master key material and resets the count, which is one more reason to actually schedule KV2 rotation rather than merely support it. +**Wrap ceiling.** Where RustFS wraps DEKs locally (Local, Static, Vault KV2) every DEK is wrapped with AES-256-GCM under the master key using a random 96-bit nonce, and NIST SP 800-38D caps AES-GCM at 2^32 invocations per key under random nonces. Each encrypted-object write is one wrap, so the count tracks lifetime encrypted writes. Rotation installs fresh material and restarts the count; on Local and Static there is no rotation, so the ceiling can only be escaped by migrating. -**Vault KV2: bring your own scheduler, deliberately.** RustFS performs the rotation but does not decide when: there is no built-in rotation worker, by design rather than omission. A timer inside the server cannot verify the [cluster-upgrade precondition](#upgrade-before-first-rotation-hard-constraint) before firing, and rotation is not idempotent — without leader election, N nodes running the same schedule would perform N rotations per period, advancing the key version N times. Run exactly one external scheduler, point it at the admin rotate endpoint with credentials scoped to `kms:RotateKey`, and use the [rotation readiness fields](#rotation-readiness-reported-never-acted-on) plus the `KmsKeyRotationOverdue` alert in the [KMS observability runbook](kms-observability-runbook.md#kmskeyrotationoverdue) to verify the schedule is actually keeping up. - -**Vault Transit: exactly one owner of the version cadence.** Configure `auto_rotate_period` on the Transit key and let Vault own the schedule. Layering an external scheduler that calls the RustFS rotate endpoint on top of `auto_rotate_period` creates two competing owners of the key's version cadence, and the effective rotation period stops being the one either owner was configured with. The data path is indifferent to who rotates — Transit ciphertext self-describes the version that wrapped it, so envelopes never pin a version RustFS tracked — but the key version RustFS reports only advances when rotation goes through RustFS, so on an auto-rotating key treat the reported version as a floor, not the truth. - -**AWS KMS: native automatic rotation for cadence, `RotateKeyOnDemand` for incidents.** The RustFS rotate endpoint maps to AWS `RotateKeyOnDemand`, and AWS enforces a lifetime limit on the number of on-demand rotations a key may receive (see the AWS KMS documentation) — a periodic scheduler driving the RustFS endpoint will exhaust that quota and then fail forever. Configure AWS's automatic rotation for periodic cadence and keep the RustFS endpoint for what on-demand rotation is for: incident response and one-off rotations. Note that RustFS neither enables nor observes AWS automatic rotation, and it records no rotation timestamp for AWS keys, so the readiness fields and the rotation-age gauge measure key age on this backend — verify the actual cadence in AWS, not through RustFS. +**Why there is no built-in rotation timer (KV2).** A timer inside the server cannot verify the [upgrade-before-first-rotation constraint](#upgrade-before-first-rotation-hard-constraint), and rotation is not idempotent: without leader election, N nodes on the same schedule would advance the key version N times per period. Verify that your scheduler keeps up with the [rotation readiness fields](#rotation-readiness-reported-never-acted-on) and the `KmsKeyRotationOverdue` alert in the [KMS observability runbook](kms-observability-runbook.md#kmskeyrotationoverdue). **Pre-rotation checklist** (before the first rotation of any key, and before enabling any schedule): -1. Every node in the cluster runs a build that understands the `master_key_version` envelope field — the [hard upgrade-ordering constraint](#upgrade-before-first-rotation-hard-constraint) below. A timer cannot check this; you must. +1. Every node runs a build that understands the `master_key_version` envelope field — the [hard constraint](#upgrade-before-first-rotation-hard-constraint) below. A timer cannot check this; you must. 2. No rolling upgrade is in progress — see [Do not do these during a mixed-version window](#do-not-do-these-during-a-mixed-version-window). -3. The [retention and destruction preconditions](#retention-and-destruction-preconditions) are understood: every version record a stored DEK envelope references must remain readable forever, and no retention tooling prunes the version subtree. -4. For KV2, exactly one scheduler exists, so no two callers race the same rotation period. +3. The [retention and destruction preconditions](#retention-and-destruction-preconditions) are understood: every version record a stored DEK envelope references must remain readable, and no retention tooling prunes the version subtree. +4. For KV2, exactly one scheduler exists. 5. `RUSTFS_KMS_ROTATION_MAX_AGE_SECS` is set to the rotation period your policy requires, so the per-key `rotation_due` verdict and the rotation-age alert verify the schedule instead of assuming it. ### Rotation readiness: reported, never acted on -RustFS does not rotate keys on a schedule. There is no built-in rotation worker, deliberately: rotation is a policy decision with a per-backend cost and a hard upgrade-ordering constraint (see below), and a server that rotated on its own would make that decision on an operator's behalf at a moment it did not choose. What the server does instead is tell you which keys have outlived a period you configure. +RustFS reports which keys have outlived a period you configure; nothing consults the verdict before encrypting or decrypting, and it has no effect on readiness or liveness. -Set `RUSTFS_KMS_ROTATION_MAX_AGE_SECS` to that period in whole seconds. Unset — the default — leaves the verdict unreported rather than assuming a policy: how often keys must be rotated is a compliance decision, and a built-in default would report keys as overdue against a rule nobody wrote. An unparsable value is treated the same way, with a warning, instead of silently falling back to a number the operator did not choose. Values below one hour are raised to one hour, because a threshold of seconds reports every key as overdue moments after it was rotated and teaches operators to ignore the signal. +| Setting | Meaning | Unset or unparsable | Floor | +| --- | --- | --- | --- | +| `RUSTFS_KMS_ROTATION_MAX_AGE_SECS` | Rotation period in whole seconds; keys older than this report `rotation_due` with reason `age` or `never_rotated` | No age verdict is reported (a warning is logged for an unparsable value) — how often keys must rotate is a compliance decision, not a built-in default | 1 hour | +| `RUSTFS_KMS_ROTATION_MAX_WRAPS` | Data keys one key's material may wrap before `rotation_due` with reason `wraps` | No wrap verdict is reported | 1,000,000 (wraps are accounted in reserved blocks of that size) | -A second, independent threshold covers the cryptographic bound rather than the policy one. `RUSTFS_KMS_ROTATION_MAX_WRAPS` is the number of data keys one key's material may wrap before the verdict reports `rotation_due` with reason `wraps`. It follows the same discipline — unset or unparsable leaves the verdict unreported, and values below one million are raised to one million because wraps are accounted in reserved blocks of that size, so a smaller threshold would trip on the first reservation. Only backends where RustFS wraps locally and can rotate report a count (Vault KV2 today); Transit and AWS wrap externally and report none, so the wrap half stays silent there rather than guessing. When both thresholds are crossed the reported reason is `wraps`: the AES-GCM random-nonce ceiling is not negotiable, while the age period is a policy an operator chose. - -`GET /rustfs/admin/v3/kms/keys` then carries two additional fields per key: - -- `rotation_due` — whether the key has outlived the configured period. -- `rotation_due_reason` — `age` when the key was rotated but longer ago than the period, `never_rotated` when it has never been rotated and has been in use longer than the period, and `unsupported` when the backend cannot rotate at all. Absent when there is no verdict. - -The verdict is advisory in the strongest sense: nothing consults it before encrypting or decrypting, a key reported as due keeps serving traffic unchanged, and it has no effect on readiness or liveness. It is computed in one place, from the backend's declared rotation capability plus the key's own timestamps, so no two backends can disagree about what "overdue" means — and a backend that cannot rotate is reported as `unsupported` rather than being told to do something it cannot. - -`GET /rustfs/admin/v3/kms/keys/{key_id}` does **not** carry these fields. Its response type records a creation date but no rotation timestamp, so a verdict computed there could not tell a key rotated last week from one never rotated at all, and reporting `never_rotated` for a key that was in fact rotated would be worse than reporting nothing. Read the verdict from the listing. - -Driving the rotation itself remains external: call `POST /rustfs/admin/v3/kms/keys/rotate` from your own scheduler, having first satisfied the upgrade-ordering constraint below — and only on the backend where that is the right scheduling model; see [Rotation drivers and scheduling, per backend](#rotation-drivers-and-scheduling-per-backend). +`GET /rustfs/admin/v3/kms/keys` carries `rotation_due` and `rotation_due_reason` per key: `age` (rotated, but longer ago than the period), `never_rotated` (in use longer than the period, never rotated), `wraps` (wrap budget exceeded; wins over `age` when both hold, because the AES-GCM ceiling is not negotiable), or `unsupported` (the backend cannot rotate). Only backends where RustFS wraps locally and can rotate count wraps (Vault KV2); Transit and AWS report no count. `GET /rustfs/admin/v3/kms/keys/{key_id}` does **not** carry these fields — its response records a creation date but no rotation timestamp, so read the verdict from the listing. ### Vault KV2 versioned retention model @@ -133,59 +101,53 @@ Decryption loads exactly the version recorded in the envelope and fails closed w ### Retention and destruction preconditions -- Every version record that any stored DEK envelope references must remain readable. The bulk rekey sweep (`POST /rustfs/admin/v3/kms/keys/rekey`, gated on `kms:Rekey`) rewraps stored envelopes onto the current version; until a sweep has completed with zero failures after the last rotation, assume **every** version of a rotated key is referenced: destroying a version record permanently orphans all objects whose DEKs it wrapped. A completed sweep is evidence, not authority — the deletion gate stays the decision point. Replication strips encryption metadata in transit, so a sweep never propagates to a replica site: each site runs its own. -- Version records are ordinary KV v2 secrets under the key subtree. Never run `kv metadata delete` or `kv destroy` against `{prefix}/{key_id}/versions/*`, and do not apply `delete-version-after` or retention tooling to that subtree. RustFS-managed retention does not rely on KV2's own secret versioning (each version record has a single KV revision), so KV `max-versions` settings do not protect or endanger history — but metadata deletion always removes a record entirely. -- Permanent key deletion through RustFS (`force_immediate` after `PendingDeletion`) purges the key's version records together with the key record; that is the only supported way to remove them. It is refused by default: the server must set `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true`, and the request must be a `DELETE` with a JSON body that sets `force_immediate` and echoes the key id back as `confirm_key_id` — the query-parameter form (`?force_immediate=true`) is refused outright, whatever the gate is set to. Leave the gate off unless you are actively destroying keys, and turn it off again afterwards — the pending-deletion window plus `CancelKeyDeletion` is the only recovery path for objects encrypted under the key. +- Every version record that any stored DEK envelope references must remain readable. The bulk rekey sweep (`POST /rustfs/admin/v3/kms/keys/rekey`, `kms:Rekey`) rewraps stored envelopes onto the current version; until a sweep has completed with zero failures after the last rotation, assume **every** version of a rotated key is referenced. A completed sweep is evidence, not authority — the deletion gate stays the decision point. Replication strips encryption metadata in transit, so each replica site runs its own sweep. +- Version records are ordinary KV v2 secrets under the key subtree. Never run `kv metadata delete` or `kv destroy` against `{prefix}/{key_id}/versions/*`, and do not apply `delete-version-after` or retention tooling to that subtree. Each version record has a single KV revision, so KV `max-versions` settings neither protect nor endanger history — but metadata deletion removes a record entirely. +- Permanent key deletion through RustFS (`force_immediate` after `PendingDeletion`) purges the key's version records together with the key record; that is the only supported way to remove them. It requires `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true` on the server and a `DELETE` with a JSON body that sets `force_immediate` and echoes the key id as `confirm_key_id`; the query-parameter form is refused outright. Leave the gate off except while actively destroying keys — the pending-deletion window plus `CancelKeyDeletion` is the only recovery path for objects encrypted under the key. +- `force_immediate` is refused with `409 Conflict` while any bucket's default encryption configuration names the key or the key is the KMS service default key. A scheduled deletion is not refused for that reason: it destroys nothing and stays cancellable, and the background sweep re-checks the same references before destroying material. - For Vault Transit, retention is governed by the Transit key's `min_decryption_version`: never raise it above the oldest version that may still protect live ciphertext. -- `force_immediate` is additionally refused, with a `409 Conflict`, while any bucket's default encryption configuration still names the key, or while the key is the KMS service default key. A scheduled deletion is not refused for that reason: it destroys nothing and stays cancellable, and the background sweep re-checks the same references before it destroys the material. ### Reading the `impact` section -`DeleteKey` responses always carry an `impact` section listing the configuration that currently points at the key — the buckets whose default encryption names it, and whether it is the service default key — so the references that will refuse the destruction are visible when the deletion is scheduled rather than only in a server-side log once the window has run out. +`DeleteKey` responses always carry an `impact` section listing the configuration that points at the key (buckets whose default encryption names it, and whether it is the service default key). `DescribeKey` (`GET /rustfs/admin/v3/kms/keys/{key_id}`) returns the same section only when asked with `impact=true`, because collecting it lists every bucket; a value other than `true`/`false` is rejected with `400`. **An absent section means "not collected", never "nothing references this key".** -`DescribeKey` (`GET /rustfs/admin/v3/kms/keys/{key_id}`) can return the same section, but only when the request asks for it with `impact=true`. It is opt-in there because collecting it lists every bucket and `DescribeKey` is polled; without the parameter the endpoint does exactly the work it did before and returns no `impact` field at all. A value other than `true` or `false` is rejected with `400` rather than treated as `false`, so a typo can never answer a request for the section with a response that merely lacks one. **An absent section means "not collected", never "nothing references this key".** - -Read it for what it says and nothing more. `coverage.scanned` names the sources that were read; `coverage.not_scanned` names the ones that were not, which currently includes every object encrypted under the key. `completeness` is `exact` only over the scanned sources, and `unavailable` when a source could not be read at all — an unavailable report is not an empty one, and both an unreadable source and an outstanding reference will stop the sweep from destroying the material. - -**An empty `references` list does not mean the key is unused.** No object metadata is consulted, so a key with no configuration references can still protect an arbitrary amount of live data, which stays readable only until the material is gone. There is no field in the response that asserts otherwise, and none should be inferred from one. +`coverage.scanned` names the sources that were read and `coverage.not_scanned` the ones that were not — which currently includes every object encrypted under the key. `completeness` is `exact` only over the scanned sources and `unavailable` when a source could not be read; both an unreadable source and an outstanding reference stop the sweep from destroying material. **An empty `references` list does not mean the key is unused:** no object metadata is consulted, so a key with no configuration references can still protect live data. ### Upgrade before first rotation (hard constraint) -Do not rotate any key until **every** RustFS node in the cluster runs a build that understands the `master_key_version` envelope field. Older binaries ignore the field and always decrypt with the current material: harmless while nothing has been rotated, but after a rotation they will fail to decrypt every object wrapped by an earlier key version. Complete the rolling upgrade of the entire cluster first, then rotate. - -This is the sharpest instance of a broader class of constraints; the rest are collected in [Mixed-version clusters during a rolling upgrade](#mixed-version-clusters-during-a-rolling-upgrade). +Do not rotate any key until **every** RustFS node runs a build that understands the `master_key_version` envelope field. Older binaries ignore the field and always decrypt with the current material: harmless while nothing has been rotated, but after a rotation they fail to decrypt every object wrapped by an earlier key version. Complete the rolling upgrade of the entire cluster first, then rotate. The rest of this constraint class is collected in [Mixed-version clusters during a rolling upgrade](#mixed-version-clusters-during-a-rolling-upgrade). ## Mixed-version clusters during a rolling upgrade -During a rolling upgrade the cluster runs two RustFS builds at once. That window matters more for KMS than for most subsystems, because KMS state is shared three ways: **Vault** holds the key records and Transit metadata, **cluster storage** holds the persisted KMS configuration, and **each node's process memory** holds caches and the live backend instance. Nodes on different builds agree on the first, may disagree on the third, and — for configuration — can disagree for as long as the operator leaves them running, because the reload broadcast that converges configuration is one of the things an older build rejects. - -This section states only what is true of the current implementation. It is written for the KV2 and Transit backends; the Local backend is unsupported for multi-node deployments regardless of version (see the [deployment support matrix](#deployment-support-matrix)). +During a rolling upgrade KMS state is shared three ways: **Vault** holds key records and Transit metadata, **cluster storage** holds the persisted KMS configuration, and **each node's process memory** holds caches and the live backend instance. Nodes on different builds agree on the first, may disagree on the third, and can disagree on configuration for as long as the operator leaves them running, because the reload broadcast that converges configuration is one of the things an older build rejects. This section is written for the KV2 and Transit backends; the Local backend is unsupported for multi-node deployments regardless of version (see the [deployment support matrix](#deployment-support-matrix)). ### Persisted formats are backward compatible in both directions -Nothing in this list requires a coordinated format cutover. The compatibility is deliberate and is covered by decode tests. +No coordinated format cutover is required; the compatibility is deliberate and covered by decode tests. -- **DEK envelopes.** `DataKeyEnvelope::master_key_version` is optional and omitted when absent, so envelopes written by non-rotating backends stay byte-identical to the historical seven-field JSON shape. An upgraded node reading a pre-versioning envelope resolves `None` to the key's recorded baseline version, or — for a key that was never rotated, and so has no baseline — to the current version, which is exactly the pre-versioning behavior. Unknown values are skipped while parsing; a bounded field-name sample is emitted at a progressively rate-limited `warn` level, and `rustfs_kms_persisted_unknown_fields_total{record_kind="data-key-envelope"}` counts every observed field. -- **Local key records.** Each `.key` record carries `format_version: 1`; records written before that field existed default to version 1 when read, and the pre-version reader ignores the added v1 marker. A reader accepts a record whose version is at most the version it understands, and rejects a newer version with `UnsupportedFormatVersion` before it attempts to decrypt key material. Unknown fields remain accepted for rollback compatibility. Their values are ignored while parsing; a bounded field-name sample is emitted at a progressively rate-limited `warn` level, and `rustfs_kms_persisted_unknown_fields_total{record_kind="local-key-record"}` counts every observed field. Once a future version greater than 1 has written a key record, do not roll back to a build that predates this marker: such a build cannot reject that future version before interpreting the rest of the record. -- **KV2 key records.** `baseline_version` is read with a serde default, so records written by older builds deserialize unchanged, and `None` correctly means "never rotated". -- **Transit metadata records.** Metadata persisted in KV v2 by either build decodes on the other. - -The one-way hazard is the rotation constraint above: an older binary reading a *new* envelope silently ignores the version field and decrypts with the current material. +| Record | Compatibility mechanism | Caveat | +| --- | --- | --- | +| DEK envelopes | `DataKeyEnvelope::master_key_version` is optional and omitted when absent, so envelopes from non-rotating backends stay byte-identical to the historical seven-field JSON. An upgraded node resolves a pre-versioning envelope to the key's `baseline_version`, or to the current version for a never-rotated key. Unknown fields are skipped; a bounded field-name sample is logged at a rate-limited `warn` and counted by `rustfs_kms_persisted_unknown_fields_total{record_kind="data-key-envelope"}` | An older binary reading a *new* envelope ignores the version field and decrypts with the current material — the rotation constraint above | +| Local key records | Each `.key` carries `format_version: 1`; records without it default to 1. A reader accepts a version at most the one it understands and rejects newer with `UnsupportedFormatVersion` before decrypting. Unknown fields are accepted and counted by `rustfs_kms_persisted_unknown_fields_total{record_kind="local-key-record"}` | Once a future version greater than 1 has written a record, do not roll back to a build predating the marker | +| KV2 key records | `baseline_version` is read with a serde default; `None` means "never rotated" | An old build drops the field on write-back (see below) | +| Transit metadata records | Decode on either build | — | ### DEK envelope context binding (`RUSTFS_KMS_ENVELOPE_AAD`) -Historically the KV2 and Local backends sealed only the DEK plaintext; the `encryption_context` rode in the envelope unauthenticated and was checked by field comparison alone, so a party able to rewrite the stored envelope could rewrite the context to match whatever it presented. With `RUSTFS_KMS_ENVELOPE_AAD=true`, newly wrapped envelopes bind the canonical context bytes as AES-GCM additional data and carry `context_binding: 1`; rewriting the stored context, or stripping the flag, then fails authentication. (Static, Vault Transit and AWS already bound the context through their own mechanisms and are unaffected.) +Historically the KV2 and Local backends sealed only the DEK plaintext; the `encryption_context` rode in the envelope unauthenticated and was checked by field comparison alone, so a party able to rewrite the stored envelope could rewrite the context. With `RUSTFS_KMS_ENVELOPE_AAD=true`, newly wrapped envelopes bind the canonical context bytes as AES-GCM additional data and carry `context_binding: 1`; rewriting the stored context, or stripping the flag, then fails authentication. Static, Vault Transit and AWS already bound the context through their own mechanisms and are unaffected. -Rollout constraint: **reading bound envelopes needs no switch, but a node that predates the field cannot open them** — its unwrap runs without the additional data and fails authentication. Enable the switch only after every node in the cluster runs a release that understands `context_binding`; the default stays off for one release for exactly this reason, mirroring the `RUSTFS_ENCRYPTION_FRAME_V2` rollout. Rewrap migrates existing envelopes: with the switch on, a rewrap sweep upgrades unbound envelopes to the bound format (converging to zero writes on re-run), and a bound envelope never regresses to the unbound shape whatever the switch says. An envelope carrying an unrecognized `context_binding` value is refused rather than decrypted without its binding. +Rollout constraint: reading bound envelopes needs no switch, but **a node that predates the field cannot open them** — its unwrap runs without the additional data and fails authentication. The switch defaults off (`ENV_KMS_ENVELOPE_AAD` in `crates/kms/src/config.rs`); enable it only after every node runs a release that understands `context_binding`, mirroring the `RUSTFS_ENCRYPTION_FRAME_V2` rollout. With the switch on, a rewrap sweep upgrades unbound envelopes to the bound format (converging to zero writes on re-run); a bound envelope never regresses to the unbound shape, and an envelope carrying an unrecognized `context_binding` value is refused rather than decrypted without its binding. ### Guarantees that hold only once every node is upgraded -These are properties of the upgraded code, so a single node left behind removes them for the whole cluster. +These are properties of builds from `1.0.0-rc.1` onward; a single older node removes them for the whole cluster. -- **Check-and-set lifecycle writes.** Upgraded builds write every KV2 lifecycle mutation — create, enable, disable, tag metadata, schedule deletion, cancel deletion — as a versioned read followed by a check-and-set write, retrying on conflict by re-reading and re-validating the state gate (rustfs/rustfs#5518). Transit metadata writes got the same treatment (rustfs/rustfs#5520). Builds older than those write blind. A blind write from an old node can overwrite a check-and-set commit from an upgraded node without any conflict being reported, which is precisely the lost update the change was made to eliminate. -- **`baseline_version` survives a write-back.** The KV2 key record does not deny unknown fields, so an old build reads a new record without error — and drops `baseline_version` when it writes that record back for any reason. A key that loses its baseline resolves pre-versioning envelopes to the current version again, which after a rotation means the wrong master key material. Any lifecycle operation issued to an old node is enough to trigger this. -- **`wrap_budget_reserved` keeps overestimating.** The KV2 key record's approximate wrap counter (`wrap_budget_reserved`, behind the `rustfs_kms_max_key_wrap_operations` gauge) is dropped the same way when an old build rewrites the record, regressing the count toward zero — the one way this deliberately overestimate-only counter can understate the wraps actually performed. Nothing breaks: the counter is advisory, and the next block reservation from an upgraded node re-establishes a floor. Just do not trust a *low* gauge reading taken during or shortly after a mixed-version window. -- **Version-record awareness.** Rotation stores each historical version under `{prefix}/{key_id}/versions/{N}` as a create-only record (check-and-set of 0), so two nodes racing the same version number produce exactly one creator; the loser adopts the persisted, never-current material or fails without touching the current pointer. Old builds have no concept of that sub-path: they never read or write it, and their key listing reports the KV2 directory entry (`my-key/`) as though it were a key, because the directory filter only exists in upgraded builds. +| Guarantee | Upgraded behaviour | What an old node does | +| --- | --- | --- | +| Check-and-set lifecycle writes | Every KV2 lifecycle mutation (create, enable, disable, tag, schedule/cancel deletion) and every Transit metadata write is a versioned read followed by a check-and-set write, retried on conflict | Writes blind, so it can overwrite a check-and-set commit without any conflict being reported — the lost update the change eliminated | +| `baseline_version` survives write-back | Preserved | Reads the record without error and drops `baseline_version` on any write-back; the key then resolves pre-versioning envelopes to the current version, which after a rotation is the wrong material | +| `wrap_budget_reserved` only overestimates | The KV2 record's wrap counter (behind `rustfs_kms_max_key_wrap_operations`) is reserved in blocks and never understates | Drops the field on write-back, regressing the count toward zero. Nothing breaks — the counter is advisory and the next reservation re-establishes a floor — but do not trust a *low* reading taken during or shortly after a mixed-version window | +| Version-record awareness | Version records under `{prefix}/{key_id}/versions/{N}` are create-only (check-and-set of 0), so two nodes racing a version number produce exactly one creator | Never reads or writes the sub-path, and its key listing reports the KV2 directory entry (`my-key/`) as though it were a key | ### Windows in which nodes can legitimately disagree @@ -193,79 +155,65 @@ Even with every node on the same build, some state is process-local. These windo | What can diverge | Bound | Mechanism | | --- | --- | --- | -| Transit key lifecycle state used by the `encrypt` and `generate_data_key` gates | ≤ 300 s (`METADATA_CACHE_TTL`) | Each node caches Transit metadata in process, TTL- and capacity-bounded, with targeted invalidation when a data-path call reports the key is gone server-side. A disable or schedule-deletion performed on one node is enforced on the others within one TTL at the latest, sooner if they hit that signal. | -| `describe_key` output | One metadata cache TTL: 300 s by default, otherwise whatever `cache_ttl_seconds` was configured with, clamped to 24 h | The manager-level key metadata cache, built from the configured cache settings. This is a reporting cache; the KV2 state gates do not read it. | -| KV2 key lifecycle state | None | The KV2 backend re-reads the key record from Vault for every lifecycle and data-key operation, so a committed disable is effective on every upgraded node immediately. | -| Active KMS configuration | One best-effort reload broadcast; unbounded for any peer that did not apply it | See below. | - -Builds older than rustfs/rustfs#5520 held the Transit metadata cache with no TTL and no capacity bound. On such a node the divergence window is not 300 seconds but "until the process restarts": it can keep encrypting under a key that another node disabled, indefinitely. - -The `describe_key` bound is the only one on that list an operator sets, so compute it rather than assuming the default: the window is the `cache_ttl_seconds` the KMS configure request was given, 300 s when it was omitted, clamped down to 24 h at use if it is larger (clamped rather than rejected, so an oversized setting still starts). Zero is refused outright while caching is enabled. `kms service-status` and the KMS configuration endpoint report the effective, post-clamp value, so the number the admin API shows is the number the cache honours. Note that this is the Transit row's neighbour and not its equal: `METADATA_CACHE_TTL` above is a separate, deliberately non-tunable 300 s, because that cache does gate cryptographic operations. - -One upgrade caveat: builds older than rustfs/rustfs#5569 ignored `cache_ttl_seconds` and ran a hardcoded 300 s, while their configure converters persisted 3600 s as the default value. A cluster configured through the admin API before that fix therefore widens its `describe_key` staleness window from an effective 300 s to the 3600 s already stored in `config/kms_config.json`, with no configuration change of its own. Read the reported value back after upgrading instead of assuming it stayed at 300 s. No cryptographic or authorization path widens with it — encrypt, decrypt and data-key generation go straight to the backend and never read this cache. +| Transit key lifecycle state used by the `encrypt` and `generate_data_key` gates | ≤ `METADATA_CACHE_TTL` (300 s, not tunable — this cache gates cryptographic operations) | Per-node in-process Transit metadata cache, TTL- and capacity-bounded, with targeted invalidation when a data-path call reports the key gone server-side. Builds older than `1.0.0-rc.1` held this cache with no TTL and no capacity bound: on such a node the window is "until the process restarts" | +| `describe_key` output | One metadata cache TTL: `cache_ttl_seconds` from the KMS configure request, 300 s when omitted, clamped down to 24 h at use (clamped rather than rejected; zero is refused while caching is enabled) | Manager-level key metadata cache; a reporting cache the KV2 state gates never read. `kms service-status` and the configuration endpoint report the effective post-clamp value. Builds older than `1.0.0-rc.1` ignored `cache_ttl_seconds` and ran 300 s while persisting 3600 s as the default, so a cluster configured before then widens to the stored 3600 s on upgrade — read the reported value back | +| KV2 key lifecycle state | None | The KV2 backend re-reads the key record from Vault for every lifecycle and data-key operation | +| Active KMS configuration | One best-effort reload broadcast; unbounded for any peer that did not apply it | See below | ### Configuration changes converge through a best-effort peer reload -`POST /rustfs/admin/v3/kms/configure` and `POST /rustfs/admin/v3/kms/reconfigure` persist the new configuration to cluster storage at `config/kms_config.json`, switch the KMS service **on the node that handled the request**, and then broadcast a reload signal to every peer. A peer that accepts the signal re-reads the persisted configuration and reconfigures itself, so a runtime change normally reaches the whole cluster without any restart. A peer already running that exact configuration treats the signal as a no-op. +`POST /rustfs/admin/v3/kms/configure` and `/kms/reconfigure` persist the new configuration to cluster storage at `config/kms_config.json`, switch the KMS service **on the node that handled the request**, and broadcast a reload signal once to every peer. A peer that accepts the signal re-reads the persisted configuration and reconfigures itself; a peer already running that configuration treats it as a no-op. Convergence is best effort and the request never fails on account of a peer: -Convergence is best effort by contract, and the request never fails on account of a peer: the local node has already switched, and KMS configuration has no quorum or authoritative holder to roll back to. What that leaves: +- There is no background retry. A peer that is unreachable, whose build predates the KMS subsystem, or whose reload fails keeps its previous configuration until a later `reconfigure` reaches it or it restarts. For those peers the split is unbounded. +- The admin response reports success either way, but its message names every peer that did not converge, and the server logs one `kms_peer_config_reload_failed` warning per peer. +- While a split lasts, both configurations are live: if the change switched backends, or changed the Vault mount or key prefix, nodes write new key material to different places and a key created through one node is invisible to the others. -- The broadcast is sent **once**, with no background retry. A peer that is unreachable, that rejects the signal because its build predates the KMS subsystem, or whose reload itself fails keeps serving its previous configuration until a later `reconfigure` reaches it, or until it restarts and loads the persisted configuration during startup. For those peers the split window is still unbounded. -- The admin response reports success either way, but its message names every peer that did not converge, and the server logs one `kms_peer_config_reload_failed` warning per peer. Read the message: an operation that reports success can still have left the cluster split. -- For as long as a split lasts, both configurations are live. If the change switched backends, or changed the Vault mount or key prefix, different nodes write new key material to different places, and a key created through one node is invisible to the others. - -`GET /rustfs/admin/v3/kms/service-status` makes the split observable from a single request: it returns a `cluster_config` object holding one redacted configuration fingerprint per node plus a `consistent` flag. `consistent` is true only when every node answered with the same fingerprint — an unreachable peer, a peer whose build reports no fingerprint, and a node with no configuration at all each read as divergent rather than as agreement. Secrets are substituted out before a configuration is fingerprinted, so two nodes on the same backend holding different credentials still fingerprint alike; the field detects a configuration split, not a credential split. - -Treat a `configure` or `reconfigure` whose response names unconverged peers as an unfinished cluster-wide operation: re-issue it once those peers are reachable, or restart them. +`GET /rustfs/admin/v3/kms/service-status` returns a `cluster_config` object holding one redacted configuration fingerprint per node plus a `consistent` flag, true only when every node answered with the same fingerprint (an unreachable peer, a build reporting no fingerprint, and an unconfigured node each read as divergent). Secrets are substituted out before fingerprinting, so the field detects a configuration split, not a credential split. Treat a `configure` or `reconfigure` whose response names unconverged peers as unfinished: re-issue it once those peers are reachable, or restart them. ### Recommended rolling upgrade order -Follow the node-at-a-time procedure in the [multi-node restart runbook](rolling-restart.md); this adds the KMS-specific sequencing around it. +Follow the node-at-a-time procedure in the [multi-node restart runbook](rolling-restart.md); this adds the KMS-specific sequencing. 1. **Freeze KMS administrative traffic** for the duration: no key creation, enable, disable, tagging, schedule-deletion, cancel-deletion, rotation, or reconfiguration. Object read and write traffic continues normally. 2. **Upgrade one node at a time**, waiting for each to report ready before starting the next. -3. **Verify no node is left behind** before unfreezing. A single old node is enough to reintroduce blind writes and to strip `baseline_version` on its next lifecycle write. +3. **Verify no node is left behind** before unfreezing. A single old node reintroduces blind writes and strips `baseline_version` on its next lifecycle write. 4. **Resume administrative traffic.** -5. **Only then perform the first rotation of any key.** Once the whole cluster understands `master_key_version`, rotation is safe; before that it is not. -6. **If the KMS configuration was changed at any point**, confirm `cluster_config.consistent` is true in the `service-status` response, and re-issue the change — or restart the node — for every peer still reporting a different fingerprint. A peer whose build predates the reload signal never converges on its own. +5. **Only then perform the first rotation of any key.** +6. **If the KMS configuration was changed at any point**, confirm `cluster_config.consistent` is true in the `service-status` response, and re-issue the change — or restart the node — for every peer still reporting a different fingerprint. ### Do not do these during a mixed-version window -- **Rotate any key.** This is the hard constraint stated above; a rotation is unrecoverable for objects an old node must read. -- **Issue any KV2 lifecycle write to an old node.** Its blind write can clobber a concurrent check-and-set commit and will drop `baseline_version` from the record. -- **Create the same key ID from two nodes.** The create path is create-only on upgraded builds, but an old node's blind write does not honor that: the later writer's material wins and every DEK already wrapped with the earlier material becomes permanently unwrappable. -- **Assume a disable or schedule-deletion took effect cluster-wide.** Old Transit nodes cache lifecycle state without expiry; confirm per node, or restart the old nodes, before treating a key as no longer in use. -- **Reconfigure the KMS backend and consider it done.** The reload broadcast is exactly what an old build rejects, so during a mixed-version window the change reaches only the node that served it and the already-upgraded peers. Check the response message and `cluster_config.consistent` before assuming otherwise. +- **Rotate any key.** Unrecoverable for objects an old node must read. +- **Issue any KV2 lifecycle write to an old node.** Its blind write can clobber a concurrent check-and-set commit and drops `baseline_version`. +- **Create the same key ID from two nodes.** The create path is create-only on upgraded builds, but an old node's blind write does not honor that: the later writer's material wins and every DEK wrapped with the earlier material becomes permanently unwrappable. +- **Assume a disable or schedule-deletion took effect cluster-wide.** Old Transit nodes cache lifecycle state without expiry; confirm per node, or restart the old nodes. +- **Reconfigure the KMS backend and consider it done.** The reload broadcast is exactly what an old build rejects; check the response message and `cluster_config.consistent`. - **Delete or prune version records** under `{prefix}/{key_id}/versions/*` for any reason. This is never safe, mixed-version or not; see [Retention and destruction preconditions](#retention-and-destruction-preconditions). ## Choosing between Vault KV2 and Vault Transit -Use **Vault Transit** (`VaultTransit`) when key material must be cryptographically isolated from anyone holding storage-level read access: Transit keeps key-encryption keys inside Vault and only ever returns ciphertext, and supports server-side key versioning/rotation. - -Use **Vault KV2** only when you accept that the Vault ACL on the key path *is* the confidentiality boundary and you want the operational simplicity of a single KV mount. +Use **Vault Transit** (`VaultTransit`) when key material must be cryptographically isolated from anyone holding storage-level read access: Transit keeps key-encryption keys inside Vault, only ever returns ciphertext, and supports server-side key versioning and rotation. Use **Vault KV2** only when you accept that the Vault ACL on the key path *is* the confidentiality boundary and want the operational simplicity of a single KV mount. ## AWS KMS: deviations from the shared backend contract -Select it with `RUSTFS_KMS_BACKEND=aws`. Credentials and region resolution are delegated entirely to the standard `aws-config` provider chain (environment, shared profile, container/IMDS role), so RustFS never stores, persists, or redacts AWS credential material of its own. Only two non-credential settings are read: `RUSTFS_KMS_AWS_REGION` and `RUSTFS_KMS_AWS_ENDPOINT_URL`. A plaintext (`http://`) endpoint override would expose every KMS request including plaintext data keys, so it is refused unless the development opt-in is set. +Select it with `RUSTFS_KMS_BACKEND=aws`. Credentials and region resolution are delegated entirely to the standard `aws-config` provider chain (environment, shared profile, container/IMDS role), so RustFS never stores, persists, or redacts AWS credential material. Only two non-credential settings are read: `RUSTFS_KMS_AWS_REGION` and `RUSTFS_KMS_AWS_ENDPOINT_URL`. A plaintext (`http://`) endpoint override would expose every KMS request including plaintext data keys, so it is refused unless the development opt-in is set. -AWS owns key state, backing-key rotation, and the deletion window, and this backend mirrors none of it locally. That makes four behaviours differ from every RustFS-managed backend. Verify each against your operational assumptions before switching: +AWS owns key state, backing-key rotation, and the deletion window, and this backend mirrors none of it locally. Four behaviours therefore differ from every RustFS-managed backend: | Behaviour | RustFS-managed backends | AWS KMS backend | | --- | --- | --- | | Decryption with a `Disabled` or `PendingDeletion` key | Kept working, so disabling a key never breaks reads of objects already encrypted under it | **Refused by AWS.** Objects encrypted under a key that is later disabled become unreadable until it is re-enabled | -| Key deletion | Physical deletion available | **No physical delete.** `ScheduleKeyDeletion` is the only removal path; AWS destroys the material when the 7-30 day window elapses. RustFS never destroys AWS-held material, and `force_immediate` is refused | -| Cancelling a scheduled deletion | Key returns to `Enabled` | Key is left **`Disabled`**; enable it explicitly to make it usable again | -| Creating a key under a caller-chosen name | The requested name becomes the key id | **Refused.** AWS assigns identifiers and this backend does not manage aliases, so a named create would produce a key unreachable by that name | +| Key deletion | Physical deletion available | **No physical delete.** `ScheduleKeyDeletion` is the only removal path; AWS destroys the material when the 7-30 day window elapses. `force_immediate` is refused | +| Cancelling a scheduled deletion | Key returns to `Enabled` | Key is left **`Disabled`**; enable it explicitly | +| Creating a key under a caller-chosen name | The requested name becomes the key id | **Refused.** AWS assigns identifiers and this backend does not manage aliases | -Two consequences follow from that last row: **SSE-S3 key auto-creation and the synthetic KMS probe are unavailable on this backend**, because both address a key by a name they choose. Pre-create keys in AWS and reference them by AWS key id or ARN. +Consequences of the last row: **SSE-S3 key auto-creation and the synthetic KMS probe are unavailable on this backend**, because both address a key by a name they choose. Pre-create keys in AWS and reference them by AWS key id or ARN. -The AWS backend is intentionally exempt from `backends::contract_tests::assert_state_machine_contract`. That shared driver assumes that disabled and pending-deletion keys still decrypt, that cancelling deletion returns a key to `Enabled`, and that creation accepts a caller-assigned key name. AWS rejects decryption for the first case, leaves a cancelled key `Disabled`, and assigns key identifiers itself, so running the driver would encode the wrong behavior. The exemption is pinned by the offline `aws_backend_shared_contract_exemption_is_pinned` test in `crates/kms/src/backends/aws.rs`; if AWS changes any of these semantics, integrate the backend into the shared driver and remove this exemption rather than weakening the shared assertions. +The AWS backend is exempt from `backends::contract_tests::assert_state_machine_contract`, whose assumptions (disabled keys still decrypt, cancel returns to `Enabled`, caller-assigned names) AWS violates. The exemption is pinned by the offline `aws_backend_shared_contract_exemption_is_pinned` test in `crates/kms/src/backends/aws.rs`; if AWS changes any of these semantics, integrate the backend into the shared driver and remove the exemption rather than weakening the shared assertions. -Key versions are opaque. AWS addresses backing keys internally and picks the right one to decrypt with, so RustFS reports `key_version` as 1 and cannot enumerate versions. Rotation uses `RotateKeyOnDemand`, which retains prior backing keys for decryption; AWS's separate automatic yearly rotation is neither enabled nor reported on by RustFS. +Key versions are opaque: RustFS reports `key_version` as 1 and cannot enumerate versions. Rotation uses `RotateKeyOnDemand`, which retains prior backing keys for decryption; AWS's automatic yearly rotation is neither enabled nor reported on by RustFS. -The KMS admin API accepts the AWS backend as `"backend_type": "AWS"` (aliases `aws`, `aws-kms`, `aws_kms`, `AwsKms`) on `/v3/kms/configure` and `/v3/kms/reconfigure`. The body carries `region` (**required**), and optionally `endpoint_url`, `default_key_id`, and the shared timeout/retry/cache settings. It accepts no credential fields at all — unknown fields are rejected — because every node resolves credentials through its own provider chain. - -`region` is mandatory on this path even though `RUSTFS_KMS_AWS_REGION` is optional at startup: the admin configuration is persisted once and replayed on every node, so a request that left the region to each node's ambient chain would let nodes address different regions, and therefore different keys, while reporting an identical configuration. `default_key_id` must be an AWS key id or ARN that already exists — this backend never creates keys by name. +The KMS admin API accepts the backend as `"backend_type": "AWS"` (aliases `aws`, `aws-kms`, `aws_kms`, `AwsKms`) on `/v3/kms/configure` and `/v3/kms/reconfigure`. The body carries `region` (**required**), and optionally `endpoint_url`, `default_key_id`, and the shared timeout/retry/cache settings; credential fields are rejected as unknown, because every node resolves credentials through its own provider chain. `region` is mandatory even though `RUSTFS_KMS_AWS_REGION` is optional at startup: the admin configuration is replayed on every node, and a request that left the region to each node's ambient chain would let nodes address different regions — and therefore different keys — under an identical configuration. `default_key_id` must be an AWS key id or ARN that already exists. ## Vault TLS: custom CA and mutual TLS @@ -273,40 +221,34 @@ Both Vault backends (KV2 and Transit) support a private certificate authority an | Setting | Environment variable | Admin configure field | Meaning | | --- | --- | --- | --- | -| CA bundle | `RUSTFS_KMS_VAULT_CA_CERT` | `ca_cert_path` | Path to a PEM CA bundle trusted for the Vault connection, in addition to nothing else: when set, only this bundle is trusted | +| CA bundle | `RUSTFS_KMS_VAULT_CA_CERT` | `ca_cert_path` | Path to a PEM CA bundle; when set, only this bundle is trusted | | Client certificate | `RUSTFS_KMS_VAULT_CLIENT_CERT` | `client_cert_path` | Path to a PEM client certificate presented to Vault; requires the client key | | Client key | `RUSTFS_KMS_VAULT_CLIENT_KEY` | `client_key_path` | Path to the PEM private key matching the client certificate | | Skip verification | `RUSTFS_KMS_VAULT_SKIP_TLS_VERIFY` | `skip_tls_verify` | Disables server certificate verification; gated on the insecure development defaults opt-in | -Paths are read on the node applying the configuration, so the files must exist at the same path on every node. The certificate and key must be configured together; configuration validation rejects one without the other, and the files are read and parsed when the backend starts, so a bad path or malformed PEM fails the configuration instead of a later request. The `kms/status` backend summary reports `has_custom_ca` and `has_client_identity` booleans (never the file contents). - -The Vault client library would otherwise fall back to the `VAULT_CACERT`, `VAULT_CAPATH`, `VAULT_CLIENT_CERT` and `VAULT_CLIENT_KEY` process environment variables; RustFS always sets the trust roots and identity explicitly - to the configured values or to empty - so stray Vault environment variables cannot splice TLS material into the connection behind the KMS configuration. +Paths are read on the node applying the configuration, so the files must exist at the same path on every node. Certificate and key must be configured together; the files are read and parsed when the backend starts, so a bad path or malformed PEM fails the configuration rather than a later request. The `kms/status` backend summary reports `has_custom_ca` and `has_client_identity` booleans, never file contents. RustFS always sets the trust roots and identity explicitly — to the configured values or to empty — so the `VAULT_CACERT`, `VAULT_CAPATH`, `VAULT_CLIENT_CERT` and `VAULT_CLIENT_KEY` process environment variables cannot splice TLS material into the connection behind the KMS configuration. ## Local backend durability and deployment support matrix -The Local backend stores one JSON record per key (`.key`) plus an Argon2id salt file (`.master-key.salt`) inside the configured `key_dir`. This section documents which deployments that layout supports and how the backend recovers from a crash or power loss. For where the key material lives and who can read it, see the [backend comparison](#backend-comparison) above. +The Local backend stores one JSON record per key (`.key`) plus an Argon2id salt file (`.master-key.salt`) inside the configured `key_dir`. For where the key material lives and who can read it, see the [backend comparison](#backend-comparison). ### Positioning -The facts today: - -- `Local` is the current default backend (`kms_backend` defaults to `local`). -- The RustFS Kubernetes operator places the key directory on a PersistentVolumeClaim, so the keys survive pod rescheduling. -- The in-code documentation labels the backend "for development and testing only", and configuration validation enforces stricter rules outside explicit development mode: a master key is required and `key_dir` must not live under the process temp directory. +- `Local` is the default backend (`kms_backend` defaults to `local`) and is a development, testing and demo backend; it is not supported for production. Activating a backend whose capabilities report `production_supported: false` logs a `kms_backend_positioning` warning on every start, restart and reconfigure, and the `kms/status` capability matrix carries the same flag. The positioning is a warning, not a gate. +- Configuration validation enforces stricter rules outside explicit development mode: a master key is required and `key_dir` must not live under the process temp directory. +- The RustFS Kubernetes operator places the key directory on a PersistentVolumeClaim, so keys survive pod rescheduling. - Production multi-node deployments should use the Vault Transit backend. -The backend's positioning is settled (owner decision, 2026-08): `Local` is a development, testing and demo backend and is not supported for production. The runtime now states this itself — activating a backend whose capabilities report `production_supported: false` logs a `kms_backend_positioning` warning on every start, restart and reconfigure, and the `kms/status` capability matrix carries the same flag for consoles and tooling. Existing deployments are not blocked: the positioning is a warning, not a gate. This section describes what the implementation guarantees for those who accept that positioning. - ### Deployment support matrix | Deployment | Supported | Notes | | --- | --- | --- | -| Local filesystem (ext4, XFS, APFS, ...) | Yes | The commit protocol relies on POSIX `rename`/`hard_link` atomicity and `fsync` durability, which local filesystems provide | +| Local filesystem (ext4, XFS, APFS, ...) | Yes | The commit protocol relies on POSIX `rename`/`hard_link` atomicity and `fsync` durability | | Kubernetes PVC | Yes | Only when the PersistentVolume is backed by a local or block filesystem; this is how the RustFS operator provisions the key directory | -| NFS or other shared/network filesystems | No | Network filesystems do not reliably provide the atomicity and fsync semantics the commit protocol depends on; an NFS-backed PersistentVolume is this case, not the PVC case above | -| Multiple RustFS processes sharing one `key_dir` | No | Concurrent key **creation** is linearized (`hard_link` refuses to clobber an existing key), but every other write — status updates, deletion, cancellation — is a read-modify-write with no cross-process lock, so concurrent writers can silently lose updates | +| NFS or other shared/network filesystems | No | Network filesystems do not reliably provide the atomicity and fsync semantics the protocol depends on; an NFS-backed PersistentVolume is this case | +| Multiple RustFS processes sharing one `key_dir` | No | Concurrent key **creation** is linearized (`hard_link` refuses to clobber), but every other write is a read-modify-write with no cross-process lock, so concurrent writers can silently lose updates | -Within a single process, per-key write locks serialize read-modify-write updates, so concurrent API calls against one RustFS instance are safe. +Within a single process, per-key write locks serialize read-modify-write updates. ### Crash recovery behavior @@ -317,27 +259,23 @@ Every mutation of the key directory uses a durable commit protocol: 3. The file is published atomically: `rename` to replace an existing file, `hard_link` to create a new one without clobbering. 4. The parent directory is fsynced so the new directory entry is durable. -Deletion mirrors the tail of the protocol (`remove_file` followed by a parent directory fsync), so a deleted key cannot resurface after power loss. A crash at any step leaves either the complete old state or the complete new state, plus at most an unpublished temp file. +Deletion mirrors the tail of the protocol (`remove_file` followed by a parent directory fsync). A crash at any step leaves either the complete old state or the complete new state, plus at most an unpublished temp file. On startup the backend: -On startup the backend then: - -- **Removes orphaned commit temp files.** The matcher is strict (`.tmp-`, never anything ending in `.key`), so published key files — including a key the user named to look like a temp file — are never touched. Publishing is atomic, so a matching leftover can only be an unpublished remnant of an interrupted commit. +- **Removes orphaned commit temp files.** The matcher is strict (`.tmp-`, never anything ending in `.key`), so published key files are never touched. - **Validates every published `.key` file.** A record that fails to decode fails startup rather than being silently skipped. -- **Guards the salt file.** If `.master-key.salt` is missing but the directory contains keys marked `encrypted-master-key`, initialization fails closed with a configuration error naming the salt path. A regenerated salt derives a different master key and can never decrypt those keys, so the correct recovery is to **restore the salt file (or the whole directory) from backup**, never to let a fresh salt be generated. The guard is equally strict about a record it cannot read or cannot interpret — for example one written by a newer RustFS that names an at-rest protection this build does not implement: such a directory's protection state is unknown, so no replacement salt is generated for it either. Recovery is to restore the salt file, run a build that understands the record, or move the unrecognized file out of `key_dir` after confirming it is not needed. An empty directory, or a legacy directory predating the salt file, still initializes normally. +- **Guards the salt file.** If `.master-key.salt` is missing but the directory contains keys marked `encrypted-master-key`, initialization fails closed naming the salt path. A regenerated salt derives a different master key and can never decrypt those keys, so the recovery is to **restore the salt file (or the whole directory) from backup**, never to let a fresh salt be generated. The guard is equally strict about a record it cannot read or interpret (for example one written by a newer RustFS naming an at-rest protection this build does not implement): no replacement salt is generated for such a directory either. An empty directory, or a legacy directory predating the salt file, initializes normally. ### Filesystem permissions and the boundaries the protocol assumes -The key directory is held at `0o700` and every file published into it — key records, the salt, and the files a restore stages and cuts over — is written owner-only. The requested mode is applied and re-read on the open file *before* the content becomes durable, so the process umask cannot widen it, and an unspecified `file_permissions` resolves to owner-only inside the commit protocol rather than at each call site, so no write path can leave it to the umask. +- The key directory is held at `0o700` and every file published into it — key records, the salt, restore staging — is written owner-only. The requested mode is applied and re-read on the open file before the content becomes durable, so the process umask cannot widen it. +- A directory wider than `0o700` is **narrowed on every start** (and re-read to confirm), not refused: kubelet creates `emptyDir` at `0o777`, several PVC provisioners `mkdir -m 0777`, and a `--tmpfs` mount lands at `1777`. Only a directory this process cannot secure is fatal. Narrowing is logged with the previous mode whenever it was reachable beyond the owner. +- Publishing never writes through a symlink: `hard_link` refuses any existing destination (including a dangling symlink) and `rename` replaces the link itself. Startup removes anything wearing a commit-temp name that is not a directory, symlinks included; the protocol only ever creates temps with `create_new`, so such an entry is either its own leftover or something planted. -A directory wider than `0o700` is **narrowed on every start**, and the result is re-read to confirm it took effect. It is not refused: the mode is far more often the platform's than the operator's — kubelet creates an `emptyDir` `0o777`, several PVC provisioners `mkdir -m 0777`, a `--tmpfs` mount lands at `1777` — and refusing would turn each of those into a server that will not start while leaving the exposure in place on the way out. Narrowing removes it. Only a directory this process cannot secure is fatal, because at that point the mode is both dangerous and outside our control. Narrowing is logged with the previous mode whenever it was reachable beyond the owner. +Two boundaries are **not** verified, and deployments should not assume them: -Publishing never writes through a symlink. `hard_link` refuses any destination that already exists — including a dangling symlink — so a create cannot adopt an inode it did not write, and `rename` replaces the link itself rather than the file it points at. Startup removes anything wearing a commit-temp name that is not a directory, symlinks included; the protocol only ever creates temps with `create_new`, so such an entry is either its own leftover or something planted. - -Two boundaries in this area are **not** verified, and deployments should not assume them: - -- **Cross-device operations.** The temp file is always created in the destination's own directory, so `rename` and `hard_link` never cross a filesystem and `EXDEV` is unreachable by construction. That invariant is tested; a real cross-device attempt is not, because it needs a second filesystem. The restore staging directory is always `.restore-staging` inside `key_dir`, so this holds by construction unless that subdirectory is separately bind-mounted onto another filesystem — do not do that. -- **The key directory being replaced mid-commit.** Every path is re-resolved from the directory name rather than held as a directory file descriptor. If `key_dir` is swapped between the `rename` and the parent `fsync`, the fsync lands on the replacement and the new directory entry is never made durable, while the call still reports success. Reaching this requires write access to the key directory's **parent**, which nothing here checks — the mode enforcement above covers `key_dir` itself and says nothing about what encloses it. Keep the parent owner-writable too. Closing this properly means moving the protocol to `renameat`/`linkat` against a held directory descriptor; it is a real gap, recorded as one. +- **Cross-device operations.** The temp file is always created in the destination's own directory, so `rename` and `hard_link` never cross a filesystem; that invariant is tested, a real cross-device attempt is not. The restore staging directory is always `.restore-staging` inside `key_dir` — do not bind-mount it onto another filesystem. +- **The key directory being replaced mid-commit.** Paths are re-resolved from the directory name rather than held as a directory descriptor. If `key_dir` is swapped between the `rename` and the parent `fsync`, the fsync lands on the replacement and the call still reports success. Reaching this requires write access to the key directory's **parent**, which nothing checks — keep the parent owner-writable too. Closing it properly means moving to `renameat`/`linkat` against a held descriptor; it is a recorded gap. ### Backing up the key directory -Back up `key_dir` as a whole, including the hidden `.master-key.salt` file. A key file on its own is not restorable: decrypting it requires the master key derived from the configured `master_key` **and** the persisted salt. Restoring a partial directory — key files without the salt, or the salt without the key files — leaves the backend unable to decrypt, and the salt guard above will (correctly) refuse to start with encrypted keys and no salt. Losing the salt file with no backup means every key encrypted under it is unrecoverable. +Back up `key_dir` as a whole, including the hidden `.master-key.salt` file. A key file on its own is not restorable: decrypting it requires the master key derived from the configured `master_key` **and** the persisted salt. Restoring a partial directory leaves the backend unable to decrypt, and the salt guard will (correctly) refuse to start. Losing the salt file with no backup means every key encrypted under it is unrecoverable. The rehearsal procedure is in the [KMS disaster-recovery drill](kms-disaster-recovery-drill.md). diff --git a/docs/operations/kms-cryptographic-compliance.md b/docs/operations/kms-cryptographic-compliance.md index e8c19e21a..7633bf525 100644 --- a/docs/operations/kms-cryptographic-compliance.md +++ b/docs/operations/kms-cryptographic-compliance.md @@ -1,36 +1,33 @@ # Cryptographic compliance positioning -This document records where RustFS stands on cryptographic module validation, what may and may not be said about it in external material, and what each possible route to a stronger position would actually cost. It exists so that the question is answered once, from the code, instead of being re-litigated from assumptions about crate names and feature flags. +**Use this when:** writing README, CHANGELOG, release notes, marketing, RFP, or security-questionnaire text that touches FIPS, or reasoning about the `rustfs-crypto` `fips` feature and algorithm deprecation. +**Source of truth:** `scripts/check_fips_wording.sh` (the enforced guard); `crates/crypto/Cargo.toml` (`fips` feature); `crates/crypto/src/encdec/id.rs` (`ID` algorithm bytes); `rustfs/src/startup_runtime_hooks.rs` (`install_default_crypto_provider`). -For where master key material lives per backend and how rotation retention works, see [KMS backend security properties](kms-backend-security.md). +This document records where RustFS stands on cryptographic module validation and what may and may not be said about it, so the question is answered once, from the code. For where master key material lives per backend, see [KMS backend security properties](kms-backend-security.md). ## Status: not FIPS 140-3 validated -**RustFS is not FIPS 140-3 (or 140-2) validated, and no component it links is running as a validated cryptographic module.** There is no CMVP certificate covering RustFS or the libraries it uses in the shipped configuration. - -This is a deliberate position, not an oversight. It is also not a statement about algorithm strength: the algorithms in use are standard, well-reviewed AEADs. Validation is a property of a specific module build, its documented boundary, and a certificate — none of which RustFS has or currently pursues. +**RustFS is not FIPS 140-3 (or 140-2) validated, and no component it links runs as a validated cryptographic module.** There is no CMVP certificate covering RustFS or the libraries it uses in the shipped configuration. This is a deliberate position and not a statement about algorithm strength: the algorithms in use are standard, well-reviewed AEADs. Validation is a property of a specific module build, its documented boundary, and a certificate — none of which RustFS has or pursues. ### What the process actually links -The table below is the audited inventory as of this document's writing. "Validated module" asks only whether the code performing the operation is a FIPS-validated cryptographic module; the answer is uniformly no. - | Layer | Where | Implementation | Primitives | Validated module | | --- | --- | --- | --- | --- | -| TLS (S3 server, internode, outbound clients) | Process-wide default provider installed by `install_default_crypto_provider` in `rustfs/src/startup_runtime_hooks.rs` | `rustls` with the `aws-lc-rs` provider | TLS 1.2/1.3 suites, `prefer-post-quantum` hybrid key exchange | No — this is the ordinary `aws-lc-rs` build, not the `aws-lc-fips-sys`-backed FIPS variant | +| TLS (S3 server, internode, outbound clients) | Process-wide default provider installed by `install_default_crypto_provider` in `rustfs/src/startup_runtime_hooks.rs` | `rustls` with the `aws-lc-rs` provider | TLS 1.2/1.3 suites, `prefer-post-quantum` hybrid key exchange | No — the ordinary `aws-lc-rs` build, not the `aws-lc-fips-sys`-backed FIPS variant | | Object data path AEAD (SSE) | `crates/kms/src/encryption/ciphers.rs`, `crates/rio/src/encrypt_reader.rs`, `crates/rio-v2/src/encrypt_reader.rs` | RustCrypto `aes-gcm`, `chacha20poly1305` | AES-256-GCM, ChaCha20-Poly1305 | No | | DEK wrapping | `crates/kms/src/encryption/dek.rs` | RustCrypto `aes-gcm` | AES-256-GCM | No | | Local KMS backend master key | `crates/kms/src/backends/local.rs` | RustCrypto `argon2`, `aes-gcm` | Argon2id KDF, AES-256-GCM | No | | Config and IAM blobs at rest | `crates/crypto/src/encdec/` (`rustfs-crypto`) | RustCrypto `pbkdf2`/`argon2`, `aes-gcm`, `chacha20poly1305`, `sha2` | see [the `fips` feature](#the-rustfs-crypto-fips-feature-what-it-actually-does) | No | | JWT signing and verification | `jsonwebtoken` with the `aws_lc_rs` feature (`crates/crypto`, `crates/iam`, `crates/policy`) | AWS-LC through `aws-lc-rs` | Non-FIPS build | No | -Two consequences follow directly from the table and are worth stating explicitly, because both are commonly assumed the other way: +Two consequences are commonly assumed the other way: -- **AWS-LC being present does not imply FIPS.** `aws-lc-rs` has a FIPS variant; the workspace does not enable it. Every `aws-lc-rs` dependency in the workspace is the default, non-FIPS build. -- **The data path never touches AWS-LC.** Every byte of object plaintext is encrypted by RustCrypto software implementations. Swapping the TLS provider would not change that; see [route 1](#route-1-adopt-the-aws-lc-rs-fips-variant) for what would. +- **AWS-LC being present does not imply FIPS.** `aws-lc-rs` has a FIPS variant; the workspace does not enable it anywhere. +- **The data path never touches AWS-LC.** Every byte of object plaintext is encrypted by RustCrypto software implementations. Swapping the TLS provider would not change that; see route 1 below. ## Terminology red lines for external material -These rules apply to the README, CHANGELOG, release notes, marketing pages, sales decks, RFP responses, and security questionnaires. Claiming validation RustFS does not have is a false statement of fact with regulatory and contractual consequences, not a marketing overreach. +These rules apply to the README, CHANGELOG, release notes, marketing pages, sales decks, RFP responses, and security questionnaires. Claiming validation RustFS does not have is a false statement of fact with regulatory and contractual consequences. ### Never use @@ -38,22 +35,20 @@ These rules apply to the README, CHANGELOG, release notes, marketing pages, sale - "FIPS mode", "runs in FIPS mode", "FIPS-enabled" - "NIST certified", "NIST approved", "CMVP certificate", any certificate number - "meets FIPS requirements", "satisfies FIPS", or any phrasing a reader would reasonably read as validation -- The internal Cargo feature name `fips` as a product capability. It is a build-time algorithm selector (see below), and surfacing it as a feature name invites exactly the misreading this section exists to prevent. +- The internal Cargo feature name `fips` as a product capability. It is a build-time algorithm selector (see below). ### Permitted, with the qualifier attached -- **"FIPS-preferred algorithms"** — permitted only when accompanied, in the same paragraph or table cell, by an explicit non-validation statement. The defined meaning is: *the default algorithm selection is restricted to algorithms on the FIPS 140-3 approved list, implemented by software that has not been validated as a cryptographic module.* -- Naming specific primitives factually ("AES-256-GCM", "ChaCha20-Poly1305", "PBKDF2-HMAC-SHA256") is always fine. Algorithm names carry no validation claim. +- **"FIPS-preferred algorithms"** — only when accompanied, in the same paragraph or table cell, by an explicit non-validation statement. Defined meaning: *the default algorithm selection is restricted to algorithms on the FIPS 140-3 approved list, implemented by software that has not been validated as a cryptographic module.* +- Naming specific primitives factually ("AES-256-GCM", "ChaCha20-Poly1305", "PBKDF2-HMAC-SHA256") is always fine; algorithm names carry no validation claim. -Suggested boilerplate when the topic cannot be avoided: +Boilerplate when the topic cannot be avoided: > RustFS encrypts object data with AES-256-GCM and supports ChaCha20-Poly1305. These are FIPS-approved algorithms, but the implementations are not FIPS 140-3 validated cryptographic modules and RustFS makes no FIPS validation claim. ### Guard -`README.md` and `CHANGELOG.md` currently contain no FIPS-related wording; `scripts/check_fips_wording.sh` is the grep guard for that public baseline. Any future occurrence of the banned strings in either file should be treated as a defect and either removed or brought under the qualifier rule above. This document intentionally contains the terminology needed to define the policy and is not part of that narrow outward-material scan. - -The same script carries a second block for the adjacent over-claim: no file under `crates/kms` may describe the Vault KV2 backend as wrapping key material through Vault's Transit engine. `KmsBackend::VaultKv2` stores RustFS-wrapped key material in Vault's KV v2 engine and never calls Transit, so that wording would tell an operator their key material is cryptographically isolated inside Vault when it is not. Use the `VaultTransit` backend when that isolation is the requirement. +`scripts/check_fips_wording.sh` greps `README.md` and `CHANGELOG.md` for the banned phrases above, and separately rejects any wording under `crates/kms` that describes the Vault KV2 backend as wrapping key material through Vault's Transit engine (`KmsBackend::VaultKv2` stores RustFS-wrapped material in KV v2 and never calls Transit; use `VaultTransit` when cryptographic isolation is the requirement). This document is intentionally outside the scan: it needs the terminology to define the policy. ## The `rustfs-crypto` `fips` feature: what it actually does @@ -64,87 +59,46 @@ The same script carries a second block for the adjacent over-claim: no file unde | enabled (default) | `ID::Pbkdf2AESGCM` (`0x02`) | PBKDF2-HMAC-SHA256, 8192 iterations | AES-256-GCM | | disabled | `ID::Argon2idAESGCM` (`0x00`) or `ID::Argon2idChaCHa20Poly1305` (`0x01`), chosen at runtime by CPU AES support | Argon2id (64 MiB, t=1, p=4) | AES-256-GCM or ChaCha20-Poly1305 | -The selection sites are `crates/crypto/src/encdec/encrypt.rs` and `crates/crypto/src/encdec/stream_io.rs`; the algorithm identifiers and their KDF parameters live in `crates/crypto/src/encdec/id.rs`. +Selection sites are `crates/crypto/src/encdec/encrypt.rs` and `crates/crypto/src/encdec/stream_io.rs`; identifiers and KDF parameters live in `crates/crypto/src/encdec/id.rs`. -Three properties matter for anyone reasoning about this feature: +- **It affects writes only.** The decrypt path accepts all three identifiers unconditionally, and every ciphertext carries its identifier byte, so toggling the feature never orphans existing data. +- **It does not select a different implementation.** Both branches call RustCrypto; the feature cannot move RustFS toward or away from validation. +- **It is a trade-off, not an upgrade.** PBKDF2-HMAC-SHA256 at 8192 iterations is a work factor well below current password-hashing guidance, whereas the non-FIPS branch uses memory-hard Argon2id. Against an offline attack on the passphrase of a stolen config or IAM blob, the default branch is the weaker of the two. -- **It affects writes only.** The decrypt path in `crates/crypto/src/encdec/id.rs` accepts all three identifiers unconditionally, and every ciphertext carries its identifier byte. Toggling the feature therefore never orphans existing data in either direction. -- **It does not select a different implementation.** Both branches call RustCrypto. There is no validated module on either side of the switch, so the feature cannot move RustFS toward or away from validation. -- **It is a trade-off, not an upgrade.** The FIPS-preferred branch uses PBKDF2-HMAC-SHA256 at 8192 iterations, a work factor well below current password-hashing guidance, whereas the non-FIPS branch uses memory-hard Argon2id. Against an attacker who has obtained an encrypted config or IAM blob and is attacking the passphrase offline, the default branch is the weaker of the two. Enabling the feature buys approved-algorithm alignment, not more resistance. - -### Rename recommendation - -The name `fips` states a compliance property the feature does not provide, and `rustfs-crypto` is published, so the name is visible to downstream consumers. Recommended direction: - -1. Introduce `fips-preferred-algs` as the real feature name, carrying the current behavior. -2. Redefine `fips = ["fips-preferred-algs"]` so existing consumers keep building, and mark it deprecated in the crate documentation with a pointer to this document. -3. Drop the `fips` alias after one release cycle. -4. While renaming, raise the PBKDF2 iteration count or document the trade-off above at the feature definition, so the choice is explicit rather than inherited. - -This is a naming and documentation change only; no ciphertext format changes, because the identifier bytes stay as they are. +**Known naming debt.** The feature name `fips` states a compliance property the feature does not provide, and `rustfs-crypto` is published. The intended fix is to introduce `fips-preferred-algs` as the real name, keep `fips` as a deprecated alias for one release cycle, and revisit the PBKDF2 iteration count at the same time; none of this has been done, and no ciphertext format changes when it is. ## Routes to a stronger position, and what each costs -### Route 1: adopt the `aws-lc-rs` FIPS variant - -Switch the whole process to `aws-lc-rs`'s FIPS build (backed by `aws-lc-fips-sys`) so cryptographic operations run inside a validated module boundary. - -**Scope.** The TLS provider swap is the small part — one feature flag plus the provider install sites. The substantial work is the data path: every AEAD call in `crates/kms/src/encryption/ciphers.rs`, `crates/kms/src/encryption/dek.rs`, `crates/rio/src/encrypt_reader.rs`, `crates/rio-v2/src/encrypt_reader.rs`, `crates/kms/src/backends/local.rs`, and `crates/crypto/src/encdec/` would have to be re-implemented against `aws-lc-rs` primitives. Anything the validated module does not expose has to be dropped or moved out of the boundary: Argon2id has no FIPS status, so the Local backend's KDF and the non-FIPS branch of `rustfs-crypto` would need a compatibility story (read-only support for existing records, PBKDF2 for new ones), and ChaCha20-Poly1305 would become non-approved for new writes. - -**Build and platform cost.** `aws-lc-fips-sys` builds a pinned, validated source release and needs CMake, a C toolchain, and Go at build time; it supports a narrower target set than the ordinary crate. The platform matrix cost of plain AWS-LC is already documented and non-hypothetical: rustfs/backlog#883 records that the static musl release build compiles AWS-LC's `getentropy` entropy backend, which aborts on Linux kernels older than 3.17 (the Synology class of device), and that upstream considers this by design with no plan to fix it. The FIPS variant constrains the buildable matrix strictly harder than that, and pins upgrades to whatever the certified source revision allows. - -**What it would and would not buy.** Linking the validated module makes the accurate claim "cryptographic operations are performed by a FIPS 140-3 validated module", not "RustFS is FIPS validated". A product-level claim additionally requires a documented module boundary, approved-mode enforcement, power-on self-tests, key zeroization, and entropy-source documentation, plus the operational procedures to keep them true across releases. - -**Verdict.** Heavy, and it re-opens a platform-support question that is already an open problem. Justified only by a concrete customer or regulatory commitment that names FIPS as a requirement. - -### Route 2: let an externally validated KMS carry key operations - -Keep RustFS as-is and place key management inside someone else's validated boundary: the Vault Transit backend against a Vault deployment whose seal/HSM is validated, or an equivalent managed KMS. - -**Scope.** Mostly already built. The Transit backend (`VaultTransit`) never lets key-encryption key material leave Vault; RustFS only ever holds Transit ciphertext. What remains is configuration guidance, a supported-deployment statement, and the operational documentation that says which parts of the system are covered. - -**What it buys.** Master key generation, wrapping, unwrapping, and rotation happen inside the external module. That is a real, defensible partial answer to "where do keys live and who validated that": it covers the key operations, which is often the part an auditor actually asks about. - -**What it does not buy.** The object data path is untouched. DEKs are used for bulk AEAD by RustCrypto inside the RustFS process, and TLS still runs the non-FIPS AWS-LC build. The honest formulation is "key management operations are performed by an externally validated module; the object data path is not validated". - -**Verdict.** The nearest partial step, with no code rewrite and no platform-matrix risk. This is the route to point customers at when the requirement is about key custody rather than about a certificate covering the storage layer. - -### Route 3: make no validation claim (current default) - -Document the position, hold the terminology line, and revisit only when a requirement with a name attached shows up. - -**Cost.** This document plus the grep guard. Nothing else. - -**Verdict.** The current decision. FIPS 140-3 validation is explicitly not a roadmap target, and adjacent items (PKCS#11, KMIP, BYOK, signing keys) are deferred for lack of demand and because HSM-dependent paths cannot be exercised in CI. +| Route | Scope | What it buys | Verdict | +| --- | --- | --- | --- | +| 1. Adopt the `aws-lc-rs` FIPS variant (`aws-lc-fips-sys`) | The TLS provider swap is the small part. Every AEAD call in the data path (`crates/kms/src/encryption/`, `crates/rio*/src/encrypt_reader.rs`, `crates/kms/src/backends/local.rs`, `crates/crypto/src/encdec/`) would be re-implemented against `aws-lc-rs` primitives; Argon2id (no FIPS status) and ChaCha20-Poly1305 would need read-only compatibility stories. Build needs CMake, a C toolchain, and Go, on a narrower target set — the static musl release already hits AWS-LC's `getentropy` abort on kernels older than 3.17, and the FIPS variant constrains the matrix strictly harder | The accurate claim becomes "cryptographic operations are performed by a FIPS 140-3 validated module", not "RustFS is FIPS validated"; a product-level claim additionally needs a documented boundary, approved-mode enforcement, self-tests, zeroization, and entropy documentation | Heavy; re-opens an open platform-support problem. Justified only by a named customer or regulatory commitment | +| 2. Let an externally validated KMS carry key operations | Mostly built: the `VaultTransit` backend never lets key-encryption key material leave Vault, so master key generation, wrapping, unwrapping, and rotation happen inside whatever module Vault's seal/HSM is validated against. Remaining work is configuration guidance and a supported-deployment statement | A defensible partial answer to "where do keys live and who validated that". The object data path stays RustCrypto and TLS stays non-FIPS AWS-LC: "key management operations are performed by an externally validated module; the object data path is not validated" | Nearest partial step, no code rewrite. Point customers here when the requirement is key custody rather than a certificate covering the storage layer | +| 3. Make no validation claim (current default) | This document plus the grep guard | Nothing further | The current decision. FIPS 140-3 validation is not a roadmap target; adjacent items (PKCS#11, KMIP, BYOK, signing keys) are deferred for lack of demand and because HSM-dependent paths cannot be exercised in CI | ## Algorithm disablement and migration policy -Retiring an algorithm from a storage system is not a code change; it is a data migration with a code change at each end. This section fixes the sequence so that no future deprecation removes a decrypt path while data still depends on it. +Retiring an algorithm from a storage system is a data migration with a code change at each end. This section fixes the sequence so that no deprecation removes a decrypt path while data still depends on it. ### Every persisted artifact is self-describing -The precondition for safe migration already holds: nothing relies on a global "current algorithm" setting to be decodable. - - `rustfs-crypto` blobs carry the `ID` byte (`crates/crypto/src/encdec/id.rs`) immediately after the salt. - KMS ciphers are selected from the recorded `EncryptionAlgorithm` (`crates/kms/src/types.rs`). - DEK envelopes record which master key version wrapped them in `DataKeyEnvelope::master_key_version` (`crates/kms/src/encryption/dek.rs`). -So for any stored object it is decidable, from the object alone, which algorithm and which key version it needs. +For any stored object it is therefore decidable, from the object alone, which algorithm and key version it needs. ### Deprecation classes -Retirement moves an algorithm through these states, never skipping one: - -1. **Write-disabled, read-supported.** New writes select a replacement; existing data decrypts unchanged. This is the only step that is cheap and reversible. +1. **Write-disabled, read-supported.** New writes select a replacement; existing data decrypts unchanged. The only cheap, reversible step. 2. **Read-deprecated.** Reads still work but are counted and warned on, so the remaining population is measurable. 3. **Read-removed.** The decrypt path is deleted. Permitted only once the remaining population is provably zero. ### Sequencing rules - Never advance to read-removed on the strength of an argument that data "should have been" migrated. Removal requires evidence that nothing references the algorithm, not an elapsed-time policy. -- A change to default algorithm selection is a compatibility event: it changes what new nodes write, which matters in a mixed-version cluster. Record it in the release notes and in the relevant crate's feature documentation, and check it against the [mixed-version constraints](kms-backend-security.md#mixed-version-clusters-during-a-rolling-upgrade). -- Roll out write-disablement before the corresponding read change, and let the cluster fully converge in between. A build that cannot read what a peer is still writing is the failure mode to avoid. +- A change to default algorithm selection is a compatibility event: it changes what new nodes write, which matters in a mixed-version cluster. Record it in the release notes and the crate's feature documentation, and check it against the [mixed-version constraints](kms-backend-security.md#mixed-version-clusters-during-a-rolling-upgrade). +- Roll out write-disablement before the corresponding read change, and let the cluster fully converge in between. ### Known gap -Step 3 is partially reachable for object data: the bulk rekey sweep (`POST /rustfs/admin/v3/kms/keys/rekey`) migrates stored DEK envelopes off superseded **master key versions** without touching object bodies. It does not re-encrypt object data, so migrating off a data-encryption **algorithm** still has no supported path — treat every algorithm that has ever been written as permanently read-required, and confine algorithm deprecation to step 1. +The bulk rekey sweep (`POST /rustfs/admin/v3/kms/keys/rekey`, see [`kms-bulk-rekey-contract.md`](../architecture/kms-bulk-rekey-contract.md)) migrates stored DEK envelopes off superseded **master key versions** without touching object bodies. It does not re-encrypt object data, so migrating off a data-encryption **algorithm** has no supported path — treat every algorithm that has ever been written as permanently read-required, and confine algorithm deprecation to step 1. diff --git a/docs/operations/kms-disaster-recovery-drill.md b/docs/operations/kms-disaster-recovery-drill.md index 9dc3a77e3..601ca0836 100644 --- a/docs/operations/kms-disaster-recovery-drill.md +++ b/docs/operations/kms-disaster-recovery-drill.md @@ -1,12 +1,13 @@ # KMS disaster-recovery drill -A KMS backup that has never been restored is a hypothesis. This runbook turns it into evidence: it rehearses the complete loop — back up, lose the persistence layer, preflight, restore, and read historical objects again — and files a machine-readable evidence bundle for each run. For what each backend's backup actually covers, see [KMS backend security properties](kms-backend-security.md); for the metrics and alerts around KMS operations, see the [KMS observability runbook](kms-observability-runbook.md). +**Use this when:** rehearsing a KMS backup-and-restore against a lost key directory (Local backend), producing an evidence bundle for an audit, or restoring a Vault-backed KMS after Vault's own snapshot restore. +**Source of truth:** `crates/kms/examples/kms_dr_drill.rs` (operator entry point); `crates/kms/src/backup/{capability,drill,local_export,local_restore}.rs` (`DrillEvidence`, disaster matrix, restore commit marker). -The acceptance criterion of a drill is not that files came back. It is that objects encrypted before the disaster decrypt after the restore. The harness keeps the ciphertext and encryption metadata of every object it sealed before the disaster and, once the restore is complete, decrypts each one through a freshly opened backend and compares against the pre-disaster digest. Anything less proves only that a bundle is well formed. +The drill rehearses the complete loop — back up, lose the persistence layer, preflight, restore, read historical objects again — and files a machine-readable evidence bundle per run. Its acceptance criterion is not that files came back but that objects encrypted before the disaster decrypt after the restore: the harness keeps the ciphertext and encryption metadata of every object it sealed, and after the restore decrypts each through a freshly opened backend and compares against the pre-disaster digest. For what each backend's backup covers, see [KMS backend security properties](kms-backend-security.md); for KMS metrics and alerts, see the [KMS observability runbook](kms-observability-runbook.md). ## Scope -The drill covers the **Local** backend, which is the only backend RustFS produces a full-material bundle for. The responsibility split is deliberate and is described in `crates/kms/src/backup/capability.rs`: +The drill covers the **Local** backend, the only backend RustFS produces a full-material bundle for. The responsibility split is described in `crates/kms/src/backup/capability.rs`: | Backend | What a RustFS bundle carries | What restores it | | --- | --- | --- | @@ -15,7 +16,7 @@ The drill covers the **Local** backend, which is the only backend RustFS produce | Vault KV2 + Transit | KV metadata and Transit ciphertext references | Vault's native snapshot restore, then the RustFS orchestration | | Vault Transit | Metadata, configuration references, verification data | Vault's native snapshot restore, then the RustFS orchestration | -For the Vault backends there is no RustFS-side export, so there is no loop for a drill to close end to end: the cryptographic root is non-exportable and comes back through Vault's own disaster-recovery flow. What RustFS owns there is the refusal to proceed before that has happened, plus the ordering of everything after it. Rehearse it with the Vault section below. +For the Vault backends there is no RustFS-side export: the cryptographic root is non-exportable and comes back through Vault's own disaster-recovery flow. RustFS owns the refusal to proceed before that has happened and the ordering of everything after it — see the Vault section below. ## What the drill measures @@ -45,7 +46,7 @@ Optional variables: `RUSTFS_KMS_DRILL_DISASTER` (see below), `RUSTFS_KMS_DRILL_I ## Disaster matrix -Run all three; they exercise different failure surfaces and converge on the same procedure, which is the point — an operator does not have to diagnose the failure mode before acting. +Run all three; they exercise different failure surfaces and converge on the same procedure, so an operator does not have to diagnose the failure mode before acting. | `RUSTFS_KMS_DRILL_DISASTER` | Simulates | | --- | --- | diff --git a/docs/operations/kms-observability-runbook.md b/docs/operations/kms-observability-runbook.md index 8e6c6ac36..26a4cab7c 100644 --- a/docs/operations/kms-observability-runbook.md +++ b/docs/operations/kms-observability-runbook.md @@ -1,6 +1,9 @@ # KMS observability runbook -This runbook covers the KMS metrics, the Grafana dashboard that visualizes them, and the response procedure for each Prometheus alert shipped in `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml`. It is the `runbook_url` target for those alerts. For what each KMS backend protects and how Vault authentication behaves, see the [KMS backend security properties](kms-backend-security.md) and the [Vault KMS authentication runbook](vault-kms-authentication.md). +**Use this when:** a `Kms*` Prometheus alert fires (this file is their `runbook_url` target), you are building dashboards or alerts on KMS metrics, or KMS reports not-configured after a restart. +**Source of truth:** `.docker/observability/prometheus-rules/rustfs-kms-alerts.yml` (alert names, thresholds); `crates/kms/src/policy.rs` (backend operation metrics), `crates/kms/src/cache.rs`, `crates/kms/src/deletion_worker.rs`, `crates/kms/src/backends/vault_credentials.rs`, `crates/kms/src/probe.rs`; dashboard `deploy/observability/grafana/rustfs-kms-observability.json`. + +For what each backend protects and how Vault authentication behaves, see [KMS backend security properties](kms-backend-security.md) and the [Vault KMS authentication runbook](vault-kms-authentication.md). ## Metric reference @@ -19,21 +22,21 @@ All six are emitted at the single operation-policy choke point (`crates/kms/src/ | `rustfs_kms_backend_in_flight` | gauge | `backend`, `scope` | External backend attempts currently in flight after admission | | `rustfs_kms_backend_circuit_open` | gauge | `backend`, `scope` | Open or half-open circuits; `0` means closed | -Label values: +| Label | Values | +| --- | --- | +| `backend` | `vault-kv2`, `vault-transit`, `aws`, and `vault-restore` (calls a restore makes against a Vault bundle's trust root). Operation names are shared across backends (each has a `decrypt`), so this label separates a Transit latency regression from an AWS one. Vault credential logins and renewals report their backend's name and are told apart by `operation`; the `scope` label appears only on the two gauges. Local and Static serve from process memory, never enter the operation policy, and emit no `backend` series | +| `outcome` | `success`; `fatal` (non-retryable failure on first observation); `budget_exhausted` (attempt budget ran out on retryable failures); `deadline_exceeded` (operation deadline ran out before another attempt could complete); `backpressure_timeout` (deadline elapsed before capacity admission); `backpressure_rejected` (active capacity and the bounded queue were full or unavailable); `circuit_open` (a retryable failure opened the breaker, or an open breaker rejected the operation); `cancelled` (shutdown or caller cancellation) | +| `op_class` | `read_idempotent` (safe to retry); `mutating_non_idempotent` (never replayed — a retryable failure terminates after one attempt because the server may have processed the request); `auth` (login and token renewal) | +| `error_class` | `retryable_conn` (dial, TLS, broken connection); `retryable_status` (retryable backend status, e.g. Vault 5xx or a sealed Vault's 503); `attempt_timeout` (per-attempt timeout; retried like a connection failure); `fatal` (authentication, permissions, malformed request, missing key or version) | +| `operation` | Static per-call-site names, e.g. `vault_kv2_read_key_version`, `vault_kv2_cas_write_key`, `vault_transit_encrypt`, `vault_transit_decrypt`, `vault_login`, `vault_token_renew` | -- `backend`: the backend that served the call — `vault-kv2`, `vault-transit`, `aws`, and `vault-restore` for the calls a restore makes against a Vault bundle's trust root. Operation names are shared across backends (every one of them has a `decrypt`), so without this label a Vault Transit latency regression and an AWS one land in the same series. Vault credential logins and renewals report their backend's own name and are told apart by the operation, not by a separate `backend` value; the `scope` label that distinguishes them appears only on the two gauges. Local and Static serve from process memory and never enter the operation policy, so they emit no `backend` series at all. -- `outcome`: `success`, `fatal` (a non-retryable failure ended the operation on first observation), `budget_exhausted` (the attempt budget ran out on retryable failures), `deadline_exceeded` (the operation deadline ran out before another attempt could complete), `backpressure_timeout` (the deadline elapsed before capacity admission completed), `backpressure_rejected` (active capacity and the bounded queue were full or unavailable), `circuit_open` (a retryable failure opened the breaker or an open breaker rejected the operation), `cancelled` (shutdown or caller cancellation). -- `op_class`: `read_idempotent` (safe to retry), `mutating_non_idempotent` (never replayed — a retryable failure terminates after a single attempt because the server may have processed the request), `auth` (login and token renewal). -- `error_class`: `retryable_conn` (connection-level failure: dial, TLS, broken connection), `retryable_status` (retryable backend status, e.g. Vault 5xx or a sealed Vault's 503), `attempt_timeout` (the per-attempt timeout cut the attempt off; retried like a connection failure because the server may still have processed the request), `fatal` (non-retryable: authentication, permissions, malformed request, missing key or version). -- `operation`: static per-call-site names, e.g. `vault_kv2_read_key_version`, `vault_kv2_cas_write_key`, `vault_transit_encrypt`, `vault_transit_decrypt`, `vault_login`, `vault_token_renew`. +Admission sharing follows two boundaries. Total active backend capacity is shared by backend identity and capped at `DEFAULT_MAX_CONCURRENT_OPERATIONS`; ordinary operations may use that minus `RESERVED_CREDENTIAL_OPERATIONS`, so login and renewal always retain a reserved slot. Each backend configuration generation owns fresh bounded queues and circuit breakers for its policy scopes, so a failed reconfiguration candidate cannot inherit or mutate the running generation's admission state. -Admission sharing follows two different boundaries. Total active backend capacity is shared by backend identity and capped at 64; ordinary operations are limited to 63 so login and renewal always retain one reserved slot without exceeding the total cap. Each backend configuration generation owns fresh bounded queues and circuit breakers for its policy scopes, so a failed reconfiguration candidate cannot inherit or mutate the running generation's admission state. - -Instrumentation boundary: the Local and Static backends do not flow through the choke point and emit no operation metrics; bringing them under the same instrumentation is tracked separately (rustfs/backlog#1569). Absence of these six series on a cluster using those backends is expected, not an outage. The families below sit above the backend layer and are emitted regardless. +Instrumentation boundary: the Local and Static backends do not flow through the choke point and emit no operation metrics. Absence of these six series on a cluster using those backends is expected, not an outage. The families below sit above the backend layer and are emitted regardless. ### Key metadata cache metrics -Emitted by the manager-level key metadata cache (`crates/kms/src/cache.rs`), which every backend shares. Publication is gated by the cache's `enable_metrics` setting, which defaults to on and which no configure-request field sets today, so in practice these are always published. The counters behind the admin status API are maintained either way, so the switch could never blind `kms service-status`. +Emitted by the manager-level key metadata cache (`crates/kms/src/cache.rs`), which every backend shares. Publication is gated by the cache's `enable_metrics` setting, which defaults to on and which no configure-request field sets, so in practice these are always published. The counters behind the admin status API are maintained either way. | Metric | Type | Labels | Meaning | | --- | --- | --- | --- | @@ -41,48 +44,55 @@ Emitted by the manager-level key metadata cache (`crates/kms/src/cache.rs`), whi | `rustfs_kms_metadata_cache_evictions_total` | counter | `cause` | Entries dropped from the cache, by removal cause | | `rustfs_kms_metadata_cache_entries` | gauge | — | Entries the cache currently holds | -`cause` is `expired` (TTL), `size` (capacity), `explicit` (invalidated by a key lifecycle operation), or `replaced` (overwritten by a newer value). Only `expired` and `size` are true evictions — a sustained `explicit`/`replaced` rate is lifecycle traffic, not cache pressure. +| `cause` | Meaning | +| --- | --- | +| `expired` | TTL — a true eviction | +| `size` | Capacity — a true eviction | +| `explicit` | Invalidated by a key lifecycle operation; a sustained rate is lifecycle traffic, not cache pressure | +| `replaced` | Overwritten by a newer value; same reading as `explicit` | -The entry gauge is republished from every write path and from lookups that miss, because TTL expiry drops entries without any write taking place; a cache that goes completely idle can therefore hold a stale value until the next lookup. Note also that this cache only serves key metadata reads such as `describe_key` — encrypt, decrypt and data key generation never consult it, so a low hit ratio is not a data-path problem. +The entry gauge is republished from every write path and from lookups that miss, because TTL expiry drops entries without any write taking place; a cache that goes completely idle can hold a stale value until the next lookup. This cache only serves key metadata reads such as `describe_key` — encrypt, decrypt and data key generation never consult it, so a low hit ratio is not a data-path problem. ### Key lifecycle metrics -Published by the background deletion worker (`crates/kms/src/deletion_worker.rs`) at the end of each sweep, derived from the pages the sweep already walks, so observing the lifecycle costs no extra backend call. The worker only runs on backends whose capabilities include `schedule_deletion`, so a deployment on a backend without it emits none of these. +Published by the background deletion worker (`crates/kms/src/deletion_worker.rs`) at the end of each sweep, derived from the pages the sweep already walks. The worker only runs on backends whose capabilities include `schedule_deletion`, so a deployment on a backend without it emits none of these. | Metric | Type | Labels | Meaning | | --- | --- | --- | --- | | `rustfs_kms_pending_deletion_keys` | gauge | — | Keys scheduled for deletion whose deadline has not passed | | `rustfs_kms_deletion_tombstone_keys` | gauge | — | Keys left tombstoned by an interrupted removal, still awaiting the sweep | | `rustfs_kms_oldest_key_rotation_age_seconds` | gauge | — | Seconds since the least recently rotated usable key was rotated, counting from creation for keys with no recorded rotation; `0` when there are none | -| `rustfs_kms_max_key_wrap_operations` | gauge | — | Largest reserved wrap-operation count across usable keys; published only by backends that count wraps (Vault KV2 today) | -| `rustfs_kms_deletion_sweep_keys_total` | counter | `outcome` | Keys the sweep acted on, by outcome: `removed`, `blocked`, `skipped`, `failed`, `unreadable` | +| `rustfs_kms_max_key_wrap_operations` | gauge | — | Largest reserved wrap-operation count across usable keys; published only by backends that count wraps (Vault KV2) | +| `rustfs_kms_deletion_sweep_keys_total` | counter | `outcome` | Keys the sweep acted on, by outcome | -`outcome` is `removed`, `blocked` (live configuration — the default key, or a reference reported by the injected checker — still points at the key, so the sweep refuses to remove it), `skipped` (pending but not yet due, or the state changed between inspection and removal), `failed` (the removal attempt failed and is retried next sweep), or `unreadable` (the backend listed a key record this build cannot describe — a record written by a newer build, or damaged material). Every series is emitted at zero from the first sweep on, so a `rate()` over it is defined immediately. +| `outcome` | Meaning | +| --- | --- | +| `removed` | Material destroyed | +| `blocked` | Live configuration (the default key, or a reference reported by the injected checker) still points at the key; the sweep refuses to remove it | +| `skipped` | Pending but not yet due, or the state changed between inspection and removal | +| `failed` | The removal attempt failed; retried next sweep. Also reported, with no key ids, when the listing itself failed | +| `unreadable` | The backend listed a key record this build cannot describe — written by a newer build, or damaged material | -A non-zero `unreadable` rate does not stop the sweep — the expired keys it *can* read are still destroyed — but it does suppress the lifecycle gauges for that round, because a census taken over a partially readable key set would quietly undercount. Sustained `unreadable` therefore shows up as gauges that stop advancing; investigate the named key ids from the sweep's log line before trusting a rotation-age or pending-deletion reading again. +Every series is emitted at zero from the first sweep on, so a `rate()` over it is defined immediately. A non-zero `unreadable` rate does not stop the sweep, but it suppresses the lifecycle gauges for that round, because a census over a partially readable key set would undercount; sustained `unreadable` therefore shows up as gauges that stop advancing — investigate the key ids named in the sweep's log line before trusting a rotation-age or pending-deletion reading again. When *no* key in a complete listing is readable, the backend fails the listing outright (see the [key listing contract](kms-admin-contract.md#key-listing-contract)), so the sweep reports `outcome="failed"` with the listing error and names no key ids: `failed` climbing while `unreadable` stays at zero and the gauges freeze means the whole key set is unreadable on this node — a mixed-version node, or a credential that cannot open any record. Gauges are republished only by a sweep that saw the whole key set; keys already on their way out are excluded from the rotation-age and wrap gauges. -Total damage looks different, and it is worth knowing which you are seeing. When *no* key in a complete listing is readable, the backend fails the listing outright rather than returning an empty page (see the key listing contract in the admin contract page), so the sweep never gets a page to count: it reports `outcome="failed"` with the listing error in its `warn!` line and names no key ids. So `failed` climbing while `unreadable` stays at zero and the gauges freeze means the whole key set is unreadable on this node — a mixed-version node, or a credential that cannot open any record — not that individual removals are failing. +`rustfs_kms_max_key_wrap_operations` tracks the AES-GCM wrap ceiling described under [Rotation drivers and scheduling, per backend](kms-backend-security.md#rotation-drivers-and-scheduling-per-backend). The value is a reservation-based approximation that by design *overestimates*: nodes reserve wrap budget from the key record in blocks of one million and count individual wraps in memory only, so a crash discards unused budget, never a counted wrap. Alert on it approaching 2^32 and rotate the key. It can understate in two bounded, logged cases: a node whose reservation writes keep failing continues wrapping under the warn `Vault KMS wrap budget reservation failed`, and an old build rewriting the key record during a mixed-version window drops the field (see the [mixed-version notes](kms-backend-security.md#mixed-version-clusters-during-a-rolling-upgrade)). Transit and AWS wrap inside the KMS and Local/Static cannot rotate, so none of them publish this series. -The gauges are republished only by a sweep that saw the whole key set; a sweep that could not finish listing leaves the previous, complete values standing rather than understating them. Keys already on their way out are excluded from the rotation-age and wrap gauges, so neither stays pinned high by a key that will never be rotated — or wrap — again. - -`rustfs_kms_max_key_wrap_operations` exists because AES-256-GCM caps one key at 2^32 encryptions under random nonces (NIST SP 800-38D), and the KV2 backend wraps every DEK locally with the key's current material — so wraps track encrypted-object writes and the bound is real. The value is a reservation-based approximation that by design *overestimates*: nodes reserve wrap budget from the key record in blocks of one million and count individual wraps in memory only, so a crash discards unused budget, never a counted wrap. Alert on it approaching 2^32 and rotate the key — rotation installs fresh material and resets the counter. Two ways it can understate, both bounded and logged: a node whose reservation writes keep failing continues wrapping under a warn (`Vault KMS wrap budget reservation failed`), and an old build rewriting the key record during a mixed-version window drops the field (see the [mixed-version notes](kms-backend-security.md#mixed-version-clusters-during-a-rolling-upgrade)). Backends that do not wrap locally with rotatable material publish nothing here: Transit and AWS wrap inside the KMS, and Local/Static cannot rotate, so a counter would be an alarm with no remediation. - -The rotation age comes from whatever the backend reports as the last rotation, and backends only report a rotation they recorded themselves. Today only the Vault KV2 backend persists that timestamp — it is stamped in the same check-and-set write that commits the rotation (`crates/kms/src/backends/vault.rs`), so it exists if and only if the rotation did. Vault Transit and AWS KMS record no rotation timestamp at all: their key listings always report the rotation time as absent, so on those backends every key ages from creation permanently, the gauge measures key age rather than rotation age, and rotating does not reset it. A KV2 key rotated before the timestamp existed likewise ages from creation until its next rotation stamps the record. In every case the gauge overstates rather than invents — it can report an already-rotated key as overdue, never a stale key as fresh — so an alert on it fires early rather than late. Backends that cannot rotate at all (Local, Static) age every key from creation by construction. +The rotation age comes from whatever the backend reports as the last rotation, and only Vault KV2 persists that timestamp — stamped in the same check-and-set write that commits the rotation (`crates/kms/src/backends/vault.rs`). Vault Transit and AWS KMS record none, so on those backends every key ages from creation permanently, the gauge measures key age rather than rotation age, and rotating does not reset it. A KV2 key rotated before the timestamp existed likewise ages from creation until its next rotation. In every case the gauge overstates rather than invents, so an alert on it fires early rather than late. ### Vault credential metrics -Published by the Vault credential provider (`crates/kms/src/backends/vault_credentials.rs`), so they exist only on Vault-backed backends. Both are label-less: there is exactly one credential generation to describe, and the Vault address, mount, auth path and token are all off limits as label values. +Published by the Vault credential provider (`crates/kms/src/backends/vault_credentials.rs`), so they exist only on Vault-backed backends. Both are label-less: there is exactly one credential generation to describe, and the Vault address, mount, auth path and token are off limits as label values. | Metric | Type | Labels | Meaning | | --- | --- | --- | --- | | `rustfs_kms_vault_token_ttl_seconds` | gauge | — | Seconds left before the active Vault token expires; `0` once it has | | `rustfs_kms_vault_credentials_fail_closed` | gauge | — | `1` while the provider refuses to hand out its token because it is inside the fail-closed safety window, `0` otherwise | -The renewal loop republishes both on a 10-second cadence while it waits, generating no extra Vault traffic, so a scrape landing between refresh cycles never reads a TTL frozen at the last refresh. `rustfs_kms_vault_credentials_fail_closed` at `1` is the metric form of the fail-closed window described in the [Vault KMS authentication runbook](vault-kms-authentication.md): while it is set, Vault-backed operations fail rather than run on a credential that may already be invalid. +The renewal loop republishes both on a 10-second cadence while it waits, generating no extra Vault traffic. `rustfs_kms_vault_credentials_fail_closed` at `1` is the metric form of the fail-closed window described in the [Vault KMS authentication runbook](vault-kms-authentication.md): while it is set, Vault-backed operations fail rather than run on a credential that may already be invalid. ### Synthetic probe metrics -Published by the background probe worker (`crates/kms/src/probe.rs`), which generates a data key under a reserved probe key, decrypts it, and compares the material. It runs every `RUSTFS_KMS_PROBE_INTERVAL_SECS` seconds (default 60, raised to a floor of 5, `0` disables the probe entirely), and the status it publishes is what KMS readiness reads. +Published by the background probe worker (`crates/kms/src/probe.rs`), which generates a data key under a reserved probe key, decrypts it, and compares the material. It runs every `RUSTFS_KMS_PROBE_INTERVAL_SECS` seconds (default `DEFAULT_PROBE_INTERVAL`, raised to a floor of `MIN_PROBE_INTERVAL`, `0` disables the probe), and the status it publishes is what KMS readiness reads. | Metric | Type | Labels | Meaning | | --- | --- | --- | --- | @@ -92,17 +102,22 @@ Published by the background probe worker (`crates/kms/src/probe.rs`), which gene | `rustfs_kms_probe_last_success_timestamp_seconds` | gauge | — | Unix timestamp of the most recent successful round | | `rustfs_kms_probe_consecutive_failures` | gauge | — | Rounds that have failed since the last success | -`failure_kind` is `key_provisioning` (the probe key could not be described or created), `generate`, `decrypt`, or `mismatch` — the last means both calls answered but the material did not survive the round trip, which is as serious as an outage and is reported as loudly. +| `failure_kind` | Meaning | +| --- | --- | +| `key_provisioning` | The probe key could not be described or created | +| `generate` | Data key generation failed | +| `decrypt` | Decryption failed | +| `mismatch` | Both calls answered but the material did not survive the round trip — as serious as an outage | -`unsupported` means the backend cannot host the probe key. It is deliberately counted as its own result and never as a failure, and the worker stops after recording it, so failure-counter alerts stay silent on such deployments; the AWS KMS backend is the case in practice, because it refuses a caller-named create. Note also that `rustfs_kms_probe_last_success_timestamp_seconds` only ever moves forward on a success, so while the probe fails its age keeps growing — alert on that age, not on the presence of a failure counter. +`unsupported` means the backend cannot host the probe key (the AWS KMS backend, which refuses a caller-named create). It is counted as its own result, never as a failure, and the worker stops after recording it, so failure-counter alerts stay silent on such deployments. `rustfs_kms_probe_last_success_timestamp_seconds` only moves forward on a success, so while the probe fails its age keeps growing — alert on that age, not on the presence of a failure counter. Export path: the `metrics` facade feeds the OTel recorder in `crates/obs`, which exports over OTLP to the collector scraped by Prometheus. Histograms therefore appear in Prometheus as `_bucket`/`_sum`/`_count` series. None of these metrics carry the RustFS `server` label used by the node observability dashboard — distinguish nodes through your scrape topology (`job`/`instance` or promoted OTel resource attributes such as `service_instance_id`). ## Dashboard -Import `deploy/observability/grafana/rustfs-kms-observability.json` into Grafana and select a Prometheus data source that scrapes RustFS metrics. The dashboard has two variables: `datasource` (Prometheus data source) and `operation` (multi-select over the `operation` label). In the docker-compose observability stack (`.docker/observability/`), dashboards are provisioned from a directory (`grafana/provisioning/dashboards/dashboard.yml` points at `/etc/grafana/dashboards`), so no per-file registration is needed there. +Import `deploy/observability/grafana/rustfs-kms-observability.json` into Grafana and select a Prometheus data source that scrapes RustFS metrics. The dashboard has two variables: `datasource` and `operation` (multi-select over the `operation` label). In the docker-compose observability stack (`.docker/observability/`), dashboards are provisioned from a directory (`grafana/provisioning/dashboards/dashboard.yml` points at `/etc/grafana/dashboards`), so no per-file registration is needed. -The shipped dashboard covers the backend operation metrics only. Its "Planned Panels (TODO)" text panel still describes the cache, lifecycle, Vault credential and probe families as not landed — that panel is stale: the emitting code is merged and the metric names, types and label values are in [Metric reference](#metric-reference) above. Until real panels replace it, query those families ad hoc; nothing in the shipped dashboard or alert rules reads them. See [Coverage gaps](#coverage-gaps). +The shipped dashboard covers the backend operation metrics only. Its "Planned Panels (TODO)" text panel is stale: the cache, lifecycle, Vault credential and probe families are emitted and documented in [Metric reference](#metric-reference) above. Until panels replace it, query those families ad hoc. See [Coverage gaps](#coverage-gaps). ## Alert rules @@ -114,14 +129,12 @@ Every threshold in that file is a conservative default chosen without a producti ### KmsBackendFatalErrors -Meaning: attempts are failing with `error_class="fatal"` — failures the policy never retries. Each one is a KMS backend call that failed permanently (authentication, permissions, malformed request, or a missing key/version), so callers are seeing errors right now. This is the highest-signal KMS alert: fatal failures do not appear as background noise in a healthy system. - -Investigation: +Meaning: attempts are failing with `error_class="fatal"` — failures the policy never retries (authentication, permissions, malformed request, or a missing key/version), so callers are seeing errors right now. This is the highest-signal KMS alert: fatal failures do not appear as background noise in a healthy system. 1. Break the rate down by operation: `sum by (operation) (rate(rustfs_kms_backend_attempt_failures_total{error_class="fatal"}[5m]))`. -2. If the failing operations are `vault_login` or `vault_token_renew` (`op_class="auth"`), the Vault credentials are invalid or expired. Follow the [Vault KMS authentication runbook](vault-kms-authentication.md) — note that credential refresh is fail-closed, so a broken credential eventually takes down all Vault-backed operations, not just auth. Look for the `Vault token renewal failed; falling back to a fresh login` and `Vault credential refresh failed; retrying until the credentials recover` warnings in the RustFS logs; `rustfs_kms_vault_credentials_fail_closed` at `1`, or `rustfs_kms_vault_token_ttl_seconds` at or near `0`, confirms that state without reading logs. -3. If the failing operations are `vault_kv2_*` or `vault_transit_*`, check for Vault permission denials: compare the token's policy against the minimal policy in [KMS backend security properties](kms-backend-security.md) (a policy that drifted or was re-scoped produces 403s that classify as fatal), and check the Vault audit log for the corresponding denied requests. -4. A fatal `KeyVersionNotFound` on decrypt-path operations means a DEK envelope references a key version whose record is missing. Decryption deliberately fails closed with no fallback — see the rotation retention preconditions in [KMS backend security properties](kms-backend-security.md) and verify nobody destroyed version records under the key subtree. +2. If the failing operations are `vault_login` or `vault_token_renew` (`op_class="auth"`), the Vault credentials are invalid or expired. Follow the [Vault KMS authentication runbook](vault-kms-authentication.md) — credential refresh is fail-closed, so a broken credential eventually takes down all Vault-backed operations. Look for the `Vault token renewal failed; falling back to a fresh login` and `Vault credential refresh failed; retrying until the credentials recover` warnings; `rustfs_kms_vault_credentials_fail_closed` at `1`, or `rustfs_kms_vault_token_ttl_seconds` at or near `0`, confirms that state without reading logs. +3. If the failing operations are `vault_kv2_*` or `vault_transit_*`, check for Vault permission denials: compare the token's policy against the [minimal policy](kms-backend-security.md#minimal-vault-policy-for-the-kv2-backend) (a re-scoped policy produces 403s that classify as fatal), and check the Vault audit log for the denied requests. +4. A fatal `KeyVersionNotFound` on decrypt-path operations means a DEK envelope references a key version whose record is missing. Decryption deliberately fails closed with no fallback — see the [retention and destruction preconditions](kms-backend-security.md#retention-and-destruction-preconditions) and verify nobody destroyed version records under the key subtree. 5. Confirm blast radius with the outcome view: `sum by (operation) (rate(rustfs_kms_backend_operations_total{outcome="fatal"}[5m]))`. Related signals: the "Attempt Failure Rate by Error Class" and "Backend Operation Rate by Outcome" dashboard panels; Vault server audit and server logs; S3-level 5xx on encrypted buckets. @@ -130,41 +143,35 @@ Related signals: the "Attempt Failure Rate by Error Class" and "Backend Operatio Meaning: more than 5% of KMS operations are terminating without success (`fatal`, `budget_exhausted`, `deadline_exceeded`, `backpressure_timeout`, `backpressure_rejected`, or `circuit_open`; `cancelled` is excluded because shutdown windows legitimately produce it). A traffic guard suppresses the alert below ~0.02 ops/s so a single failure on a near-idle cluster does not page. -Investigation: - 1. Break the failures down by outcome: `sum by (outcome) (rate(rustfs_kms_backend_operations_total{outcome!~"success|cancelled"}[5m]))`. 2. If `fatal` dominates, follow [KmsBackendFatalErrors](#kmsbackendfatalerrors). -3. If `budget_exhausted` or `deadline_exceeded` dominates, follow [KmsBackendRetryBudgetExhausted](#kmsbackendretrybudgetexhausted) — the backend is unavailable or too slow for longer than the retry policy can bridge. -4. If `backpressure_timeout` or `backpressure_rejected` dominates, compare `rustfs_kms_backend_in_flight` by `backend` and `scope`; total active capacity is shared by backend identity, one slot is reserved for credential refresh, and each configuration generation has fresh scope-local bounded queues. +3. If `budget_exhausted` or `deadline_exceeded` dominates, follow [KmsBackendRetryBudgetExhausted](#kmsbackendretrybudgetexhausted). +4. If `backpressure_timeout` or `backpressure_rejected` dominates, compare `rustfs_kms_backend_in_flight` by `backend` and `scope`; total active capacity is shared by backend identity with one slot reserved for credential refresh, and each configuration generation has fresh scope-local bounded queues. 5. If `circuit_open` dominates, follow [KmsBackendCircuitOpen](#kmsbackendcircuitopen). 6. Correlate with client impact: encrypted-object PUT/GET failures and S3 error rates on buckets with encryption configured. -Related signals: the "Non-Success Outcome Ratio" dashboard panel; the KMS-related warnings listed under the other alerts in this runbook. +Related signals: the "Non-Success Outcome Ratio" dashboard panel; the KMS-related warnings listed under the other alerts. ### KmsBackendP99LatencyHigh -Meaning: the p99 wall-clock duration of KMS operations is sustained above 2s. The histogram includes retries and backoff sleeps, so a high p99 with a healthy p50 usually means a slow retry tail (a subset of calls failing and being retried), not a uniform slowdown. +Meaning: the p99 wall-clock duration of KMS operations is sustained above 2s. The histogram includes retries and backoff sleeps, so a high p99 with a healthy p50 usually means a slow retry tail, not a uniform slowdown. -Investigation: - -1. Compare p50 and p99 on the "Operation Duration p50 / p99" panel. Flat p50 with elevated p99 points at retries; both elevated points at the backend or the network path being uniformly slow. -2. Split by backend and operation with `histogram_quantile(0.99, sum by (le, backend, operation) (rate(rustfs_kms_backend_operation_duration_seconds_bucket[5m])))` to see whether one backend call or all of them regressed. -3. Check the attempts histogram: an average meaningfully above 1 confirms the latency is retry-driven; follow [KmsBackendAttemptFailureSpike](#kmsbackendattemptfailurespike) for the failure classes. -4. If latency is not retry-driven, check the network path to Vault (TLS handshakes, DNS, proxies) and Vault's own telemetry (storage backend latency, load). -5. Remember that this latency sits inside S3 request latency for encrypted objects: sustained p99 near the operation deadline will start converting into `deadline_exceeded` outcomes. +1. Compare p50 and p99 on the "Operation Duration p50 / p99" panel. Flat p50 with elevated p99 points at retries; both elevated points at the backend or network path being uniformly slow. +2. Split by backend and operation: `histogram_quantile(0.99, sum by (le, backend, operation) (rate(rustfs_kms_backend_operation_duration_seconds_bucket[5m])))`. +3. Check the attempts histogram: an average meaningfully above 1 confirms retry-driven latency; follow [KmsBackendAttemptFailureSpike](#kmsbackendattemptfailurespike) for the failure classes. +4. If not retry-driven, check the network path to Vault (TLS handshakes, DNS, proxies) and Vault's own telemetry (storage backend latency, load). +5. This latency sits inside S3 request latency for encrypted objects: sustained p99 near the operation deadline starts converting into `deadline_exceeded` outcomes. Related signals: the "Operation Duration p99 by Operation" and "Operation Attempts Distribution" panels; `KMS backend attempt failed with a retryable error; backing off before retry` warnings (fields: `operation`, `attempt`, `error_class`, `backoff`). ### KmsBackendAttemptFailureSpike -Meaning: individual attempts are failing at a sustained rate across all error classes. The retry policy may still be absorbing these — operations can keep succeeding while this alert fires — but the system is burning retry budget and running degraded, and a small further degradation will surface to callers. - -Investigation: +Meaning: individual attempts are failing at a sustained rate across all error classes. The retry policy may still be absorbing them — operations can keep succeeding while this fires — but the system is burning retry budget and a small further degradation will surface to callers. 1. Break the rate down by class: `sum by (error_class) (rate(rustfs_kms_backend_attempt_failures_total[5m]))`. 2. `retryable_conn`: network-level failures — check connectivity, TLS, DNS, and whether Vault is down or restarting. -3. `retryable_status`: the backend answered with a retryable error — check Vault health and seal status (a sealed Vault returns 503, which lands here), and Vault-side rate limiting. -4. `attempt_timeout`: attempts are being cut off by the per-attempt timeout — either the backend is slow (correlate with [KmsBackendP99LatencyHigh](#kmsbackendp99latencyhigh)) or the configured attempt timeout is too tight for the deployment's network path. +3. `retryable_status`: the backend answered with a retryable error — check Vault health and seal status (a sealed Vault returns 503), and Vault-side rate limiting. +4. `attempt_timeout`: attempts are cut off by the per-attempt timeout — either the backend is slow (correlate with [KmsBackendP99LatencyHigh](#kmsbackendp99latencyhigh)) or the configured attempt timeout is too tight for the network path. 5. `fatal`: follow [KmsBackendFatalErrors](#kmsbackendfatalerrors). 6. Grep RustFS logs for `KMS backend attempt failed with a retryable error; backing off before retry` — the structured fields (`operation`, `attempt`, `error_class`, `backoff`) identify which call sites are cycling. @@ -174,71 +181,65 @@ Related signals: the "Attempt Failure Rate by Error Class" panel; the attempts h Meaning: operations are terminating as `budget_exhausted` or `deadline_exceeded` — every individual failure was retryable, but the backend stayed unhealthy for longer than the retry policy could bridge, so callers received hard failures. -Investigation: - 1. Identify the failing operations: `sum by (operation) (rate(rustfs_kms_backend_operations_total{outcome=~"budget_exhausted|deadline_exceeded"}[5m]))`. 2. Establish how long the underlying failure has persisted from the attempt-failure rate history; follow [KmsBackendAttemptFailureSpike](#kmsbackendattemptfailurespike) for the class-specific diagnosis. -3. Note the by-design case: `mutating_non_idempotent` operations (e.g. `vault_kv2_cas_write_key`, `vault_transit_create_key`) are never replayed, so a single retryable failure terminates them as `budget_exhausted` after one attempt. A spike confined to mutating operations means write-path failures, not an exhausted retry loop. -4. `deadline_exceeded` clustering with duration p99 near the operation deadline means the budget is being spent on slow attempts rather than fast failures — treat as a latency problem first. -5. Confirm client impact and, if the backend outage is confirmed external (Vault down), coordinate recovery there; RustFS will resume without intervention once the backend recovers. +3. By-design case: `mutating_non_idempotent` operations (e.g. `vault_kv2_cas_write_key`, `vault_transit_create_key`) are never replayed, so a single retryable failure terminates them as `budget_exhausted` after one attempt. A spike confined to mutating operations means write-path failures, not an exhausted retry loop. +4. `deadline_exceeded` clustering with duration p99 near the operation deadline means the budget is spent on slow attempts rather than fast failures — treat as a latency problem first. +5. Confirm client impact; if the backend outage is external (Vault down), coordinate recovery there — RustFS resumes without intervention once the backend recovers. -Related signals: the "Backend Operation Rate by Outcome" panel; retry-backoff warnings in RustFS logs; Vault availability monitoring. +Related signals: the "Backend Operation Rate by Outcome" panel; retry-backoff warnings; Vault availability monitoring. ### KmsBackendCircuitOpen -Meaning: `rustfs_kms_backend_circuit_open` has remained above `0` for a `backend` and `scope` for one minute. This direct gauge alert does not depend on operation traffic: it remains visible when the circuit is open and rejecting calls, and while the single half-open recovery probe is running. - -Investigation: +Meaning: `rustfs_kms_backend_circuit_open` has remained above `0` for a `backend` and `scope` for one minute. This direct gauge alert does not depend on operation traffic: it stays visible while the circuit rejects calls and while the single half-open recovery probe runs. 1. Identify the affected scope with `rustfs_kms_backend_circuit_open > 0`. 2. Break recent rejections down by operation: `sum by (operation) (rate(rustfs_kms_backend_operations_total{outcome="circuit_open"}[5m]))`. 3. Check `sum by (error_class) (rate(rustfs_kms_backend_attempt_failures_total{error_class=~"retryable_conn|retryable_status|attempt_timeout"}[5m]))` to distinguish transport failures, retryable backend responses such as a sealed Vault, and attempt timeouts. An attempt timeout counts toward the breaker as a retryable connection failure. -4. After the open interval, the next eligible operation is the only half-open probe. A success or non-retryable failure closes the circuit; a retryable failure reopens it. A non-retryable probe still fails as `fatal`, so follow [KmsBackendFatalErrors](#kmsbackendfatalerrors) even after the circuit gauge clears. Do not restart RustFS just to clear the state. -5. Remember the sharing boundary: each configuration generation has fresh scope-local breaker and queue state, while total active capacity is shared by backend identity with one slot reserved for credential refresh. Check other scopes for capacity pressure even when their circuits remain closed. +4. After the open interval, the next eligible operation is the only half-open probe. A success or non-retryable failure closes the circuit; a retryable failure reopens it. A non-retryable probe still fails as `fatal`, so follow [KmsBackendFatalErrors](#kmsbackendfatalerrors) even after the gauge clears. Do not restart RustFS just to clear the state. +5. Each configuration generation has fresh scope-local breaker and queue state, while total active capacity is shared by backend identity with one slot reserved for credential refresh. Check other scopes for capacity pressure even when their circuits remain closed. Related signals: `circuit_open`, `backpressure_timeout`, and `backpressure_rejected` on the "Backend Operation Rate by Outcome" panel; `rustfs_kms_backend_in_flight`; Vault availability and seal status. ### KmsKeyRotationOverdue -Meaning: `rustfs_kms_oldest_key_rotation_age_seconds` — seconds since the least recently rotated usable key was rotated, counting from creation for keys with no recorded rotation — has been above 400 days for an hour. This is a compliance and hygiene signal, not an outage: encryption and decryption continue unchanged, and nothing in RustFS acts on the verdict. But the longer master key material stays in service the larger the blast radius of its compromise, and on backends where RustFS wraps DEKs locally (Local, Static, Vault KV2) the AES-GCM random-nonce invocation ceiling (NIST SP 800-38D: at most 2^32 wraps under one key) is consumed by every encrypted object write and only ever resets through rotation. +Meaning: `rustfs_kms_oldest_key_rotation_age_seconds` — seconds since the least recently rotated usable key was rotated, counting from creation for keys with no recorded rotation — has been above 400 days for an hour. This is a compliance and hygiene signal, not an outage: encryption and decryption continue unchanged, and nothing in RustFS acts on the verdict. The reasons rotation matters (blast radius, and the AES-GCM wrap ceiling on backends where RustFS wraps DEKs locally) are stated once in [Rotation drivers and scheduling, per backend](kms-backend-security.md#rotation-drivers-and-scheduling-per-backend). -Investigation: - -1. Find which keys are due. The gauge deliberately names no key — a per-key label would carry key identifiers into the metric stream — so read the per-key verdict from the listing: `GET /rustfs/admin/v3/kms/keys` carries `rotation_due` and `rotation_due_reason` (`age`, `never_rotated`, `wraps`, or `unsupported`) per key, computed against `RUSTFS_KMS_ROTATION_MAX_AGE_SECS` and `RUSTFS_KMS_ROTATION_MAX_WRAPS`. A `wraps` reason means the key's material has wrapped more data keys than the configured budget — the AES-GCM random-nonce ceiling rather than an age policy, so it is not satisfied by relaxing the age threshold. The verdict appears only on the listing, not on single-key describe. If `RUSTFS_KMS_ROTATION_MAX_AGE_SECS` is unset, set it to your policy's rotation period so the per-key verdict and this alert agree on what "overdue" means. -2. If the reason is `unsupported`, the backend cannot rotate at all (Local, Static). There is no key-level response; the decision is a backend migration, and the wrap ceiling above is the reason it cannot be deferred forever. See the [rotation drivers and scheduling matrix](kms-backend-security.md#rotation-drivers-and-scheduling-per-backend). -3. On a backend that can rotate, act per the driver matrix: on **Vault KV2**, check why your external rotation scheduler did not run (or set one up — RustFS deliberately ships none) and satisfy the [pre-rotation checklist](kms-backend-security.md#rotation-drivers-and-scheduling-per-backend) before rotating, above all the [upgrade-ordering hard constraint](kms-backend-security.md#upgrade-before-first-rotation-hard-constraint) — never respond to this alert by rotating in the middle of a rolling upgrade. On **Vault Transit**, check `auto_rotate_period` on the key in Vault. On **AWS KMS**, check the key's automatic rotation status in AWS — and do not schedule rotation through the RustFS endpoint, which maps to quota-limited `RotateKeyOnDemand`. -4. Know the gauge's blind spot on Transit and AWS before chasing a rotation that already happened: only KV2 persists a rotation timestamp, so Transit and AWS keys age from creation permanently and this alert will not clear after a rotation there. Confirm the real cadence at the owning system — the Transit key's version history in Vault, or the key's rotation status in AWS — and treat a confirmed-healthy cadence as a known overstatement of this gauge rather than an overdue key. +1. Find which keys are due. The gauge deliberately names no key, so read the per-key verdict from the listing: `GET /rustfs/admin/v3/kms/keys` carries `rotation_due` and `rotation_due_reason` (`age`, `never_rotated`, `wraps`, or `unsupported`) per key, computed against `RUSTFS_KMS_ROTATION_MAX_AGE_SECS` and `RUSTFS_KMS_ROTATION_MAX_WRAPS` — see [Rotation readiness](kms-backend-security.md#rotation-readiness-reported-never-acted-on). A `wraps` reason is not satisfied by relaxing the age threshold. If `RUSTFS_KMS_ROTATION_MAX_AGE_SECS` is unset, set it to your policy's rotation period so the per-key verdict and this alert agree. +2. If the reason is `unsupported`, the backend cannot rotate at all (Local, Static); the only response is a backend migration. +3. On a backend that can rotate, act per the driver matrix in [Rotation drivers and scheduling, per backend](kms-backend-security.md#rotation-drivers-and-scheduling-per-backend) — external scheduler for Vault KV2, `auto_rotate_period` for Vault Transit, AWS automatic rotation for AWS KMS — and satisfy its pre-rotation checklist first, above all the [upgrade-ordering hard constraint](kms-backend-security.md#upgrade-before-first-rotation-hard-constraint). Never respond to this alert by rotating in the middle of a rolling upgrade. +4. Know the gauge's blind spot on Transit and AWS: only KV2 persists a rotation timestamp, so Transit and AWS keys age from creation permanently and this alert will not clear after a rotation there. Confirm the real cadence at the owning system (the Transit key's version history, or the key's rotation status in AWS) and treat a confirmed-healthy cadence as a known overstatement of this gauge. 5. If a KV2 key was genuinely rotated and the gauge stays high, remember the gauge is republished only by a sweep that saw the whole key set: check `rustfs_kms_deletion_sweep_keys_total` for `unreadable` or `failed` outcomes freezing the lifecycle gauges (see [Key lifecycle metrics](#key-lifecycle-metrics)), and that the deletion worker is running at all — it only runs on backends with the `schedule_deletion` capability, which is also why the Static backend never emits this series. -Related signals: `rotation_due` / `rotation_due_reason` on the key listing; `rustfs_kms_deletion_sweep_keys_total{outcome=~"unreadable|failed"}` (a frozen gauge is stale, not healthy); the [rotation drivers and scheduling matrix](kms-backend-security.md#rotation-drivers-and-scheduling-per-backend) and pre-rotation checklist in the backend security properties document. +Related signals: `rotation_due` / `rotation_due_reason` on the key listing; `rustfs_kms_deletion_sweep_keys_total{outcome=~"unreadable|failed"}` (a frozen gauge is stale, not healthy). ## Startup persisted-configuration load KMS configured through the admin API is persisted to cluster storage and restored on every startup. The load result is visible in two places; check both before concluding that KMS "was never configured": -- **Startup log**, `event="kms_persisted_config_lookup"` (`target: rustfs::init`): `state="found"` means the persisted configuration was loaded and applied; `state="not_found"` means no persisted configuration exists on disk; `state="load_failed"` means one exists but reading, unsealing, or decoding it failed. -- **`GET /rustfs/admin/v3/kms/service-status`**: `"NotConfigured"` matches `not_found` (nothing persisted — configuring from scratch is the correct response), while a status of `Error("Failed to load persisted KMS configuration: ...")` or `Error("Failed to apply persisted KMS configuration: ...")` matches `load_failed`. The two states call for different operator actions; do not resubmit a full configuration to recover from `load_failed`. +| Startup log `event="kms_persisted_config_lookup"` (`target: rustfs::init`) | `GET /rustfs/admin/v3/kms/service-status` | Meaning and action | +| --- | --- | --- | +| `state="found"` | configured | Persisted configuration loaded and applied | +| `state="not_found"` | `"NotConfigured"` | Nothing persisted; configuring from scratch is the correct response | +| `state="load_failed"` | `Error("Failed to load persisted KMS configuration: ...")` or `Error("Failed to apply persisted KMS configuration: ...")` | A configuration exists but reading, unsealing, or decoding it failed. Do not resubmit a full configuration; use reload | -To recover from `load_failed` — or from any state where the server runs but its in-memory KMS lags the persisted configuration — call `POST /rustfs/admin/v3/kms/reload` (requires `kms:ServiceControl`). It re-reads the persisted configuration from cluster storage and reconfigures the service without resubmitting secrets, then broadcasts the reload to peer nodes. If reload keeps failing, check cluster storage health first (the read needs quorum), then `RUSTFS_KMS_CONFIG_SECRET`: an unseal error means the secret is missing or differs from the one that sealed the persisted copy — it must be identical on every node. +To recover from `load_failed` — or from any state where the server runs but its in-memory KMS lags the persisted configuration — call `POST /rustfs/admin/v3/kms/reload` (`kms:ServiceControl`). It re-reads the persisted configuration from cluster storage and reconfigures the service without resubmitting secrets, then broadcasts the reload to peer nodes. If reload keeps failing, check cluster storage health first (the read needs quorum), then `RUSTFS_KMS_CONFIG_SECRET`: an unseal error means the secret is missing or differs from the one that sealed the persisted copy — it must be identical on every node. -A separate event, `kms_config_load_skipped` with `reason="storage_uninitialized"`, comes from the ambient loader used by the peer-reload RPC path; during normal startup the loader receives the store explicitly, so seeing this event outside a peer reload indicates a request arrived before storage initialization finished. +A separate event, `kms_config_load_skipped` with `reason="storage_uninitialized"`, comes from the ambient loader used by the peer-reload RPC path; seeing it outside a peer reload indicates a request arrived before storage initialization finished. ## Threshold calibration -Every numeric traffic or latency threshold in `rustfs-kms-alerts.yml` (5% error ratio, 2s p99, 0.5/s attempt failures, 0.05/s budget exhaustion) is a conservative default chosen without a production baseline, biased toward not paging on healthy-but-busy systems. Before relying on these alerts for paging: run the workload in staging for at least a week, record the steady-state values of the expressions above, then tighten thresholds to sit clearly above observed peaks. `KmsBackendCircuitOpen` is different: its gauge is direct state, and the one-minute hold only suppresses a circuit that recovers immediately. `KmsKeyRotationOverdue` is different in the other direction: its 400-day threshold is a policy default (sitting above a common one-year rotation period), not a traffic default — calibrate it against the rotation period your compliance policy requires and against `RUSTFS_KMS_ROTATION_MAX_AGE_SECS`, not against a staging baseline. Once a stable baseline exists, consider converting `KmsBackendAttemptFailureSpike` to a baseline-relative form (`offset 1d` ratio, see `.docker/observability/prometheus-rules/rustfs-get-optimization-alerts.yaml` for the pattern). Formal SLO targets for KMS operations are deliberately out of scope until that baseline exists (rustfs/backlog#1584). +Every numeric traffic or latency threshold in `rustfs-kms-alerts.yml` (5% error ratio, 2s p99, 0.5/s attempt failures, 0.05/s budget exhaustion) is a conservative default chosen without a production baseline, biased toward not paging on healthy-but-busy systems. Before relying on these alerts for paging: run the workload in staging for at least a week, record the steady-state values of the expressions above, then tighten thresholds to sit clearly above observed peaks. `KmsBackendCircuitOpen` is different: its gauge is direct state, and the one-minute hold only suppresses a circuit that recovers immediately. `KmsKeyRotationOverdue` is different in the other direction: its 400-day threshold is a policy default sitting above a common one-year rotation period — calibrate it against the rotation period your compliance policy requires and against `RUSTFS_KMS_ROTATION_MAX_AGE_SECS`, not against a staging baseline. Once a stable baseline exists, consider converting `KmsBackendAttemptFailureSpike` to a baseline-relative form (`offset 1d` ratio; see `.docker/observability/prometheus-rules/rustfs-get-optimization-alerts.yaml` for the pattern). Formal SLO targets for KMS operations are deliberately out of scope until that baseline exists. ## Coverage gaps -The four metric families designed under rustfs/backlog#1584 — key-cache effectiveness, key lifecycle, Vault credentials, synthetic probe — have all landed and are documented in [Metric reference](#metric-reference). What is still missing: - -- **No dashboard panels for those four families, and an alert rule for only one of them.** The key lifecycle family has one rule — [`KmsKeyRotationOverdue`](#kmskeyrotationoverdue) on the rotation-age gauge — while the cache, Vault credential, and probe families are emitted but neither visualized nor alerted on, so they surface only in ad-hoc queries. Building against them is safe now: the names and label values above are what the code emits. -- **The Local and Static backends emit no operation metrics**, because they do not flow through the operation-policy choke point; bringing them under the same instrumentation is tracked separately (rustfs/backlog#1569). Their cache metrics are emitted normally. -- **No formal SLO targets**, deliberately, until a production baseline exists — see [Threshold calibration](#threshold-calibration). - -When a panel or alert rule for one of the landed families is added, replace the corresponding TODO bullet in the dashboard's "Planned Panels" text panel and update this section. +- The cache, Vault credential, and probe metric families have no dashboard panels and no alert rules; the lifecycle family has only [`KmsKeyRotationOverdue`](#kmskeyrotationoverdue). Building against them is safe: the names and label values above are what the code emits. +- The Local and Static backends emit no operation metrics, because they do not flow through the operation-policy choke point. Their cache metrics are emitted normally. +- No formal SLO targets until a production baseline exists — see [Threshold calibration](#threshold-calibration). ## Related documents -- [KMS backend security properties](kms-backend-security.md) — backend trust boundaries, minimal Vault policies, rotation retention preconditions. +- [KMS backend security properties](kms-backend-security.md) — backend trust boundaries, minimal Vault policies, rotation drivers and retention preconditions. - [Vault KMS authentication runbook](vault-kms-authentication.md) — credential sources, refresh behavior, and the fail-closed window. +- [KMS admin API contract](kms-admin-contract.md) — endpoint actions and the key listing contract. - `deploy/observability/README.md` — dashboard import notes for all RustFS dashboards. diff --git a/docs/operations/kms-per-key-authorization.md b/docs/operations/kms-per-key-authorization.md index 51e577d34..07b372d83 100644 --- a/docs/operations/kms-per-key-authorization.md +++ b/docs/operations/kms-per-key-authorization.md @@ -1,8 +1,9 @@ # Per-key KMS authorization -RustFS authorizes KMS access with identity policies. A statement may name the keys it applies to, so a grant such as `kms:DisableKey` no longer implies every key in the cluster, and an SSE-KMS request is checked against the key it actually resolves to. +**Use this when:** writing IAM policies that grant KMS actions on specific keys, choosing a canned KMS role, or enabling SSE-KMS data-path enforcement (`RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY`). +**Source of truth:** `crates/policy/src/policy/resource.rs` (KMS ARN grammar, `KMS_ALIAS_SEGMENT`); `crates/policy/src/policy/action.rs` (`KmsAction`); the canned `KMSKeyAdministrator` / `KMSKeyUser` / `KMSAuditor` policies in `crates/policy`; `rustfs/src/admin/route_policy.rs` (which admin routes are per-key, see [KMS admin API contract](kms-admin-contract.md)). -This page covers the resource grammar, the built-in role templates, the two enforcement planes, and the migration path. For where master key material lives per backend, see [KMS backend security properties](kms-backend-security.md). +RustFS authorizes KMS access with identity policies. A statement may name the keys it applies to, so a grant such as `kms:DisableKey` no longer implies every key in the cluster, and an SSE-KMS request is checked against the key it actually resolves to. This page covers the resource grammar, the built-in role templates, the two enforcement planes, and the migration path. For where master key material lives per backend, see [KMS backend security properties](kms-backend-security.md). ## Resource grammar @@ -81,7 +82,7 @@ Scope and exemptions: - **SSE-KMS only.** SSE-S3 wraps its data key with a server-owned key the caller never names, and SSE-C never reaches KMS; both are exempt, matching AWS. - **The resolved key**, not the header. A bucket default encryption rule naming a KMS key is authorized the same way an explicit `x-amz-server-side-encryption-aws-kms-key-id` header is. -- **Anonymous requests are denied.** An anonymous caller has no identity policy and therefore holds no `kms` grants, so under enforcement every anonymous read or write of an SSE-KMS object fails with `AccessDenied` — even when a bucket policy makes the bucket public. This matches AWS, where anonymous requests cannot use SSE-KMS objects at all, and it keeps the per-key gate meaningful: were anonymous requests exempt, any denied identity could bypass the gate on a public bucket by simply dropping its credentials. **A public bucket serving SSE-KMS objects is incompatible with enforcement** — serve public content unencrypted or under SSE-S3 instead. With enforcement off (the default), anonymous access to SSE-KMS objects remains governed by bucket policy alone. The server warns once per process when it first denies an anonymous request; per-request denials appear on audit entries (`kmsOutcome=failure`, `kmsErrorClass=access_denied`, empty requester identity) and at debug level. +- **Anonymous requests are denied.** An anonymous caller has no identity policy and therefore no `kms` grants, so under enforcement every anonymous read or write of an SSE-KMS object fails with `AccessDenied`, even on a bucket a bucket policy makes public (matching AWS, and closing the bypass of dropping credentials on a public bucket). **A public bucket serving SSE-KMS objects is incompatible with enforcement** — serve public content unencrypted or under SSE-S3. With enforcement off (the default), anonymous access to SSE-KMS objects is governed by bucket policy alone. The server warns once per process on the first anonymous denial; per-request denials appear on audit entries (`kmsOutcome=failure`, `kmsErrorClass=access_denied`, empty requester identity) and at debug level. - **Internal work is exempt.** Replication, lifecycle transitions, healing and the scanner run as the system principal. - **Authorization runs before key state is checked**, so a denial cannot be used to probe whether a key exists, is disabled, or is pending deletion. The response is always `AccessDenied`. - **Multipart uploads are authorized at create time**, where the session data key is generated. Part uploads and completion reuse that envelope and are not re-authorized against the destination key. diff --git a/docs/operations/log-diagnose.md b/docs/operations/log-diagnose.md index ed895a714..5ffb3763a 100644 --- a/docs/operations/log-diagnose.md +++ b/docs/operations/log-diagnose.md @@ -1,66 +1,66 @@ -# 日志故障诊断(`rustfs diagnose`) +# Log diagnosis (`rustfs diagnose`) -对客户/现场提供的日志文件做离线故障归因:解析 RustFS 的 JSON 日志 -(含 `kubectl logs` / docker compose / journald 采集前缀与 stderr panic 块), -匹配内置故障规则库,输出按严重度排序的诊断报告。设计与规则清单见 -rustfs/backlog#1281(总纲)。 +**Use this when:** you have RustFS log files from a customer or a cluster (plain, rotated, archived, or `kubectl logs` output) and need an offline root-cause report, or you need to add or override a diagnosis rule without waiting for a release. +**Source of truth:** `rustfs/src/config/cli.rs` (`DiagnoseOpts`, `DiagnoseFormat`); `rustfs/src/diagnose.rs` (exit codes, time parsing); `crates/log-analyzer/src/rules/model.rs` (`Severity`, `Matcher`, `Rule`); `crates/log-analyzer/src/rules/external.rs` (`EXTERNAL_FILE` mirrors the example below). -不启动存储、不联网、跑完即退,任何装有 `rustfs` 二进制的机器都可用。 +`rustfs diagnose` parses RustFS JSON logs — including `kubectl logs`, docker compose, and journald collection prefixes, and stderr panic blocks — matches them against the built-in rule library, and prints findings sorted by severity. It starts no storage, opens no network connection, and exits once the report is written; any host with the `rustfs` binary can run it. -## 用法 +## Usage ```bash -# 分析一个目录(自动处理 .zst/.gz 轮转归档) +# Analyze a directory (rotated .zst/.gz archives are handled automatically) rustfs diagnose /var/log/rustfs/ -# 客户打包的多节点日志(zip/tar.gz 自动递归展开,第一层目录名作为节点标签) +# Multi-node bundle from a customer (zip/tar.gz are expanded recursively; +# the first-level directory name becomes the node label) rustfs diagnose customer-logs.zip -# 从 stdin 读(容器场景) +# Read from stdin (container workflows) kubectl logs rustfs-0 | rustfs diagnose - -# 只看最近 24 小时,输出 Markdown 直接贴工单 +# Last 24 hours only, Markdown output ready to paste into a ticket rustfs diagnose logs.tar.gz --since 24h --format md > report.md ``` -常用参数:`--format text|json|md`(默认 text;JSON 带稳定 `schema_version`)、 -`--since/--until`(RFC-3339 或相对时间 `30m`/`24h`/`7d`)、`--min-level`、 -`--top N`(未识别模式条数)、`--samples N`(每个 finding 的样本行数)。 +| Flag | Meaning | Default | +| --- | --- | --- | +| `...` | Files, directories, archives (`.zip`, `.tar`, `.tar.gz`, `.zst`, `.gz`), or `-` for stdin | required | +| `--format text\|json\|md` | Output format; JSON carries a stable `schema_version` | `text` | +| `--since`, `--until` | Time window bounds: RFC-3339, or relative (`30m`, `24h`, `7d`) counted back from now | unbounded | +| `--min-level` | Minimum level to analyze (`trace\|debug\|info\|warn\|error`) | all levels | +| `--top N` | Number of unrecognized error patterns to list | 20 | +| `--samples N` | Sample lines per finding | 3 | +| `--redact` | Hash customer identifiers in the report | off | +| `--rules ` | Extra rules file; same-id rules override built-ins | none | -## 报告解读 +Exit codes: `0` — diagnosis completed, with or without findings; `2` — a rejected argument value (`--since`, `--until`, `--min-level`, `--rules`) or no readable input; `1` — a clap usage error (missing path, unknown flag). Findings never fail the process: this is a diagnosis tool, not a CI gate. -- **发现(按严重度)**:P0 数据风险 → P1 服务不可用 → P2 降级 → P3 客户端侧 - → P4 提示。每条带诊断结论、建议动作、证据字段与样本行。 -- **因果折叠**:当症状类发现(如 quorum 刷屏)在时间上跟随其已知根因 - (如盘 faulty)出现时,报告把症状折叠进根因块的"级联症状"行,根因块 - 提升到两者中更高的严重度位置——报告首块直接回答"最可能的原因"。 - JSON 输出保留全部 findings(`collapsed_into` / `caused` 字段标注关系)。 -- **时间线异常(提示)**:三个确定性启发,均为提示不定罪——混合 UTC 偏移 - (伴签名错误时点名时钟偏移)、节点时间范围完全不重叠、时间线断档 - (断档后紧跟 startup 类发现时升级为"重启证据")。 -- **低频提示**:命中数低于规则阈值的匹配(如零星的签名错误),仅供参考。 -- **未识别的高频错误**:规则库未覆盖的 WARN/ERROR 消息模板聚类。这一节是 - 规则库迭代的输入——反复出现的新模板请提给维护者补规则 - (`crates/log-analyzer/src/rules/seed/`)。 -- **跳过的输入与时区提示**:所有被跳过的文件(二进制/超限)逐条披露; - 混合 UTC 偏移会显式提醒(时钟偏移本身就是 `SignatureDoesNotMatch` - 的常见根因)。 +## Reading the report -## `--redact`(报告需要转发时) +| Section | Content | +| --- | --- | +| Findings, by severity | `P0 data risk` → `P1 unavailable` → `P2 degraded` → `P3 client side` → `P4 info`. Each finding carries a diagnosis, a suggested action, evidence fields, and sample lines | +| Causal folding | When a symptom finding (for example a burst of quorum errors) follows its known root cause (for example a faulty disk) in time, the symptom is folded into the root-cause block as a cascaded symptom and that block is promoted to the higher severity of the two, so the first block answers "most likely cause". JSON output keeps every finding and marks the relation with `collapsed_into` / `caused` | +| Timeline anomalies (hints) | Three deterministic heuristics, advisory only: mixed UTC offsets (naming clock skew when signature errors coincide), node time ranges that do not overlap, and log gaps (upgraded to restart evidence when a startup finding follows the gap) | +| Low-confidence hits | Matches below a rule's `min_count` threshold (for example sporadic signature errors), for reference only | +| Unrecognized high-frequency errors | Clustered WARN/ERROR message templates the rule library does not cover. Recurring new templates are the input for new rules under `crates/log-analyzer/src/rules/seed/` | +| Skipped inputs and timezone hints | Every skipped file (binary, or over the size cap) is listed; mixed UTC offsets are called out explicitly because clock skew is a common root cause of `SignatureDoesNotMatch` | -把 bucket/object/AK、IPv4/IPv6、peer 与磁盘路径、节点标签、来源文件路径等客户标识替换为稳定哈希(同值同哈希,保持可关联);规则 id、诊断文本、模块 target 与 panic 源码位置(RustFS 自身代码,非客户数据)保留。样本会连同其完整 `fields` 一起脱敏。 +## `--redact` -覆盖是尽力而为,不是绝对保证:结构化字段与 `key=value`/IP 形态的文本都会被处理,但散落在自由文本里、既非字段形态也非 IP 的标识(例如句子里顺带提到的一个 bucket 名)可能仍有残留。转发前建议再抽查一遍。 +Replaces customer identifiers — bucket, object, and access-key names, IPv4/IPv6 addresses, peer and disk paths, node labels, source file paths — with stable hashes: equal values hash equally, so correlation survives. Rule ids, diagnosis text, module targets, and panic source locations (RustFS code, not customer data) are kept. Samples are redacted together with their full `fields`. -## 自定义规则(`--rules `) +Coverage is best-effort, not a guarantee: structured fields and `key=value` / IP-shaped text are handled, but an identifier embedded in free text that is neither field-shaped nor an IP (for example a bucket name mentioned mid-sentence) can survive. Spot-check the report before forwarding it. -支持团队可以在不等发版的情况下补规则,或热修一条误报的内置规则: +## Custom rules (`--rules `) + +Support teams can add rules, or hot-fix a false positive in a built-in rule, without waiting for a release: ```bash rustfs diagnose customer.zip --rules extra-rules.json ``` -文件格式(`Rule` 的 JSON 表示与内置规则完全一致): +The file is the JSON form of `Rule`, identical to the built-in rules. Rule text fields (`title`, `diagnosis`, `suggestion`) are operator-facing Chinese by design; this example is the one mirrored by `EXTERNAL_FILE` in `crates/log-analyzer/src/rules/external.rs`: ```json { @@ -79,20 +79,18 @@ rustfs diagnose customer.zip --rules extra-rules.json } ``` -- `severity`:`p0_data_risk | p1_unavailable | p2_degraded | p3_client_side | p4_info`; -- `matcher`:`message_prefix` / `message_contains` / `message_regex` / - `field_equals {name, value}` / `target_prefix` / `is_panic` / `min_level` / - `all [..]` / `any [..]`,与内置规则同一套类型; -- 可选字段:`evidence_fields`、`min_count`(默认 1)、`implies_root_cause` - (参与因果折叠)、`anchors`; -- **同 id 覆盖内置规则**(用于热修误报);合并后的规则集整体校验,任何错误 - (坏 regex、重复 id、空 matcher 组)会逐条打印并以退出码 2 失败——不会 - 带着半坏的规则集分析; -- 外部规则的 `anchors` 不受 CI 锚点守卫约束,质量由文件作者自担。 +| Field | Values | +| --- | --- | +| `severity` | `p0_data_risk`, `p1_unavailable`, `p2_degraded`, `p3_client_side`, `p4_info` | +| `matcher` | `message_prefix`, `message_contains`, `message_regex`, `field_equals {name, value}`, `target_prefix`, `is_panic`, `min_level`, `all [..]`, `any [..]` — the same types the built-in rules use | +| Optional fields | `evidence_fields`, `min_count` (default 1), `implies_root_cause` (participates in causal folding), `anchors` | -## 已知边界 +- A rule with the same `id` as a built-in rule replaces it (this is the hot-fix path for a false positive). +- The merged rule set is validated as a whole; every error (bad regex, duplicate id, empty matcher group) is printed and the command exits with code 2 rather than analyzing with a half-broken set. +- `anchors` in external rules are not checked by the CI anchor guard (`scripts/check_log_analyzer_rules.sh`); their quality is the author's responsibility. -- panic 只出现在 stderr:若客户只采集了 stdout,panic 不会出现在日志里, - 但 `rwlock ... poisoned` 类发现会提示"曾发生 panic"。 -- 审计日志(camelCase 外发流)不在本工具范围内。 -- 规则锚点由 CI 守卫(rustfs/backlog#1289)保证与源码日志文案同步。 +## Known boundaries + +- Panics appear only on stderr. If a customer collected stdout only, the panic itself is absent, but a `rwlock ... poisoned` finding hints that a panic occurred. +- Audit logs (the camelCase outbound stream) are out of scope. +- Built-in rule anchors are kept in sync with the source log messages by the CI guard `scripts/check_log_analyzer_rules.sh`. diff --git a/docs/operations/nats-jetstream.md b/docs/operations/nats-jetstream.md index 1116a031e..00e748466 100644 --- a/docs/operations/nats-jetstream.md +++ b/docs/operations/nats-jetstream.md @@ -1,39 +1,25 @@ # NATS JetStream Operations Guide -The guide covers enabling and operating the JetStream publish path for the NATS -notify and audit targets in RustFS. It is written for operators who need -at-least-once delivery of events to NATS, must pre-provision and size the stream -the targets publish to, and need to diagnose delivery failures. +**Use this when:** you need at-least-once delivery of notification or audit events to NATS, must pre-provision and size the stream RustFS publishes to, or are diagnosing JetStream delivery failures. +**Source of truth:** `crates/config/src/constants/targets.rs` (`NATS_JETSTREAM_*` keys and the ack-timeout bounds), `crates/config/src/notify/nats.rs` and `crates/config/src/audit/nats.rs` (env names), `crates/targets/src/target/nats/{validation,jetstream}.rs` (stream validation, `retry_lifetime`, outcome classification), `crates/targets/src/runtime/mod.rs` (`REPLAY_MAX_RETRIES`, `REPLAY_BASE_RETRY_DELAY`), `crates/targets/src/store.rs` (`FAILED_STORE_MAX_ENTRIES`, `FAILED_STORE_TTL`). -## What the JetStream Path Does +## What the JetStream path does -By default the NATS notify and audit targets publish with NATS Core, which -returns once the message is written to the socket. A process or server failure -in the gap between the socket write and the server persisting the message loses -the event. The JetStream path closes that gap. With it enabled, each event is -published to a JetStream stream and the local store-and-forward queue entry -clears only after the server returns a publish acknowledgement, which means the -stream leader has accepted and sequenced the message, or after a terminal -rejection has been recorded in the failed-events store. A server restart -mid-flight loses nothing, because an unacknowledged event stays in the local -queue and replays after the server returns. +By default the NATS notify and audit targets publish with NATS Core, which returns once the message is written to the socket; a failure between that write and the server persisting the message loses the event. With JetStream enabled, each event is first written to the local store-and-forward queue and then published to a JetStream stream with a stable `Nats-Msg-Id` header; the queue entry clears only after the server returns a publish acknowledgement (the stream leader has accepted and sequenced the message) or after a terminal rejection has been recorded in the failed-events store. An unacknowledged event survives a process restart on disk and replays. -The path is opt-in and off by default. With it off, behaviour is the NATS Core -path. It applies to both the notify NATS target and the audit NATS target. +The path is opt-in and off by default, and applies to both the notify NATS target and the audit NATS target. RustFS never creates the stream: the operator owns the stream and its retention, storage, and replication policy. RustFS validates that the stream exists and is writable, reports a validation failure otherwise, and a store-backed target keeps queueing events through a failed validation until the stream is repaired. -RustFS gets each event to the server without losing it before the -acknowledgement. The operator owns the stream and its policy. RustFS validates -that the stream exists and is writable and reports a validation failure -otherwise. A store-backed target keeps running through a failed validation, -holding queued events until the stream is repaired. RustFS never creates the -stream. Stream retention, storage, and replication policy stay under operator -control. +## Configuration -## Enabling JetStream: Recommended Configuration +Each key has a configuration-key form and an environment-variable form per target; the audit target uses the `RUSTFS_AUDIT_NATS_` prefix in place of `RUSTFS_NOTIFY_NATS_`. -Three keys turn the path on and tune it. Each is available on both the notify -NATS target and the audit NATS target, and each has a configuration-key form and -an environment-variable form per target. +| Key | Env (notify) | Required when enabled | Default / range | Meaning | +| --- | --- | --- | --- | --- | +| `jetstream_enable` | `RUSTFS_NOTIFY_NATS_JETSTREAM_ENABLE` | — | off | Turns the JetStream path on for the target | +| `jetstream_stream_name` | `RUSTFS_NOTIFY_NATS_JETSTREAM_STREAM_NAME` | yes | none | Pre-provisioned stream to publish to | +| `jetstream_ack_timeout_secs` | `RUSTFS_NOTIFY_NATS_JETSTREAM_ACK_TIMEOUT_SECS` | no | `NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS` (30); accepted range `NATS_JETSTREAM_ACK_TIMEOUT_MIN_SECS`..=`NATS_JETSTREAM_ACK_TIMEOUT_MAX_SECS` (10..=120) | How long a publish waits for an acknowledgement before it is treated as timed out and retried. Each attempt, including connection establishment, is bounded by this deadline | +| `queue_dir` | `RUSTFS_NOTIFY_NATS_QUEUE_DIR` | yes | none | Local store-and-forward queue directory; durability needs a local store to replay from | +| `queue_limit` | `RUSTFS_NOTIFY_NATS_QUEUE_LIMIT` | no | target default | Maximum live-queue entries; see sizing below | ```bash RUSTFS_NOTIFY_NATS_JETSTREAM_ENABLE=true @@ -42,315 +28,94 @@ RUSTFS_NOTIFY_NATS_JETSTREAM_ACK_TIMEOUT_SECS=30 RUSTFS_NOTIFY_NATS_QUEUE_DIR=/var/lib/rustfs/notify-nats ``` -The audit target takes the same keys under the RUSTFS_AUDIT_NATS_ prefix. +Enabling the path without a stream name or without a queue directory, or with an out-of-range acknowledgement timeout, is rejected at startup and in the admin validation path (`validate_jetstream_settings`). The default 30 s timeout suits production: a server replicating to several replicas acknowledges only after replication and the deferred fsync, which legitimately takes well over 100 ms at the tail. -- jetstream_enable turns the JetStream path on for the target. Off by default. - Environment forms RUSTFS_NOTIFY_NATS_JETSTREAM_ENABLE and - RUSTFS_AUDIT_NATS_JETSTREAM_ENABLE. -- jetstream_stream_name is the name of the pre-provisioned stream to publish to. - Required when enable is on, with no default. Environment forms - RUSTFS_NOTIFY_NATS_JETSTREAM_STREAM_NAME and - RUSTFS_AUDIT_NATS_JETSTREAM_STREAM_NAME. -- jetstream_ack_timeout_secs is how long a publish waits for an acknowledgement - before it is treated as timed out and retried. Range 10 to 120, default 30. - Environment forms RUSTFS_NOTIFY_NATS_JETSTREAM_ACK_TIMEOUT_SECS and - RUSTFS_AUDIT_NATS_JETSTREAM_ACK_TIMEOUT_SECS. -- queue_dir is the local store-and-forward queue directory and is required when - enable is on. Durability is not achievable without a local store to replay - from. Environment forms RUSTFS_NOTIFY_NATS_QUEUE_DIR and - RUSTFS_AUDIT_NATS_QUEUE_DIR. +A mistyped key name is not a value error and escapes that check: the key reads as absent and the target silently stays on the NATS Core path. After enabling, confirm the target reports its JetStream fields on startup and logs the stream-validation success line naming the configured stream; absence of that line for a target meant to be enabled indicates an unrecognised key name. -A configuration that enables the path without a stream name or without a queue -directory is rejected at startup and in the admin validation path. An -acknowledgement timeout outside the 10 to 120 second range is rejected the same -way. +## Stream requirements -The 30 second acknowledgement timeout suits production. A server replicating to -multiple replicas acknowledges only after the replication and the deferred fsync, -which legitimately takes well over 100 milliseconds at the tail, so a short -timeout would spuriously fail a publish mid-replication. +RustFS reads the stream once at target init and in the admin validation path, and reports a validation failure rather than publishing into a stream where writes would silently fail. Revalidation runs while the verdict is unset and stops once one validation passes, resuming only after the verdict is reset (reconnect, TLS rotation, an acknowledgement naming an unexpected stream, or a stream-not-found publish outcome). The stream must: -After enabling, confirm the target is on the JetStream path before relying on it. -Value errors, an out-of-range acknowledgement timeout or a missing stream name -while enable is on, are already rejected loudly at startup and in the admin -validation path. A mistyped configuration key name is not a value error, so it -escapes that check: the key reads as absent, and the target stays on the NATS -Core path with no durability. Verify the enable took effect by confirming the -target reports its JetStream fields on startup and logs the stream-validation -success line naming the configured stream. Absence of that line for a target that -was meant to be enabled indicates a key name the server did not recognise. +1. Exist and be writable. A missing or unreachable stream fails validation; a stream provisioned after the path is enabled picks up the queued events on the next replay, because stream-not-found is retryable. +2. Capture the configured publish subject in its subject filter, by literal match or NATS wildcard. +3. Acknowledge writes (`no_ack` false); otherwise the queue would never clear. +4. Not be sealed. +5. Set a duplicate window of at least the worst-case retry span (next section). -## How Delivery Works +Choose retention, storage type, and replica count for the durability the deployment needs. Use file storage if a server restart must preserve already-persisted events. For durability across a node failure, provision at least 3 replicas: an acknowledgement returns after the stream leader commits, and a leader that acknowledges and then fails before replicating to a quorum can lose that message on failover, so a single-replica stream is durable only against a clean restart. -An event is queued to the local store-and-forward queue first, then published -through JetStream. The publish carries a stable Nats-Msg-Id header and the path -awaits the real publish acknowledgement. +## Duplicate window and the retry span -- The queue entry clears only after the durable acknowledgement returns, or - after a terminal rejection has been recorded in the failed-events store. A - timeout, or any retryable error, retains the entry for a retry. The entry is - never cleared without one of those two outcomes and never silently dropped. A - process interruption between the failed-record write and the live-entry removal - can leave a record whose event a later replay still delivers, a diagnostic - residue and not a lost event. -- The Nats-Msg-Id is minted once when the event is queued and stored with it. It - is identical across every retry and replay of that entry, and unique per - entry, so the server collapses retries and replays of the same event within - the stream duplicate window. A retry after a slow acknowledgement reuses the - same identifier and is not delivered twice. -- A crash before the acknowledgement leaves the entry in the queue. The entry - replays after restart. Closing the target releases the cached connection and - context but never the queued entries, so an entry queued before close survives - on disk and replays on the next start. -- A broker outage detected at connection establishment keeps queued events on - disk. Replay retries with backoff while the broker is unreachable, and the - queued events deliver when the connection recovers, keeping entries queued like - the NATS Core path rather than replicating its mechanics byte for byte. Every - retryable publish failure on an established connection is treated the same way, - kept on the live queue and retried until it delivers, which includes a - connection dropping during the stream-validation lookup. Only a non-retryable - rejection is moved to the failed-events store. +The `Nats-Msg-Id` is minted once when the event is queued and reused on every retry and replay of that entry, so the server collapses duplicates within the stream duplicate window. The window must therefore cover the worst-case span from the first publish attempt of a stored entry to its last within one replay cycle, or a late retry of an already-persisted event is delivered twice. -For durability across a node failure, provision the stream with a replica count -of at least 3. An acknowledgement returns after the stream leader commits. A -leader that acknowledges and then fails before the message replicates to a quorum -can lose that message on failover, so a single-replica stream is durable only -against a clean restart, not against the loss of the node holding the data. +`retry_lifetime` (`crates/targets/src/target/nats/jetstream.rs`) computes that span, and validation rejects a stream whose duplicate window is below it, naming the configured and required window in the log line: -## Pre-provisioning the Stream +```text +duplicate_window >= REPLAY_MAX_RETRIES * ack_timeout_secs + + inter_attempt_backoff_sum(REPLAY_MAX_RETRIES) + + replay_backoff_term(REPLAY_MAX_RETRIES) +``` -The stream is provisioned by the operator before the path is enabled. RustFS -reads the stream once at target init and in the admin validation path, asserts -the requirements below, and reports a validation failure rather than publishing -into a stream where writes would silently fail. A failed validation does not -stop a store-backed target. Events keep queueing to the local store. Revalidation -runs while the validation verdict is unset and stops once one validation passes, -resuming only after the verdict is reset, and the queued events deliver once the -stream is repaired. The stream is not auto-created. A missing stream is an -error. +- `REPLAY_MAX_RETRIES` is the number of publish attempts per replay cycle. +- The sleep before the retry at shift `n` is `replay_backoff_term(n) = REPLAY_BASE_RETRY_DELAY * 2^n`; `inter_attempt_backoff_sum` adds the terms for shifts `1..REPLAY_MAX_RETRIES` (no sleep follows the last attempt). +- The final `replay_backoff_term(REPLAY_MAX_RETRIES)` is a deliberate headroom term above the realized span. -The stream must: +With the constants as shipped (5 attempts, 2 s base) the backoff sum is 60 s and the headroom 64 s, so the default 30 s acknowledgement timeout requires a window of at least 274 s; each extra second of acknowledgement timeout adds `REPLAY_MAX_RETRIES` seconds to the required window. Raise the stream duplicate window in step whenever you raise `jetstream_ack_timeout_secs`. -- Exist and be writable. A missing or unreachable stream fails validation. -- Capture the configured publish subject in its subject filter, by literal match - or by a NATS wildcard. A stream that does not capture the subject fails - validation. -- Acknowledge writes (no_ack false). A stream with no_ack set never returns an - acknowledgement, so the queue would never clear. It fails validation. -- Not be sealed. A sealed stream rejects writes and fails validation. -- Set a duplicate window of at least the retry span (see the next section). A - window below that span fails validation. +The validated window covers one retry cycle. An entry that exhausts a cycle without delivering stays on the live queue and is retried on a later cycle, so an entry surviving across cycles can be delivered again — consistent with at-least-once delivery. Throttling `RUSTFS_NOTIFY_TARGET_STREAM_CONCURRENCY` below the number of concurrently backlogged targets inserts untimed waits between the retries of one entry and can push a late retry past the window; keep the default or add window margin when lowering it. -Choose retention, storage type, and replica count to match the durability the -deployment needs. Use file storage if a server restart must preserve -already-persisted events. These are operator decisions and RustFS does not set -them. If the stream is provisioned shortly after the path is enabled, the queued -events deliver on the next replay rather than being lost, because a -stream-not-found error is retryable and keeps the events on the live queue until -the stream exists. +## Retryable versus terminal outcomes -## Setting the Duplicate Window - -The acknowledgement timeout and the stream duplicate window are linked. The -duplicate window must cover the worst-case retry span, or a late retry of an -event the server already persisted is delivered a second time instead of being -recognised as a duplicate. - -The worst-case retry span is the retry count times the acknowledgement timeout, -plus the realized backoff sleeps across the retries and a final headroom term: - - duplicate_window >= (retry_count * ack_timeout_secs) + realized_backoff + headroom - -There are 5 retry attempts. The realized backoff sleeps sum to 60 seconds -(4 + 8 + 16 + 32), and no sleep follows the last attempt, so a 64 second headroom -term is added above the realized span. At the default 30 second acknowledgement -timeout the required window is: - - (5 * 30) + 60 + 64 = 274 seconds - -Set the stream duplicate window to at least 274 seconds when the acknowledgement -timeout is at the 30 second default. A configuration whose duplicate window is -below this span is rejected at validation, with the configured and required -window named in the log line. - -The span scales with the acknowledgement timeout. Raising -jetstream_ack_timeout_secs requires raising the stream duplicate window in step -using the formula above. The 60 second realized backoff and the 64 second headroom -stay fixed, so each extra second of acknowledgement timeout adds 5 seconds to the -required window. At the 120 second maximum timeout the required window is -(5 * 120) + 60 + 64 = 724 seconds. Each publish attempt, including connection -establishment, is bounded by a single deadline equal to the acknowledgement -timeout. The formula is a conservative upper bound on the retry span, and the -final headroom term holds the required window above the realized span. - -The validated window covers one retry cycle: the worst-case span from the first -publish attempt of a stored entry to its last within a single replay cycle. An -entry that exhausts a cycle without delivering stays on the live queue and is -retried on a later cycle, so an entry surviving across cycles can spend longer in -the queue than the duplicate window and be delivered again, consistent with -at-least-once delivery. - -## The Failed-events Store - -An event that cannot be delivered is recorded, not silently dropped. Only one -case produces a failed event: - -- A terminal rejection. A message that exceeds the maximum payload size, a wrong - expected last message identifier or last sequence, or a sealed stream. These do - not improve with a retry, so they fail fast. - -Every other rejection is retryable and never produces a failed event. A timeout, -backpressure, no responders, a missing stream, a subject the stream does not -capture, or an authorization failure keeps the event on the live queue and -retries it until it delivers or an operator intervenes. A wrong subject and wrong -credentials surface through the validation and connection path as retryable, so -they hold the events on the live queue rather than moving them to the failed -store. - -A failed event is moved to an on-disk failed store that sits in a failed -child directory inside the queue directory, one per target. The directory is created -lazily on the first failed write, so a target that never fails terminally leaves -no failed directory. The entry preserves the event body, its routing metadata, the -deduplication identifier, an error class tag of terminal, the failure time, and -the retry count. An error-level log line is written for each failed event, naming -the bucket, object, event name, and the error, so a broken integration is visible -rather than hidden. A record in the failed store is diagnostic only and is never -republished, so a condition repaired later, for example a raised broker payload -limit, delivers only events still on the live queue. The NATS Core path without -JetStream instead retries such an event until the limit allows it. - -The failed store is bounded by count (10000 entries per target) and by age (a 72 -hour retention), and is kept separate from the live queue limit so an -accumulation of failures cannot crowd out new events. The count is maintained as a -cached value, seeded at startup and reconciled to the directory on each -maintenance interval, so it stays accurate without a directory scan on the hot -path. Failed-store writes and the maintenance scan run under one exclusive guard, -so the at-bound check and the write stay atomic and the bound holds against -concurrent writers. A change made to the directory outside the store can drift the -cached count until the next maintenance interval reconciles it. When the count -bound is reached the oldest failed entry is dropped, with a warning naming the -trimmed entry, so a newer failure is never lost in favour of an older one. Entries -past the retention bound are removed as expired on the replay maintenance tick. - -Because a retryable failure keeps events on the live queue, a long outage grows -the queue toward its configured queue_limit bound. Once the queue reaches that -bound, new events are rejected at ingest with a logged error rather than -overwriting queued events, so the backlog is bounded and visible. - -Size queue_limit for the longest outage the deployment must survive without -rejecting new events. Multiply the peak event rate in events per second by the -outage window in seconds. A target that averages 50 events per second and must -ride out a one hour broker outage needs a queue_limit of at least 50 * 3600 = -180000 entries. Add headroom above the calculated figure, and provision the -queue directory storage for the resulting entry count. - -## Publish Outcome Handling - -Every publish outcome falls into one of two families. Transient conditions are -retried until they deliver and the entry stays on the live queue, so a transient -condition never reaches the failed store. Permanent conditions move to the failed -store immediately. Transient conditions classify toward retry because a terminal -misclassification risks losing an entry once the failed store retention lapses. -Permanent conditions move immediately so a poison message cannot block the queue. - -Both families cover publish outcomes on an established connection. A failure to -establish the connection at all, a refused connection or unreadable TLS -material, is handled before either family applies: the entry retries with -backoff and stays queued until the connection recovers. A publish-level failure -on an established connection, including a connection that drops during the -stream-validation lookup, is retried on the live queue when it is transient and -moved to the failed store only when it is a permanent rejection. +Every publish outcome falls into one of two families, classified once in `crates/targets/src/target/nats/jetstream.rs`. Retryable conditions keep the entry on the live queue and retry until it delivers or an operator intervenes; they never reach the failed store, because a misclassification there would lose an entry once the failed-store retention lapses. Terminal conditions move to the failed store immediately so a poison message cannot block the queue. A failure to establish the connection at all (refused connection, unreadable TLS material) is handled before either family: the entry retries with backoff and stays queued until the connection recovers. | Outcome family | Examples | Handling | | --- | --- | --- | -| Connectivity | connection lost mid-publish, broken pipe | Retried until delivered, live queue | -| Timeouts | no acknowledgement within the timeout, attempt deadline reached | Retried until delivered, live queue | -| Cluster in transition | no leader elected, peer membership changing | Retried until delivered, live queue | -| Stream offline | stream or JetStream subsystem temporarily offline, stream not found | Retried until delivered, live queue | -| Resource and quota exhaustion | insufficient server resources, storage, memory, or account quota reached | Retried until delivered, live queue | -| Server errors | any rejection reporting a 5xx status, with or without a specific error code | Retried until delivered, live queue | -| Permanent rejections | payload too large, wrong expected sequence or message identifier, sealed stream | Failed store immediately | +| Connectivity | Connection lost mid-publish, broken pipe, connection dropped during the stream-validation lookup | Retryable | +| Timeouts | No acknowledgement within the timeout, attempt deadline reached | Retryable | +| Cluster in transition | No leader elected, peer membership changing | Retryable | +| Stream offline | Stream or JetStream subsystem temporarily offline, stream not found | Retryable | +| Resource and quota exhaustion | Insufficient server resources, storage, memory, or account quota | Retryable | +| Server errors | Any rejection reporting a 5xx status | Retryable | +| Wrong subject, wrong credentials, no responders, authorization failure | Surface through the validation and connection path | Retryable — they hold events on the live queue until the configuration is fixed | +| Permanent rejections | Payload exceeds the maximum size, wrong expected last message id or last sequence, sealed stream | Terminal — failed store immediately | -A sealed stream is classified at three points, with three outcomes. Startup validation and the admin -validation path reject a sealed stream before the target serves traffic, so init fails and no event -is queued against it. A stream sealed while the target runs is caught by the next validation on the -publish path, which classifies it retryable and keeps the entry on the live queue while validation -keeps failing. A publish that reaches an already-cached validation pass and is then rejected with the -sealed-stream code is terminal and moves to the failed store immediately. The permanent-rejections -row lists that last outcome. +A sealed stream is classified at three points: startup and admin validation reject it before the target serves traffic; a stream sealed while the target runs is caught by the next validation on the publish path and treated as retryable; a publish that reaches an already-cached validation pass and is then rejected with the sealed-stream code is terminal. + +A process interruption between the failed-record write and the live-entry removal can leave a failed record whose event a later replay still delivers — a diagnostic residue, not a lost event. + +## The failed-events store + +A terminal rejection is recorded, not silently dropped. The failed store is an on-disk `failed` child directory inside the queue directory, one per target, created lazily on the first terminal write. Each entry preserves the event body, routing metadata, the deduplication identifier, an error class tag of `terminal`, the failure time, and the retry count, and an error-level log line names the bucket, object, event name, and error. Records are diagnostic only and are never republished: a condition repaired later (for example a raised broker payload limit) delivers only events still on the live queue, whereas the NATS Core path would have kept retrying such an event. + +The store is bounded by `FAILED_STORE_MAX_ENTRIES` per target and by `FAILED_STORE_TTL` (`crates/targets/src/store.rs`), separately from the live queue limit so failures cannot crowd out new events. When the count bound is reached the oldest failed entry is dropped with a warning naming it; entries past the TTL are removed on the replay maintenance tick. The count is a cached value seeded at startup and reconciled to the directory on each maintenance interval; writes and the maintenance scan share one exclusive guard so the bound holds against concurrent writers, and a change made to the directory outside the store drifts the cached count until the next reconciliation. + +## Sizing `queue_limit` + +Because retryable failures keep events on the live queue, a long outage grows the queue toward `queue_limit`. At the bound, new events are rejected at ingest with a logged error rather than overwriting queued events. Size it for the longest outage the deployment must survive without rejecting: peak events per second times the outage window in seconds — a target averaging 50 events/s that must ride out a one-hour broker outage needs at least 50 * 3600 = 180000 entries — plus headroom, and provision the queue directory storage accordingly. ## Observability -The failed_store_length gauge reports the number of entries in the failed-events -store per target, next to the existing failed_messages and queue_length gauges, -on both the notify and audit metric paths. A rising failed_store_length points to -terminal rejections accumulating for a target. An exhaustion warning marks each -cycle where an entry spends its full retry budget without delivering, after which -the entry stays queued and retries on the next scan. The warning repeats once per -retry cycle while the entry stays queued, so a persistent delivery problem is -visible at warn level without per-attempt noise. The failed_messages count -advances only on terminal and dropped events, never on a retry or an exhaustion, -so it counts entries that left the queue for good rather than entries still -retrying. +| Gauge | Meaning | +| --- | --- | +| `queue_length` | Entries on the live queue (existing gauge) | +| `failed_store_length` | Entries in the failed-events store per target; a rising value means terminal rejections are accumulating | +| `failed_messages` | Advances only on terminal and dropped events, never on a retry or an exhaustion — entries that left the queue for good | + +All three are emitted on both the notify and audit metric paths (`crates/targets/src/runtime/mod.rs`). An exhaustion warning marks each cycle in which an entry spends its full retry budget without delivering; the entry stays queued and the warning repeats once per cycle, so a persistent delivery problem is visible at warn level without per-attempt noise. ## Troubleshooting -- Validation fails at startup with a missing-stream error. The stream named in - jetstream_stream_name does not exist on the server, or is not reachable. - Provision it, or correct the name. -- Validation fails with a subject, no_ack, sealed, or duplicate-window error. The - pre-provisioned stream does not capture the publish subject, has no_ack set, is - sealed, or has a duplicate window below the worst-case retry span. Correct the - stream configuration to satisfy the pre-provisioning requirements above. -- Repeated stream-not-found errors in the log. The stream disappeared or was - never created. The publish is retryable, so the events stay on the live queue - and retry. Provision the stream so the queued events deliver on the next - replay. -- A warning that a publish was acknowledged by an unexpected stream. The - configured stream no longer captures the publish subject and another stream - does. The mismatched publish is rejected with a retryable error, the entry - stays on the live queue, and the mismatch resets the stream-validation - verdict. Later retries then fail validation before publishing, so the log - shows the validation failure rather than a repeating mismatch warning. The - mismatch warning fires again only after a validation pass lets another - publish through. The acknowledging stream named in the warning has persisted - a copy of each mismatched publish. Inspect that stream and remove the stray - copies. Delivery resumes once the configured stream captures the subject - again and validation passes. -- Health checks slow while a stream fails validation. Health reflects the last - stream-validation verdict rather than a live lookup on every check. The verdict - resets on a reconnect, a TLS rotation, an acknowledgment naming an unexpected - stream, or a stream-not-found publish outcome, and a reset forces a live stream - lookup on the next check or publish, bounded by the acknowledgement timeout. A - health snapshot taken right after a reset can take up to that timeout per - affected target until the stream is repaired. After a broker connection heals - on its own, publishes and health rely on the last validation verdict until an - error outcome resets it, a wrong-stream acknowledgment or a stream-not-found - publish outcome, so a stream reconfigured during a silent reconnect is - detected on the first publish evidence rather than immediately. -- Events stay in the queue and do not clear. The server is not acknowledging. - Check connectivity, that the stream subject filter captures the publish - subject, and that the server has capacity. Unacknowledged events are retained, - not lost. -- Duplicate deliveries observed. The duplicate window is shorter than the - worst-case retry span. Raise the stream duplicate window using the formula - above, especially after raising the acknowledgement timeout. An outage or a - restart that keeps an unacknowledged entry queued longer than the duplicate - window can also produce a duplicate on replay, consistent with at-least-once - delivery. Throttling RUSTFS_NOTIFY_TARGET_STREAM_CONCURRENCY below the number - of concurrently backlogged targets inserts untimed waits between the retries - of one entry and can push a late retry past the window. Keep the default or - add window margin when lowering it. -- Failed-store entries accumulating. A terminal rejection is recurring: a - message over the maximum payload size, a wrong expected last message identifier - or last sequence, or a sealed stream. Failed entries are on-disk diagnostic - records carrying the error class and a fixed diagnostic detail for inspection. - They are not re-published, and they are removed after the retention period. A - wrong subject or wrong credentials do not land here. They surface as retryable - and hold the events on the live queue until the configuration is fixed. +| Symptom | Cause | Action | +| --- | --- | --- | +| Validation fails at startup with a missing-stream error | The stream named in `jetstream_stream_name` does not exist or is unreachable | Provision it, or correct the name | +| Validation fails with a subject, `no_ack`, sealed, or duplicate-window error | The stream violates one of the requirements above | Correct the stream configuration | +| Repeated stream-not-found errors in the log | The stream disappeared or was never created; the publish is retryable | Provision the stream; queued events deliver on the next replay | +| Warning that a publish was acknowledged by an unexpected stream | The configured stream no longer captures the publish subject and another stream does. The publish is rejected as retryable and the validation verdict is reset, so later retries fail validation instead of repeating the warning | Inspect the acknowledging stream named in the warning and remove the stray copies; delivery resumes once the configured stream captures the subject and validation passes | +| Health checks slow while a stream fails validation | Health reflects the last validation verdict; a reset forces a live stream lookup on the next check or publish, bounded by the acknowledgement timeout, so a snapshot right after a reset can take up to that timeout per affected target | Repair the stream. After a silent reconnect, a reconfigured stream is detected on the first publish evidence (wrong-stream acknowledgement or stream-not-found), not immediately | +| Events stay in the queue and do not clear | The server is not acknowledging | Check connectivity, that the subject filter captures the publish subject, and server capacity. Unacknowledged events are retained, not lost | +| Duplicate deliveries observed | Duplicate window shorter than the worst-case retry span, an outage or restart that kept an entry queued longer than the window, or `RUSTFS_NOTIFY_TARGET_STREAM_CONCURRENCY` throttled below the backlogged-target count | Raise the stream duplicate window per the formula above, especially after raising the acknowledgement timeout | +| Failed-store entries accumulating | A terminal rejection is recurring (payload too large, wrong expected sequence or message id, sealed stream) | Fix the cause; the records are diagnostic, are not republished, and expire after `FAILED_STORE_TTL`. Wrong subject or credentials never land here — they hold events on the live queue | -## Disabling the Path +## Disabling the path -Set jetstream_enable off. The target reverts to the NATS Core path. Events -already in the queue are delivered by the standard replay. No failed-store -entries are created while the path is off. +Set `jetstream_enable` off. The target reverts to the NATS Core path; events already in the queue are delivered by the standard replay, and no failed-store entries are created while the path is off. diff --git a/docs/operations/no-parity-bitrot-recovery.md b/docs/operations/no-parity-bitrot-recovery.md index 6f5bf123d..6386edc0f 100644 --- a/docs/operations/no-parity-bitrot-recovery.md +++ b/docs/operations/no-parity-bitrot-recovery.md @@ -1,98 +1,49 @@ # No-Parity Bitrot Recovery Guide -This guide covers historical objects written with erasure data shards but no -parity shards, for example an object whose `xl.meta` reports `EcM=1` and -`EcN=0`. PR #5179 prevents RustFS from committing new objects after it detects -this class of no-parity bitrot failure, but operators may still find already -committed objects on disk. +**Use this when:** a GET or deep heal reports `FileCorrupt` / `bitrot hash mismatch` on an object whose `xl.meta` shows `EcN=0` (data shards, no parity), and the raw `part.N` file is still readable from the filesystem. +**Source of truth:** `crates/ecstore/src/set_disk/ops/heal.rs` (heal returns `FileCorrupt` for confirmed no-parity bitrot, `ErasureReadQuorum` otherwise); `crates/filemeta/examples/dump_fileinfo.rs` (offline `xl.meta` decoder). + +This guide covers historical objects written with erasure data shards but no parity shards (for example `EcM=1`, `EcN=0`). The write path now self-verifies no-parity writes and refuses to commit an object whose shard fails bitrot verification, so new objects of this class cannot be created, but already committed ones may still exist on disk. ## Symptom -An affected object can look surprising during incident response: +- The raw shard file (`part.1`) is visible and readable from the local filesystem. +- An S3 GET or deep heal reports an integrity failure: `FileCorrupt`, `bitrot hash mismatch`, an unrecoverable heal result, or a truncated streaming response. +- No parity shards exist to reconstruct the corrupted data shard. -- the raw shard file, such as `part.1`, is visible and readable from the local - filesystem; -- an S3 GET or deep heal reports an integrity failure, commonly through - `FileCorrupt`, `bitrot hash mismatch`, or an unrecoverable heal result; -- there are no parity shards available to reconstruct the corrupted data shard. +The filesystem-readable `part.N` is evidence, not trusted object data: the stored hash no longer matches the bytes on disk, and RustFS must not bypass bitrot validation to serve it. -The filesystem-readable `part.N` file is therefore evidence, not trusted object -data. RustFS must not bypass bitrot validation to serve it through S3, because -the stored hash no longer matches the bytes on disk. - -## What To Capture +## What to capture Before deleting or moving anything, capture: -- bucket name, object key, and version ID if versioning is enabled; -- the RustFS version and whether the deployment was running with no parity - (`EcN=0`) at the time the object was written; -- the heal or GET error, including any `FileCorrupt`, `bitrot hash mismatch`, - `ErasureReadQuorum`, or truncated streaming response message; -- `xl.meta` from every shard disk that still has the object; -- the raw `part.N` file from every shard disk that still has the object. +1. Bucket name, object key, and version ID if versioning is enabled. +2. The RustFS version, and whether the deployment ran with no parity (`EcN=0`) when the object was written. +3. The heal or GET error text. +4. `xl.meta` and the raw `part.N` file from every shard disk that still has the object. -For local inspection, decode metadata with: +Decode metadata locally and record the erasure geometry (`EcM`, `EcN`), object size, part number, part logical size, data directory, and checksum algorithm: ```bash cargo run -p rustfs-filemeta --example dump_fileinfo -- /path/to/disk/bucket/object/xl.meta ``` -Record the erasure geometry (`EcM`, `EcN`), object size, part number, part -logical size, data directory, and checksum algorithm from the decoded metadata. +## Size accounting -## Size Accounting +Erasure shard files include bitrot hash data in addition to object bytes: with the default `HighwayHash256S` checksum each protected block adds 32 bytes, so a raw `part.1` larger than the logical object size is normal (for example 8,250,370 logical bytes in 8 blocks → 8,250,626 raw bytes). The size relationship only shows the layout is plausible; the bitrot reader is the authority for integrity. -RustFS erasure shard files include bitrot hash data in addition to object bytes. -For the default HighwayHash256S checksum, each protected block adds 32 bytes of -hash data to the shard file. A raw `part.1` size can therefore be larger than -the object logical size and still be normal. +## Recovery boundary -Example: +If `EcN=0` and a data shard fails bitrot verification, RustFS cannot reconstruct the object from the erasure set. The valid options are: -```text -logical object bytes: 8,250,370 -protected blocks: 8 -hash overhead: 8 * 32 = 256 bytes -raw part.1 bytes: 8,250,626 -``` +- Restore the object from an external backup, replica, upstream source, or a known-good copy outside the affected erasure set. +- Preserve the affected `xl.meta` and `part.N` files as incident evidence, then delete the object through the normal S3/admin path when retention policy allows. +- Quarantine by copying evidence out of the live data path first; remove or isolate the live object path only after the incident owner confirms the evidence is no longer needed. -This size relationship only proves that the file layout is plausible. It does -not prove the bytes are valid. The bitrot reader is the authority for integrity. +Do not edit `xl.meta`, rewrite `part.N`, or serve raw shard bytes to clients as the object. Those actions hide evidence and convert a detected integrity failure into silent data corruption. -## Recovery Boundary +If `EcN>0`, this guide is not the primary recovery path: run normal heal first, since parity may allow RustFS to reconstruct the missing or corrupt shard. -If `EcN=0` and a data shard fails bitrot verification, RustFS cannot reconstruct -the object from the erasure set. The valid recovery options are: +## Expected diagnostics -- restore the object from an external backup, replica, upstream source, or a - known-good copy outside the affected erasure set; -- preserve the affected `xl.meta` and `part.N` files as incident evidence, then - delete the object through the normal S3/admin path when retention policy - allows it; -- quarantine by copying evidence out of the live data path first, then remove - or isolate the live object path only after the incident owner confirms the - evidence is no longer needed. - -Do not edit `xl.meta`, rewrite `part.N`, or serve raw shard bytes to clients as -the object. Those actions hide evidence and can convert a detected integrity -failure into silent data corruption. - -If `EcN>0`, this guide is not the primary recovery path. Use normal heal first; -parity may allow RustFS to reconstruct the missing or corrupt shard. - -## Expected Diagnostics - -Deep heal should report no-parity corruption as an unrecoverable integrity -failure rather than only a generic read-quorum problem. The diagnostic context -should include: - -- bucket, object, and version ID; -- erasure data shard count and parity shard count; -- part number; -- whether the failing part had a bitrot failure; -- the number of missing or corrupt shards. - -When the failure is confirmed bitrot on a no-parity object, the heal error is -reported as `FileCorrupt`, and the heal result `detail` states that the -no-parity object is unrecoverable. +Deep heal reports no-parity corruption as an unrecoverable integrity failure rather than a generic read-quorum problem: the heal error is `FileCorrupt`, and the heal result `detail` states that the no-parity object is unrecoverable. The diagnostic context includes bucket, object, and version ID; data and parity shard counts; part number; whether the failing part had a bitrot failure; and the number of missing or corrupt shards. diff --git a/docs/operations/object-io-tuning-ab-matrix.md b/docs/operations/object-io-tuning-ab-matrix.md index baf0b906b..95d3f6f35 100644 --- a/docs/operations/object-io-tuning-ab-matrix.md +++ b/docs/operations/object-io-tuning-ab-matrix.md @@ -1,50 +1,29 @@ # Object I/O (GET/PUT) tuning A/B matrix runbook -> Scope: **parameter tuning** — measuring the effect of changing one -> `RUSTFS_*` runtime knob at a time, against a fixed binary. This is -> deliberately different from the code-change A/B gate in -> [`hotpath-warp-ab-runbook.md`](hotpath-warp-ab-runbook.md) and the formal -> ABBA validation in -> [`hotpath-warp-abba-runbook.md`](hotpath-warp-abba-runbook.md), which compare -> a baseline binary against a candidate binary. -> -> When a knob change turns out to need a code change, use those two runbooks -> for the code-level validation and come back here for the knob-level sweep. +**Use this when:** you want to measure the effect of changing one `RUSTFS_*` runtime knob at a time against a fixed binary, or you need the catalog of GET/PUT tuning knobs, their defaults, and the stage metric that validates each. +**Source of truth:** `crates/io-metrics/src/lib.rs` (stage histogram names and stage tokens); `crates/config/src/constants/object.rs` and `crates/ecstore/src/erasure/coding/encode.rs` (knob defaults); `crates/ecstore/src/set_disk/mod.rs` (codec-streaming rollout and engine defaults); `rustfs/src/app/object/get.rs` (GET experimental switches). -## 1. What this runbook answers +Scope: **parameter tuning** with a fixed binary. The code-change A/B gate that compares a baseline binary against a candidate binary is [`hotpath-warp-ab-runbook.md`](hotpath-warp-ab-runbook.md); when a knob change turns out to need a code change, validate it there and come back here for the knob-level sweep. -For each tuning knob it answers three questions: +## 1. Questions this runbook answers -1. Which stage is actually slow — `set_disk_encode`, `set_disk_rename`, - `metadata_fanout`, `bitrot_verify`, etc.? +1. Which stage is actually slow — `set_disk_encode`, `set_disk_rename`, `metadata_fanout`, `bitrot_verify`, ...? 2. Is the knob the real bottleneck, or is the stage slow for another reason? -3. Does widening/loosening the knob buy throughput without an unacceptable - memory (RSS) or tail-latency regression? +3. Does widening the knob buy throughput without an unacceptable memory (RSS) or tail-latency regression? -The core discipline is **one variable per A/B cell**. Never change two knobs in -the same cell, or the result is unexplainable. +The core discipline is **one variable per A/B cell**. A cell that changes two knobs is thrown away. ## 2. Prerequisites -- Linux bench host (or an ansible-managed cluster); a laptop smoke run is too - noisy to decide anything. -- `warp` on `PATH` (or pass `--warp-bin` to the driver). -- The observability metrics runtime **enabled**. The stage histograms below are - not emitted when `RUSTFS_OBS_METRICS_EXPORT_ENABLED=false` or the runtime is - otherwise off — see - [`hotpath-warp-ab-runbook.md`](hotpath-warp-ab-runbook.md) for the no-log / - no-monitor baseline env. -- A warm, disposable data set. Recreate the bucket per run; do not bench against - production data. +Host, `warp`, and the no-log / no-monitor baseline environment are as in [`hotpath-warp-ab-runbook.md`](hotpath-warp-ab-runbook.md). Two additions: the observability metrics runtime must be **enabled** (the stage histograms are not emitted when `RUSTFS_OBS_METRICS_EXPORT_ENABLED=false`), and the data set must be disposable — recreate the bucket per run. Load driver and gate are reused, not reimplemented: -- `scripts/run_object_batch_bench_enhanced.sh` — warp driver with rounds, - median aggregation, `baseline_compare.csv`, and Prometheus service-metric - capture. -- `scripts/hotpath_warp_ab_gate.sh` — relative budget gate over the deltas. -- `scripts/run_hotpath_warp_ab.sh` — optional orchestrator when a knob needs - the full baseline-vs-candidate treatment (e.g. two different defaults). +| Script | Role | +| --- | --- | +| `scripts/run_object_batch_bench_enhanced.sh` | warp driver with rounds, median aggregation, `baseline_compare.csv`, Prometheus service-metric capture | +| `scripts/hotpath_warp_ab_gate.sh` | relative budget gate over the deltas | +| `scripts/run_hotpath_warp_ab.sh` | orchestrator when a knob needs the full baseline-vs-candidate treatment (e.g. two different defaults) | ## 3. Fixed test conditions (lock before you start) @@ -56,9 +35,6 @@ network, erasure_set_drive_count, endpoint_mode (direct|lb), rustfs_commit_sha, warp --version, durability mode ``` -Workload matrix (the same shapes the hotpath gate uses, expanded for the -stage-breakdown object sizes): - | Workload | mode | sizes | | --- | --- | --- | | small-fixed | put / get | 4KiB, 100KiB | @@ -66,39 +42,19 @@ stage-breakdown object sizes): | large-stream | put / get | 10MiB, 16MiB, 32MiB | | mixed | mixed | 256KiB | -Concurrency ladder: `8, 16, 32, 64` (add `96, 128` on a bigger rig). Duration -`120s`, `--rounds >= 3`, cooldown `>= 30s`. - -Isolate background noise before the sweep: scanner deep-verify, heal, -replication, lifecycle transition, periodic capacity refresh — record whether -each is on rather than silently assuming it is off. +Concurrency ladder `8, 16, 32, 64` (add `96, 128` on a bigger rig); duration `120s`, `--rounds >= 3`, cooldown `>= 30s`. Record whether scanner deep-verify, heal, replication, lifecycle transition, and periodic capacity refresh are on rather than assuming they are off. ## 4. Measurement stack -The code already instruments every stage below. Drive each A/B cell with these -histograms (names verified against `crates/io-metrics/src/lib.rs`): +Drive each cell with the stage histograms emitted by `crates/io-metrics/src/lib.rs`; the stage and path label tokens are defined there and are not repeated here. -- PUT stages: `rustfs_s3_put_object_stage_duration_ms{stage=...}` — compute - P50/P95/P99 per stage. Stages: `app_bucket_validate`, `app_sse_config_lookup`, - `app_object_lock_config_lookup`, `app_put_opts_build`, `app_prelookup`, - `ingress_prepare`, `app_encryption_prepare`, `app_replication_decision`, - `app_store_put`, `app_post_store_bookkeeping`, `app_capacity_update`, - `set_disk_writer_setup`, `set_disk_encode`, `set_disk_rename`, - `set_disk_old_data_cleanup`. -- GET stages: `rustfs_io_get_object_stage_duration_seconds{path=..., stage=...}` — - the `path` label separates the read paths: `legacy_duplex`, `codec_streaming`, - `direct_memory`, `body_cache`, `inline_direct`, `internal_meta`, - `remote_transition`, `set_disk`, `empty`. Stages: `metadata`, - `metadata_cache_lookup`, `metadata_fanout`, `metadata_resolve`, `object_info`, - `path_decision`, `quorum_reached`, `range`, `reader_setup`, - `stripe_read`, `stripe_read_first_shard`, `stripe_read_quorum`, `decode`, - `reconstruct`, `emit`, `fill`, `output_poll`, `output_lock_wait`, - `bitrot_verify`, `first_byte`, `full_body`, `response_handoff`, - `lock_acquire`. -- EC memory pressure: `rustfs_ec_encode_inflight_bytes_current` and the - allocator reclaim gauge; plus node RSS and CPU. +| Metric | Labels | Use | +| --- | --- | --- | +| `rustfs_s3_put_object_stage_duration_ms` | `stage` | P50/P95/P99 per PUT stage (`app_*`, `ingress_prepare`, `set_disk_*`) | +| `rustfs_io_get_object_stage_duration_seconds` | `path`, `stage` | Per GET stage, split by read path (`legacy_duplex`, `codec_streaming`, ...) | +| `rustfs_ec_encode_inflight_bytes_current` | — | EC encode memory pressure; pair with node RSS and CPU | -Host telemetry (collect alongside every cell): +Host telemetry, collected alongside every cell: ```bash pidstat -durh 5 > telemetry/pidstat.txt & @@ -108,37 +64,34 @@ iostat -xz 5 > telemetry/iostat.txt & ## 5. Tuning knob catalog -Defaults are verified against `crates/config/src/constants/object.rs` and -`crates/ecstore/src/erasure/coding/encode.rs`. - ### 5.1 PUT | Knob | Default | Controls | Validating stage | Risk if widened | | --- | --- | --- | --- | --- | -| `RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES` | 32MiB | EC encode producer/consumer memory budget (blocks queued between encode and shard write) | `set_disk_encode` P95 + `rustfs_ec_encode_inflight_bytes_current` | RSS growth under high concurrency | -| `RUSTFS_OBJECT_IO_BUFFER_SIZE` | 128KiB | Streaming read-in / write-out block size | `ingress_prepare`, `set_disk_encode` | Larger buffers = fewer polls, more resident memory | -| `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE` | 4MiB | duplex pipe capacity (shared, but PUT path uses it less than GET) | `set_disk_encode` feed smoothness | Memory per in-flight request | -| `RUSTFS_DURABILITY_MODE` / `RUSTFS_DRIVE_SYNC_ENABLE` | mode-dependent | per-shard fsync/sync discipline on commit | `set_disk_rename` P99 | Weakening it changes the durability contract — treat as a deliberate tradeoff, not a free win | -| `RUSTFS_RUNTIME_WORKER_THREADS` / `RUSTFS_RUNTIME_MAX_BLOCKING_THREADS` | Tokio defaults | async workers + `spawn_blocking` pool feeding per-block encode | `set_disk_encode` P95 + mpstat | Oversubscription | +| `RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES` | 32MiB (`crates/ecstore/src/erasure/coding/encode.rs`) | EC encode producer/consumer memory budget (blocks queued between encode and shard write) | `set_disk_encode` P95 + `rustfs_ec_encode_inflight_bytes_current` | RSS growth under high concurrency | +| `RUSTFS_OBJECT_IO_BUFFER_SIZE` | 128KiB (`crates/config/src/constants/object.rs`) | Streaming read-in / write-out block size | `ingress_prepare`, `set_disk_encode` | Larger buffers = fewer polls, more resident memory | +| `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE` | 4MiB (`crates/config/src/constants/object.rs`) | Duplex pipe capacity (shared; PUT uses it less than GET) | `set_disk_encode` feed smoothness | Memory per in-flight request | +| `RUSTFS_DURABILITY_MODE` / `RUSTFS_DRIVE_SYNC_ENABLE` | mode-dependent | Per-shard fsync/sync discipline on commit | `set_disk_rename` P99 | Weakening it changes the durability contract — a deliberate tradeoff, never a free win | +| `RUSTFS_RUNTIME_WORKER_THREADS` / `RUSTFS_RUNTIME_MAX_BLOCKING_THREADS` | Tokio defaults | Async workers + `spawn_blocking` pool feeding per-block encode | `set_disk_encode` P95 + mpstat | Oversubscription | ### 5.2 GET | Knob | Default | Controls | Validating stage | Risk if enabled | | --- | --- | --- | --- | --- | -| `RUSTFS_GET_CODEC_STREAMING_ROLLOUT` | `off` | switches the read path from `legacy_duplex` to the pull-based `ErasureDecodeReader` (`codec_streaming`) | compare `path="legacy_duplex"` vs `path="codec_streaming"` for `decode`/`emit`/`output_lock_wait`/`stripe_read` | behavioral change to the read path; rollout is `off` by default for a reason | -| `RUSTFS_GET_CODEC_STREAMING_ENGINE` | `legacy` | `legacy` vs `rustfs` decode engine under the streaming reader | `reconstruct`/`decode` per `path` | engine swap on a correctness-critical path | -| `RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE` | `false` | multipart objects on the streaming reader | same, multipart cells | wider format coverage | -| `RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE` (+ `_MAX_SIZE`, `_FIRST_READER_SETUP`) | `false` / 512KiB | prefer data-shard readers before parity | `stripe_read_first_shard`/`stripe_read_quorum` | shard-selection order change | -| `RUSTFS_OBJECT_GET_SKIP_BITROT_VERIFY` | `false` | skip per-shard HighwayHash verify | `bitrot_verify` | **do not default on** — measures the theoretical ceiling only | -| `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE` | 4MiB | legacy GET in-process pipe capacity | `output_lock_wait`/`output_poll` | memory per in-flight GET | -| `RUSTFS_GET_SEEK_BUFFER_ENABLE` | `false` | in-memory seek buffer for small GET | `first_byte` | experimental, startup-latched — see [`get-path-experimental-switches.md`](get-path-experimental-switches.md) | -| `RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE` | `false` | adds `response_handoff` attribution (metrics only) | `response_handoff` | small per-request bookkeeping cost | +| `RUSTFS_GET_CODEC_STREAMING_ROLLOUT` | `off` (`crates/ecstore/src/set_disk/mod.rs`) | Switches the read path from `legacy_duplex` to the pull-based `ErasureDecodeReader` (`codec_streaming`) | `path="legacy_duplex"` vs `path="codec_streaming"` for `decode`/`emit`/`output_lock_wait`/`stripe_read` | Behavioral change to the read path | +| `RUSTFS_GET_CODEC_STREAMING_ENGINE` | `legacy` | `legacy` vs `rustfs` decode engine under the streaming reader | `reconstruct`/`decode` per `path` | Engine swap on a correctness-critical path | +| `RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE` | `false` | Multipart objects on the streaming reader | Same, multipart cells | Wider format coverage | +| `RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE` (+ `_MAX_SIZE`, `_FIRST_READER_SETUP`) | `false` / 512KiB | Prefer data-shard readers before parity | `stripe_read_first_shard`/`stripe_read_quorum` | Shard-selection order change | +| `RUSTFS_OBJECT_GET_SKIP_BITROT_VERIFY` | `false` | Skip per-shard HighwayHash verify | `bitrot_verify` | **Never default on** — measures the theoretical ceiling only | +| `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE` | 4MiB | Legacy GET in-process pipe capacity | `output_lock_wait`/`output_poll` | Memory per in-flight GET | +| `RUSTFS_GET_SEEK_BUFFER_ENABLE` [^latched] | `false` | Serves small GETs through an in-memory seek buffer (seek support without re-reading the object); gates `should_buffer_get_object_in_memory_with_threshold` | `first_byte` | Experimental; read by `scripts/run_get_1mib_abba_stage_metrics.sh` | +| `RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE` [^latched] | `false` | Attributes output-handoff timing to the `response_handoff` stage (metrics only) | `response_handoff` | Small per-request bookkeeping cost; set by `scripts/run_get_codec_streaming_smoke.sh` and `scripts/test_get_1mib_abba_stage_metrics.sh` | + +[^latched]: Both switches are startup-latched `OnceLock` booleans in `rustfs/src/app/object/get.rs` (`ENV_RUSTFS_GET_SEEK_BUFFER_ENABLE` / `is_get_seek_buffer_enabled`, `ENV_RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE` / `is_get_output_handoff_attribution_enabled`), read once via `get_env_bool(.., false)`; changing a value requires a process restart. They exist to support A/B runs, are referenced only by the harness scripts named above, and are kept deliberately — do not remove either switch or the seek-buffer path as dead code. Leave both unset in production. The codec-streaming switches above are startup-latched as well. ## 6. A/B matrix -Run each row as an independent cell. Baseline is the shipped default; candidate -is one knob moved. Everything else (topology, sizes, concurrency, rounds, -durability) stays fixed. +Run each row as an independent cell. Baseline is the shipped default; candidate is one knob moved. Everything else (topology, sizes, concurrency, rounds, durability) stays fixed. ### 6.1 PUT @@ -162,17 +115,12 @@ durability) stays fixed. | G5 | duplex buffer | 4MiB | 8MiB, 16MiB | `output_lock_wait`/`output_poll` (legacy path) | only if still on `legacy_duplex` | | G6 | skip bitrot verify | `false` | `true` | `bitrot_verify` | **ceiling measurement only**; do not carry into production | -## 7. Execution sequence +## 7. Execution and interpretation 1. Freeze the conditions in §3 and record the provenance block. -2. Run the baseline cell (all defaults) and capture stage histograms + host - telemetry. -3. Pick the **one** most-likely knob from the analysis. For large-object PUT - that is almost always `set_disk_encode` → P1; for GET it is G1 (the - `legacy_duplex` → `codec_streaming` switch). -4. Sweep that knob's candidate column one value at a time, same workload. -5. Read the decision table in §8; if the stage did not move, the knob is not - the bottleneck — stop widening it and pick the next stage. +2. Run the baseline cell (all defaults) and capture stage histograms plus host telemetry. +3. Pick the **one** most-likely knob from the decision table below (large-object PUT is almost always `set_disk_encode` → P1; GET is G1), sweep its candidate column one value at a time on the same workload, and stop widening as soon as the stage stops moving — the knob is then not the bottleneck. +4. Keep `baseline_compare.csv`, `median_summary.csv`, stage histograms, and host telemetry per cell; the conclusion must trace back to them. Record results in the issue tracker, not the repo. Driver invocation for one PUT cell: @@ -185,8 +133,6 @@ scripts/run_object_batch_bench_enhanced.sh \ --out-dir target/bench/put-tuning-p1-64mib ``` -## 8. Interpretation / decision table - | Stage high | Most likely cause | Next action | | --- | --- | --- | | `set_disk_encode` | per-block EC encode scheduling + in-flight budget | P1 → P4 → P2, in that order | @@ -198,21 +144,4 @@ scripts/run_object_batch_bench_enhanced.sh \ | `output_lock_wait` / `output_poll` | legacy duplex backpressure | G1 (move off duplex) or G5 | | `stripe_read*` | shard concurrency / selection | G4 shard-selection, disk/network tail | -## 9. Guardrails - -- **Never weaken correctness for throughput**: read/write quorum, bitrot verify, - `xl.meta` validation, and durability (`RUSTFS_DURABILITY_MODE`) are integrity - contracts, not knobs. P5 and G6 are ceiling measurements and must be labelled - as such; do not carry their values into production without an explicit - durability/correctness decision. -- **One variable per cell.** A cell that changes two knobs is thrown away. -- **Memory is part of the result.** A throughput win with unbounded RSS growth - is a regression; record RSS and the EC in-flight gauge for every PUT cell. -- **Startup-latched knobs** (`RUSTFS_GET_SEEK_BUFFER_ENABLE`, - `RUSTFS_GET_OUTPUT_HANDOFF_ATTRIBUTION_ENABLE`, and the codec-streaming - switches) require a process restart to change — see - [`get-path-experimental-switches.md`](get-path-experimental-switches.md). -- **Archive the raw data.** Keep the `baseline_compare.csv`, `median_summary.csv`, - stage histograms, and host telemetry per cell; the conclusion must trace back - to them. Do not commit benchmark result snapshots to the repo — record them in - the issue tracker. +Read/write quorum, bitrot verify, `xl.meta` validation, and durability are integrity contracts, not knobs: P5 and G6 are ceiling measurements and must be labelled as such. A throughput win with unbounded RSS growth is a regression — RSS and the EC in-flight gauge are part of every PUT result. diff --git a/docs/operations/oidc-console-integration.md b/docs/operations/oidc-console-integration.md new file mode 100644 index 000000000..d4aec6274 --- /dev/null +++ b/docs/operations/oidc-console-integration.md @@ -0,0 +1,217 @@ +# OIDC Console integration + +**Use this when:** connecting RustFS Console login to an OpenID Connect provider (Keycloak, Authing, or any standards-compliant IdP), or debugging an OIDC redirect, token, or policy-mapping failure. +**Source of truth:** `crates/config/src/constants/oidc.rs` (provider keys and `RUSTFS_IDENTITY_OPENID_*`), `crates/iam/src/oidc.rs` (discovery, PKCE, token validation, per-provider env suffixes), `rustfs/src/admin/handlers/oidc.rs` (authorize/callback handlers), `crates/config/src/constants/app.rs` (`ENV_RUSTFS_BROWSER_REDIRECT_URL`), `crates/utils/src/egress.rs` (`ENV_OUTBOUND_ALLOW_ORIGINS`), `crates/policy/src/policy/policy.rs` (built-in policies). + +The RustFS side is vendor-neutral and is described once; what RustFS requires from any provider is tabulated in [oidc-provider-requirements.md](oidc-provider-requirements.md). The [Keycloak](#keycloak) and [Authing](#authing) sections contain only IdP-side steps and vendor caveats. Examples use provider id `default` and public origin `https://rustfs.example.com`. + +## Integration model + +RustFS requires a standards-compliant OpenID Connect provider: discovery at `/.well-known/openid-configuration`, authorization and token endpoints, a JWKS, and an authorization-code flow that returns an `id_token`. RustFS never calls a vendor's authorization API; access is decided by RustFS IAM policies after claim mapping. Protocol requirements for IdP vendors are collected in [oidc-provider-requirements.md](oidc-provider-requirements.md). + +Login flow: + +1. The browser opens `https://rustfs.example.com/rustfs/admin/v3/oidc/authorize/`. +2. RustFS creates `state`, `nonce`, and a PKCE S256 challenge and redirects to the IdP. +3. The IdP redirects back to `/rustfs/admin/v3/oidc/callback/?code=...&state=...`. +4. RustFS exchanges the code at the token endpoint, sending `client_id` and `client_secret` in the request body (`client_secret_post`) together with the PKCE verifier. +5. RustFS validates the ID token signature (JWKS), issuer, audience, expiry, and nonce. +6. RustFS maps claim values to policy names and issues one-hour STS credentials to the Console. + +In-flight `state` and PKCE verifiers are node-local: the authorize and callback requests must reach the same RustFS node. + +## Configuration keys + +Every provider key can be set as `RUSTFS_IDENTITY_OPENID_` in the process environment or as `identity_openid` `=` through `mc admin config set`. Names are constants in `crates/config/src/constants/oidc.rs`. + +| Provider key | Environment variable | Purpose | +| --- | --- | --- | +| `enable` | `RUSTFS_IDENTITY_OPENID_ENABLE` | `on` loads the provider. | +| `config_url` | `RUSTFS_IDENTITY_OPENID_CONFIG_URL` | Issuer URL used for discovery. A trailing `/.well-known/openid-configuration` is stripped; any other `.well-known` path is rejected. | +| `issuer` | `RUSTFS_IDENTITY_OPENID_ISSUER` | Expected `iss` when it differs from `config_url` (internal discovery URL, public token issuer). | +| `client_id`, `client_secret` | `RUSTFS_IDENTITY_OPENID_CLIENT_ID`, `RUSTFS_IDENTITY_OPENID_CLIENT_SECRET` | Confidential client credentials. | +| `scopes` | `RUSTFS_IDENTITY_OPENID_SCOPES` | Comma-separated; `openid` is required. | +| `other_audiences` | `RUSTFS_IDENTITY_OPENID_OTHER_AUDIENCES` | Additional accepted `aud` values. | +| `redirect_uri` | `RUSTFS_IDENTITY_OPENID_REDIRECT_URI` | Callback URL sent to the IdP; must equal the URL registered there. | +| `redirect_uri_dynamic` | `RUSTFS_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC` | `on` derives the callback from request headers. Keep `off` behind proxies. | +| `claim_name`, `claim_prefix` | `RUSTFS_IDENTITY_OPENID_CLAIM_NAME`, `RUSTFS_IDENTITY_OPENID_CLAIM_PREFIX` | Policy claim name and a fixed string prepended to each value. `claim_prefix` is not a mapping table. | +| `groups_claim`, `roles_claim` | `RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM`, `RUSTFS_IDENTITY_OPENID_ROLES_CLAIM` | Flat top-level array claims whose values are RustFS policy names. | +| `email_claim`, `username_claim` | `RUSTFS_IDENTITY_OPENID_EMAIL_CLAIM`, `RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM` | Identity claims shown in the Console. | +| `role_policy` | `RUSTFS_IDENTITY_OPENID_ROLE_POLICY` | One fixed policy for every login from this provider. Connectivity testing only. | +| `display_name` | `RUSTFS_IDENTITY_OPENID_DISPLAY_NAME` | Login button label. | +| `hide_from_ui` | `RUSTFS_IDENTITY_OPENID_HIDE_FROM_UI` | Hides the provider from `/oidc/providers`. | + +Process-level settings (environment only, never suffixed per provider): + +| Variable | Purpose | +| --- | --- | +| `RUSTFS_BROWSER_REDIRECT_URL` | Public browser origin used for callback generation, Console success redirects, and logout fallback. | +| `RUSTFS_OUTBOUND_ALLOW_ORIGINS` | Exact `scheme://host[:port]` origins RustFS may contact for discovery, JWKS, and token requests when the IdP resolves to a private, loopback, or container-network address. See [outbound-connection-policy.md](outbound-connection-policy.md). | + +Named providers: to use provider id ``, suffix every provider env var with `_` (for example `RUSTFS_IDENTITY_OPENID_CLIENT_ID_keycloak`) and register the callback `/rustfs/admin/v3/oidc/callback/`. Suffix scanning is `parse_single_provider` in `crates/iam/src/oidc.rs`. + +Restart RustFS after changing any of these settings. + +### Environment example + +```bash +export RUSTFS_BROWSER_REDIRECT_URL="https://rustfs.example.com" + +export RUSTFS_IDENTITY_OPENID_ENABLE=on +export RUSTFS_IDENTITY_OPENID_CONFIG_URL="" +export RUSTFS_IDENTITY_OPENID_CLIENT_ID="" +export RUSTFS_IDENTITY_OPENID_CLIENT_SECRET="" +export RUSTFS_IDENTITY_OPENID_SCOPES="openid,profile,email" +export RUSTFS_IDENTITY_OPENID_REDIRECT_URI="https://rustfs.example.com/rustfs/admin/v3/oidc/callback/default" +export RUSTFS_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC=off +export RUSTFS_IDENTITY_OPENID_DISPLAY_NAME="" +export RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM="groups" +export RUSTFS_IDENTITY_OPENID_ROLES_CLAIM="roles" +export RUSTFS_IDENTITY_OPENID_EMAIL_CLAIM="email" +export RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM="preferred_username" +``` + +The same keys through admin config: + +```bash +mc admin config set rustfs identity_openid \ + enable=on config_url="" client_id="" client_secret="" \ + scopes="openid,profile,email" \ + redirect_uri="https://rustfs.example.com/rustfs/admin/v3/oidc/callback/default" \ + redirect_uri_dynamic=off display_name="" \ + groups_claim="groups" roles_claim="roles" email_claim="email" username_claim="preferred_username" +mc admin service restart rustfs +``` + +`RUSTFS_BROWSER_REDIRECT_URL` is not an `identity_openid` key; it must still be set in the process environment. + +## Redirect URL priority + +1. Provider `redirect_uri`, when set, is the callback URL sent to the IdP. +2. `RUSTFS_BROWSER_REDIRECT_URL`, when set, is the public origin for callback generation when no provider `redirect_uri` exists, and for Console success and logout fallback redirects. +3. Request headers (`Host`, `X-Forwarded-Proto`) are used only when `redirect_uri_dynamic=on` and no browser redirect URL is configured. + +Behind a reverse proxy or load balancer, set `RUSTFS_BROWSER_REDIRECT_URL` and keep session affinity for the authorize and callback requests. + +## Policy mapping + +Claim values are used verbatim as policy names (after `claim_prefix`, if any). Names must satisfy `is_safe_claim_policy_name` in `crates/iam/src/sys.rs`: ASCII letters, digits, `_`, `-`, `:`, `.` only, so a value containing `/` (for example Keycloak's full group path `/consoleAdmin`) never matches. Built-in policies: + +| Policy | Grants | +| --- | --- | +| `consoleAdmin` | Full Console, admin, KMS, and S3 access. | +| `readwrite` | S3 read/write. | +| `readonly` | S3 read-only. | +| `writeonly` | S3 write-only. | +| `diagnostics` | Diagnostic admin access. | + +For first-contact testing only, `RUSTFS_IDENTITY_OPENID_ROLE_POLICY=consoleAdmin` grants every login full access; remove it before production. + +## Validation + +1. Discovery: + +```bash +curl -fsS "/.well-known/openid-configuration" | jq '{issuer, authorization_endpoint, token_endpoint, jwks_uri, code_challenge_methods_supported, token_endpoint_auth_methods_supported, scopes_supported}' +``` + + `issuer` must equal `RUSTFS_IDENTITY_OPENID_ISSUER` when set, otherwise the issuer derived from `RUSTFS_IDENTITY_OPENID_CONFIG_URL`; `code_challenge_methods_supported` must include `S256`; `token_endpoint_auth_methods_supported` must include `client_secret_post`; `scopes_supported` must include every configured scope. + +2. Provider visibility: `curl -fsS https://rustfs.example.com/rustfs/admin/v3/oidc/providers | jq` lists the provider unless `hide_from_ui=on`. + +3. Browser login: open `https://rustfs.example.com/rustfs/admin/v3/oidc/authorize/default`. Expect a redirect to the IdP, sign-in, a redirect to `/rustfs/admin/v3/oidc/callback/default?code=...&state=...`, and then the Console with the mapped permissions. + +4. ID token claims (decode the token after a test login): `iss` matches the issuer, `aud` includes the client id, `email` and `preferred_username` are present when configured, `groups` or `roles` is a flat array of policy names. + +## Troubleshooting + +| Symptom | Common cause | Fix | +| --- | --- | --- | +| `/oidc/providers` does not list the provider | provider failed to load, or RustFS was not restarted | Check env/admin config and restart RustFS. | +| Provider or login button missing; startup logs `OIDC provider discovery blocked by outbound policy` | IdP origin is private/internal and not allowlisted | Add the exact origin to `RUSTFS_OUTBOUND_ALLOW_ORIGINS` on every node and restart. | +| IdP reports a redirect mismatch (`invalid redirect_uri`) | registered callback differs from RustFS `redirect_uri` | Use the exact `/rustfs/admin/v3/oidc/callback/` URL on both sides. | +| Callback reports missing `code` or `state` | proxy dropped the query string | Preserve the full callback URL and query string. | +| Token exchange fails | wrong secret, or the IdP rejects request-body client authentication | Confirm the client is confidential and accepts `client_secret_post`. | +| No `id_token` in the token response | `openid` scope missing or a non-OIDC OAuth flow | Add `openid`; use the authorization-code flow. | +| ID token verification fails | issuer, audience, algorithm, or JWKS mismatch | Compare discovery metadata with `CONFIG_URL`/`ISSUER`/`CLIENT_ID`; prefer `RS256`. | +| Login succeeds, access denied | no claim value matches a policy name | Emit `groups` or `roles` as a flat array equal to policy names; check for `/` prefixes. | +| Console redirects to an internal host | `RUSTFS_BROWSER_REDIRECT_URL` unset or proxy headers wrong | Set `RUSTFS_BROWSER_REDIRECT_URL` to the public origin. | +| Invalid or expired OIDC state | callback reached a different node | Configure load-balancer session affinity for authorize and callback. | + +## Production checklist + +- [ ] RustFS and the IdP use HTTPS. +- [ ] The IdP registers the exact callback URL (no wildcard) and `RUSTFS_IDENTITY_OPENID_REDIRECT_URI` matches it. +- [ ] `RUSTFS_BROWSER_REDIRECT_URL` is the public browser origin. +- [ ] PKCE S256 is allowed or required at the IdP. +- [ ] ID tokens carry `groups` or `roles` values equal to RustFS policy names. +- [ ] `role_policy` is not used as a permanent shortcut. +- [ ] The load balancer preserves query strings and pins authorize/callback to one node. +- [ ] Internal IdP origins are listed exactly in `RUSTFS_OUTBOUND_ALLOW_ORIGINS` on every node. + +## Keycloak + +| Value | Example | +| --- | --- | +| Realm | `rustfs` | +| Issuer (`config_url`) | `https://keycloak.example.com/realms/rustfs` | +| Discovery URL | `https://keycloak.example.com/realms/rustfs/.well-known/openid-configuration` | +| Client id | `rustfs-console` | +| Scopes | `openid,profile,email` | +| Groups claim | `groups` (flat array) | + +Client setup in the Keycloak Admin Console: + +1. Create or select the realm and confirm discovery returns `issuer` equal to `https://keycloak.example.com/realms/rustfs`. +2. `Clients` → create: `Client type` = `OpenID Connect`, `Client ID` = `rustfs-console`. +3. Enable `Client authentication` and `Standard flow`; disable `Implicit flow`, `Direct access grants`, and `Service accounts roles`. +4. `Valid redirect URIs` = `https://rustfs.example.com/rustfs/admin/v3/oidc/callback/default`; `Web origins` = `https://rustfs.example.com`. +5. `Proof Key for Code Exchange Code Challenge Method` = `S256`. +6. Save and copy the secret from `Credentials`. Do not apply a client policy that disables `client_secret_post`. + +Group mapper (a `Group Membership` mapper in the client's dedicated scope): + +| Mapper field | Value | +| --- | --- | +| Name | `rustfs-groups` | +| Token Claim Name | `groups` | +| Full group path | `Off` (a leading `/` breaks policy matching) | +| Add to ID token / access token / userinfo | `On` | +| Multivalued | `On` | + +Create Keycloak groups named after RustFS policies (`consoleAdmin`, `readonly`, ...) and add users to them. + +Roles instead of groups: assign realm or client roles named after policies, add a `User Realm Role` or `User Client Role` mapper that emits a flat top-level `roles` claim, and set `RUSTFS_IDENTITY_OPENID_ROLES_CLAIM=roles`. RustFS does not read Keycloak's nested `realm_access.roles` claim. + +Internal discovery URL with a public issuer (for example in-cluster Keycloak on Kubernetes): + +```bash +export RUSTFS_IDENTITY_OPENID_CONFIG_URL="http://keycloak.keycloak.svc.cluster.local:8080/realms/rustfs" +export RUSTFS_IDENTITY_OPENID_ISSUER="https://keycloak.example.com/realms/rustfs" +export RUSTFS_OUTBOUND_ALLOW_ORIGINS="http://keycloak.keycloak.svc.cluster.local:8080" +``` + +Discovery and issuer-relative JWKS requests use the `CONFIG_URL` base; `iss` validation uses `ISSUER`. The allowlist entry is the origin only (no realm or discovery path) and is read at startup on every node. Prefer HTTPS with a trusted CA for the internal URL: discovery and JWKS define the token-signing trust root, so plain HTTP is acceptable only where DNS and traffic cannot be tampered with. + +## Authing + +| Value | Example | Note | +| --- | --- | --- | +| Application domain | `https://example.authing.cn` | From the Authing application page. | +| Issuer (`config_url`) | `https://example.authing.cn/oidc` | Tenants differ (`/oidc`, `/oauth/oidc`): copy the issuer from the console and confirm discovery returns the same `issuer`. | +| App ID / App Secret | `` / `` | RustFS `client_id` / `client_secret`. | +| Scopes | `openid,profile,email,roles` | `roles` is needed when Authing emits role claims. | +| Roles claim | `roles` | Set `RUSTFS_IDENTITY_OPENID_ROLES_CLAIM=roles`. | + +Application settings in the Authing console: + +| Setting | Value | +| --- | --- | +| Protocol | OpenID Connect | +| Grant type / response type | Authorization Code / `code` | +| Token endpoint authentication | `client_secret_post` | +| PKCE | allow or require `S256` | +| ID token signing algorithm | `RS256` | +| Redirect URL | `https://rustfs.example.com/rustfs/admin/v3/oidc/callback/default` (exact) | + +Assign Authing roles whose names equal RustFS policy names; a test user with role `consoleAdmin` should produce `"roles": ["consoleAdmin"]` in the ID token. `claim_prefix` only prepends a fixed string, so keep role values equal to policy names unless policies with that prefix already exist. diff --git a/docs/operations/oidc-provider-requirements.md b/docs/operations/oidc-provider-requirements.md new file mode 100644 index 000000000..11653fc04 --- /dev/null +++ b/docs/operations/oidc-provider-requirements.md @@ -0,0 +1,26 @@ +# OIDC provider requirements + +**Use this when:** evaluating whether an identity provider (or a vendor's "OIDC-like" OAuth product) can back RustFS console SSO, or debugging why a provider fails discovery, login, or policy mapping. +**Source of truth:** `crates/iam/src/oidc.rs` (`OidcProviderConfig`, `discover_provider`, `trusted_aud`, `map_claims_to_policies`); `crates/config/src/constants/oidc.rs` (`OIDC_DEFAULT_*`, `ENV_IDENTITY_OPENID_*`); `rustfs/src/admin/handlers/oidc.rs` (`derive_callback_uri_with_provider_config`, `browser_redirect_url`). + +RustFS is a standard OpenID Connect relying party using the authorization-code flow with PKCE. It reads every claim it uses from the ID token; it never calls the UserInfo endpoint. Provider-side setup steps for Keycloak and Authing, and the RustFS-side configuration keys, are in [oidc-console-integration.md](oidc-console-integration.md). + +## Requirements + +| # | Requirement | Details | Code anchor | +| --- | --- | --- | --- | +| 1 | Discovery document | `RUSTFS_IDENTITY_OPENID_CONFIG_URL` names the provider (issuer base or full discovery URL); RustFS fetches `{issuer}/.well-known/openid-configuration` and needs `issuer`, `authorization_endpoint`, `token_endpoint`, `jwks_uri`, and the standard `*_supported` arrays. When `RUSTFS_IDENTITY_OPENID_ISSUER` is set, the document's `issuer` must equal it exactly; otherwise RustFS tries the issuer candidates derived from the config URL. | `crates/iam/src/oidc.rs` `discover_provider`, `discover_provider_from_config_url` | +| 2 | Outbound reachability | Discovery, JWKS, and token requests go through the shared egress policy. A provider on a private or loopback address needs its exact origin in `RUSTFS_OUTBOUND_ALLOW_ORIGINS`; otherwise startup logs `OIDC provider discovery blocked by outbound policy`. | `crates/iam/src/oidc.rs` `OIDC_DISCOVERY_BLOCKED_BY_OUTBOUND_POLICY`; [Outbound connection policy](outbound-connection-policy.md) | +| 3 | Signed ID token, verifiable via JWKS | The ID token signature is verified against `jwks_uri`; the key set is refreshed after `OIDC_JWKS_REFRESH_INTERVAL` and once more on a verification failure. A token response without `id_token` fails login — an OAuth-only access token is not sufficient. | `crates/iam/src/oidc.rs` `OIDC_JWKS_REFRESH_INTERVAL`, the `no id_token in token response` error | +| 4 | Authorization-code flow with PKCE (S256) | The authorization request carries `response_type=code`, `scope`, `redirect_uri`, `state`, `nonce`, `code_challenge`, `code_challenge_method=S256`; the token request carries the `code_verifier`. Providers that ignore or reject PKCE, `state`, or `nonce` are not supported. | `crates/iam/src/oidc.rs` `PkceCodeChallenge::new_random_sha256`; `crates/iam/src/oidc_state.rs` `pkce_verifier` | +| 5 | Scopes | Default `openid,profile,email` (`OIDC_DEFAULT_SCOPES`); override with `RUSTFS_IDENTITY_OPENID_SCOPES` (comma-separated). The provider must accept every configured scope. | `crates/config/src/constants/oidc.rs` `OIDC_DEFAULT_SCOPES`; `crates/iam/src/oidc.rs` `OidcProviderConfig::scopes` | +| 6 | Callback returns `state` | The callback must carry both `code` and the original `state`; `state` locates the in-flight session that holds the nonce and PKCE verifier. A callback with only `code` fails. | `rustfs/src/admin/handlers/oidc.rs` `OIDC_CALLBACK_SUFFIX`; `crates/iam/src/oidc_state.rs` | +| 7 | ID token claims | `iss`, `aud`, `exp`, and `nonce` are verified; `sub` identifies the user. `aud` must contain the client id or one of `RUSTFS_IDENTITY_OPENID_OTHER_AUDIENCES`. | `crates/iam/src/oidc.rs` `trusted_aud`, `OidcClaims` | +| 8 | Authorization claims in the ID token | Policy mapping reads the claim named by `RUSTFS_IDENTITY_OPENID_CLAIM_NAME` (default `groups`, `OIDC_DEFAULT_CLAIM_NAME`) together with `RUSTFS_IDENTITY_OPENID_GROUPS_CLAIM` and the optional `RUSTFS_IDENTITY_OPENID_ROLES_CLAIM`, as a string or array of strings, optionally prefixed by `RUSTFS_IDENTITY_OPENID_CLAIM_PREFIX`. Values must equal RustFS policy names (for example `consoleAdmin`, `readwrite`, `readonly`). Email and username come from `RUSTFS_IDENTITY_OPENID_EMAIL_CLAIM` (default `email`) and `RUSTFS_IDENTITY_OPENID_USERNAME_CLAIM` (default `preferred_username`). A provider that returns only a user id can authenticate but cannot express RustFS authorization. | `crates/iam/src/oidc.rs` `map_claims_to_policies`, `extract_canonical_group_values` | +| 9 | Registered redirect URI | The provider must accept the callback `{public-origin}/rustfs/admin/v3/oidc/callback/{provider_id}`. RustFS picks the origin in this order: the provider's `redirect_uri` (`RUSTFS_IDENTITY_OPENID_REDIRECT_URI`), then `RUSTFS_BROWSER_REDIRECT_URL`, then the request's own scheme and host — the last only when `RUSTFS_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC` is enabled. | `rustfs/src/admin/handlers/oidc.rs` `derive_callback_uri_with_provider_config`, `browser_redirect_url` | +| 10 | Logout endpoint (optional) | When discovery advertises `end_session_endpoint`, RustFS builds an RP-initiated logout URL with `id_token_hint`, `client_id`, and `post_logout_redirect_uri`. Without it, logout falls back to the console login page. | `crates/iam/src/oidc.rs` `build_logout_url` (reads `end_session_endpoint` from `ProviderMetadataWithLogout`) | + +## Deployment notes + +- Behind a load balancer, authorize and callback requests must reach the same RustFS node while the `state` is in flight, or set `RUSTFS_BROWSER_REDIRECT_URL` so the callback URL is stable; the callback error text names both remedies. +- Provider-specific setup (client registration, claim mappers, redirect-URL priority) is covered by the console integration guide for the provider in use; this page states only what any provider must offer. diff --git a/docs/operations/openid-compatibility-issues-for-vendor.md b/docs/operations/openid-compatibility-issues-for-vendor.md deleted file mode 100644 index f184dd67d..000000000 --- a/docs/operations/openid-compatibility-issues-for-vendor.md +++ /dev/null @@ -1,134 +0,0 @@ -# OIDC Vendor Compatibility Checklist - -Use this checklist when a vendor provides an OAuth or SSO document that is described as OIDC but does not clearly expose the standard OpenID Connect contract required by RustFS. - -## 1. Discovery Metadata - -RustFS expects provider metadata at: - -```text -GET {issuer}/.well-known/openid-configuration -``` - -Ask the vendor to provide the discovery URL and confirm that it returns at least: - -- `issuer` -- `authorization_endpoint` -- `token_endpoint` -- `jwks_uri` -- `response_types_supported` -- `subject_types_supported` -- `id_token_signing_alg_values_supported` - -The returned `issuer` must exactly match the issuer configured in RustFS. - -## 2. JWKS and Token Signature Verification - -RustFS must verify the ID token signature. Ask the vendor to provide: - -- `jwks_uri` -- supported signing algorithms, such as `RS256` -- key rotation behavior -- how the token `kid` maps to the JWKS key set - -Without a verifiable ID token signature, the provider is not suitable for RustFS OIDC login. - -## 3. Authorization Request Parameters - -The provider must accept the standard authorization-code request parameters: - -- `scope=openid profile email` -- `response_type=code` -- `client_id` -- `redirect_uri` -- `state` -- `nonce` -- `code_challenge` -- `code_challenge_method=S256` - -If the vendor example omits `state`, `nonce`, or PKCE, confirm whether those parameters are supported. - -## 4. Callback State - -The provider must return the original `state` value in the callback: - -```text -...?code=xxx&state=yyy -``` - -RustFS uses `state` for CSRF protection and to find the in-flight OIDC session. A callback that only returns `code` is not enough. - -## 5. Token Response - -The token endpoint response must be JSON and include at least: - -- `access_token` -- `token_type`, usually `Bearer` -- `expires_in` -- `id_token` - -RustFS requires `id_token`; an OAuth-only access token is not sufficient for Console OIDC login. - -## 6. ID Token Claims - -The ID token must contain standard claims that RustFS can verify: - -- `iss` -- `sub` -- `aud` -- `exp` -- `iat` -- `nonce` when the authorization request includes `nonce` - -Ask the vendor for a sample ID token payload and claim documentation. - -## 7. UserInfo Endpoint - -Standard OIDC UserInfo normally uses: - -```text -GET /userinfo -Authorization: Bearer -``` - -If the vendor only documents a private profile endpoint such as `/oidc/profile?access_token=...`, ask whether a standard `userinfo_endpoint` is available and returned in discovery. - -## 8. Logout Endpoint - -Standard RP-initiated logout is normally exposed through an `end_session_endpoint` in discovery. If the vendor only documents a private token removal endpoint, ask whether standard OIDC logout is available. - -RustFS can still fall back to the Console login page when the provider does not advertise an end-session endpoint. - -## 9. Authorization Claims - -OIDC primarily authenticates the user. RustFS authorization is still based on RustFS policies. The provider must emit claims that can be mapped to RustFS policies, for example: - -- `groups` -- `roles` -- `policy` -- another agreed flat array or string claim - -Ask the vendor to confirm: - -- whether group, role, or policy claims can be included in the ID token -- whether those claims can be included in UserInfo -- the exact claim names and value formats -- whether the claim values can match RustFS policy names such as `consoleAdmin`, `readwrite`, or `readonly` - -If the provider only returns a user id or token validity result, it can authenticate the user but cannot by itself express RustFS authorization. - -## 10. RustFS Redirect Requirements - -RustFS browser-facing redirect behavior depends on these values: - -- provider `redirect_uri`, when explicitly configured, is the callback URL sent to the provider -- `RUSTFS_BROWSER_REDIRECT_URL` is the public RustFS browser origin used for callback generation when no provider `redirect_uri` exists, and for Console success and logout fallback redirects -- dynamic request-header redirects are used only when no configured redirect source exists and dynamic redirects are enabled - -Ask the vendor to register the exact callback URL, for example: - -```text -https://rustfs.example.com/rustfs/admin/v3/oidc/callback/default -``` - -For load-balanced RustFS deployments, ensure authorize and callback requests reach the same RustFS node while the OIDC `state` is in flight. diff --git a/docs/operations/outbound-connection-policy.md b/docs/operations/outbound-connection-policy.md index 4494ba1ee..968b8a87e 100644 --- a/docs/operations/outbound-connection-policy.md +++ b/docs/operations/outbound-connection-policy.md @@ -1,120 +1,62 @@ # Outbound Connection Policy -This document describes the outbound connection policy that RustFS applies to -server-initiated HTTP(S) requests, and the `RUSTFS_OUTBOUND_ALLOW_ORIGINS` -allowlist operators can use to reach endpoints on private or container networks. +**Use this when:** a webhook, audit target, OIDC provider, or object-lambda endpoint on a private or container network (Compose service names, `host.docker.internal`, RFC 1918 addresses) is not being reached, or you need to know which server-initiated connections RustFS restricts and how to allowlist one. +**Source of truth:** `crates/utils/src/egress.rs` (`OutboundPolicy`, `OutboundDnsResolver`, `validate_outbound_url`, `ENV_OUTBOUND_ALLOW_ORIGINS`). -It is written for operators whose outbound integrations stopped reaching -endpoints after an upgrade — typically Docker Compose service names, -`host.docker.internal`, or RFC 1918 addresses. Webhook and audit clients adopted -this policy in `1.0.0-beta.11`; OIDC provider requests adopted it in -`1.0.0-beta.12`. +RustFS validates every operator-configured outbound destination to close a server-side request forgery (SSRF) class. Two layers exist: -## Background: what the policy protects +| Layer | What it checks | Escape hatch | +| --- | --- | --- | +| Literal URL check (`validate_outbound_url`) | Scheme is `http`/`https`; the host is not `localhost` or a loopback, private, shared, reserved, link-local, unspecified, or metadata address (IPv4-mapped and embedded IPv6 forms are classified by the embedded IPv4) | None | +| Full policy (`OutboundPolicy` + `OutboundDnsResolver`) | The literal check, plus re-validation of every address DNS returns on each new connection, so a hostname cannot be rebound to a restricted address after it was accepted | `RUSTFS_OUTBOUND_ALLOW_ORIGINS` for the loopback, private, shared, and reserved classes | -Several RustFS subsystems open connections to operator-configured URLs. To close -a server-side request forgery (SSRF) class of problem, RustFS validates every such -destination and re-checks the addresses returned by DNS on each new connection, -so a hostname cannot be rebound to a restricted address after it is first -accepted. +## Which subsystem uses which layer -The policy governs the outbound clients used by: +| Subsystem | Layer | Notes | +| --- | --- | --- | +| Event-notification webhooks (`RUSTFS_NOTIFY_WEBHOOK_*`) and audit webhooks (`RUSTFS_AUDIT_WEBHOOK_*`) | Full policy | Proxies disabled and redirects not followed, so the endpoint must be reachable directly (`crates/targets/src/target/webhook.rs`) | +| Target configuration validation (startup and admin API) | Full policy | `crates/targets/src/config/common.rs` `validate_outbound_http_url`; `rustfs/src/admin/handlers/target_descriptor.rs` | +| OIDC discovery, JWKS, and token requests | Full policy | A blocked provider logs `OIDC provider discovery blocked by outbound policy` naming the origin to allowlist (`crates/iam/src/oidc.rs`) | +| Object Lambda targets | Full policy | `rustfs/src/admin/router.rs` `outbound_policy` | +| Bucket replication targets | Literal check, relaxed | Private addresses are always allowed; loopback only with `RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET=true` (`crates/ecstore/src/bucket/bucket_target_sys.rs` `validate_replication_target_endpoint`) | +| Site replication peers | Literal check | `rustfs/src/site_replication/mod.rs` | +| Tiering warm backends (S3, MinIO, RustFS, Azure, GCS, Aliyun, Tencent, Huawei, R2) | Literal check | `crates/ecstore/src/services/tier/warm_backend.rs` `validate_endpoint`; the RustFS provider adds a debug-only, env-gated loopback exception for e2e tests | +| Keystone `auth_url` | Literal check | `crates/keystone/src/config.rs` | -- event-notification webhooks (`RUSTFS_NOTIFY_WEBHOOK_*`); -- audit webhooks (`RUSTFS_AUDIT_WEBHOOK_*`); -- OIDC identity-provider discovery, JWKS, and token requests (since `1.0.0-beta.12`); -- S3 tiering (warm-backend) endpoints; -- Keystone auth URLs. - -The webhook and audit outbound clients also **disable proxies and do not follow -redirects**, so the destination must be reachable directly at the configured URL. - -## What changed in beta.11 (and for OIDC in beta.12) - -For webhook and audit clients: - -| | beta.10 | beta.11+ | -|---|---|---| -| Literal `localhost` / private / loopback IPs | Rejected | Rejected | -| Hostnames that resolve to private/loopback addresses (`logstash`, `host.docker.internal`, Compose service DNS, …) | Allowed | **Blocked at DNS/connect time** unless allowlisted | -| Escape hatch for private destinations | None | `RUSTFS_OUTBOUND_ALLOW_ORIGINS` | -| Proxies / redirects for outbound clients | Followed | Disabled | - -Before beta.11 a webhook endpoint whose hostname happened to resolve to a -private address was accepted. Beta.11 fails that resolution check unless the -exact origin is on the allowlist. This is why a Compose setup that delivered -events on beta.10 can go silent after the upgrade even though the configuration -is unchanged. - -OIDC joined the same policy in beta.12. An internal identity provider that -worked in beta.11 can therefore fail discovery after upgrading to beta.12 unless -its exact origin is allowlisted. The policy remains active for discovery, JWKS, -and token requests. +The allowlist affects only the "Full policy" rows. A literal-check subsystem rejects a hostname that is itself a restricted IP literal, does not re-check what a hostname resolves to, and cannot be widened by `RUSTFS_OUTBOUND_ALLOW_ORIGINS`. ## Symptoms -- Bucket event rules and webhook configuration look correct. -- Uploads and audited API calls succeed. -- No HTTP POST reaches the internal webhook receiver. -- The target may appear offline or fail activation when its endpoint resolves to -a loopback, private, shared, or reserved address. -- Startup or target validation reports `webhook endpoint is not allowed: ...` - with a reason such as `private address` or `loopback host`. -- An OIDC provider or login button is missing, and startup reports - `OIDC provider discovery blocked by outbound policy` with the exact origin to - allowlist. +- Bucket event rules and webhook configuration look correct and uploads succeed, but no POST reaches the receiver. +- Target validation reports ` is not allowed: ...` with a reason such as `private address` or `loopback host`; when an exact-origin allowlist entry would fix it, the message says so. +- An OIDC login button is missing and startup logs `OIDC provider discovery blocked by outbound policy`. ## `RUSTFS_OUTBOUND_ALLOW_ORIGINS` -`RUSTFS_OUTBOUND_ALLOW_ORIGINS` is a comma-separated list of exact HTTP(S) -origins that are permitted to resolve to otherwise-restricted addresses. It is an -operator-owned process setting read once at startup; individual target -configuration cannot extend it. +A comma-separated list of exact HTTP(S) origins permitted to resolve to otherwise-restricted addresses. It is a process-level setting read once at startup; individual target configuration cannot extend it. ```bash # exact scheme://host:port — comma-separate multiple origins RUSTFS_OUTBOUND_ALLOW_ORIGINS=http://logstash:8080,http://host.docker.internal:3020 ``` -### Origin format rules - -Each entry is matched as an **exact origin** (`scheme://host:port`): - -- The scheme must be `http` or `https`. -- The host and port must match the destination exactly. An allowlisted - `http://logstash:8080` does **not** authorize `http://logstash:9090` or - `https://logstash:8080`. -- If the port is omitted, the scheme's default is used (`80` for `http`, `443` - for `https`); the destination must then use that same default port. -- Entries must be origins only. A trailing `/` is accepted, but a path, query, - or fragment (for example `http://logstash:8080/events`) is **rejected** as an - invalid origin — the process fails closed rather than silently ignoring the - path. -- Userinfo (`http://user:pass@host`) is not allowed. -- An empty entry (for example a trailing or doubled comma) is rejected. - -An invalid list fails closed: the affected subsystem reports an -`invalid outbound policy` / `invalid origin at position N` error instead of -starting with a partially applied allowlist. +| Rule | Detail | +| --- | --- | +| Exact origin | `scheme://host:port`. `http://logstash:8080` does not authorize `http://logstash:9090` or `https://logstash:8080` | +| Scheme | `http` or `https` only | +| Default port | If omitted, the scheme default (`80` / `443`) applies and the destination must use that port | +| Origin only | A trailing `/` is accepted; any path, query, or fragment (`http://logstash:8080/events`) is rejected | +| No userinfo | `http://user:pass@host` is rejected | +| No empty entries | A trailing or doubled comma is rejected | +| Fail closed | An invalid list yields `invalid outbound policy` / `invalid origin at position N` and the affected subsystem does not start with a partially applied allowlist | ### What stays blocked even when allowlisted -Allowlisting an origin only relaxes the loopback, private, shared, and reserved -address classes for that exact origin. The following remain forbidden for every -origin, allowlisted or not: +- Cloud metadata endpoints (`169.254.169.254` and the other well-known IMDS addresses). +- Link-local addresses (`169.254.0.0/16`, `fe80::/10`) and the unspecified address (`0.0.0.0`, `::`). +- IPv4-mapped, IPv4-compatible, and NAT64/6to4-embedded forms of the above; the embedded IPv4 address is what gets classified, so `::ffff:127.0.0.1` cannot bypass the policy. -- cloud metadata endpoints (for example `169.254.169.254` and the other - well-known IMDS addresses); -- link-local addresses (`169.254.0.0/16`, `fe80::/10`); -- the unspecified address (`0.0.0.0`, `::`); -- IPv4-mapped, IPv4-compatible, and NAT64/6to4-embedded forms of any of the - above (RustFS classifies the embedded IPv4 destination, so `::ffff:127.0.0.1` - and similar cannot be used to bypass the policy). - -The allowlist authorizes only the exact host you name. A DNS answer for a -different hostname that points at a private address is still rejected, and each -new connection re-validates the resolved addresses so a rebinding answer fails -closed. +The allowlist authorizes only the exact host named. A DNS answer for a different hostname that points at a private address is still rejected, and each new connection re-validates the resolved addresses. ## Docker Compose example @@ -128,25 +70,11 @@ services: RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY: "http://logstash:8080/events" RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR_PRIMARY: "/tmp/rustfs-events" # Allow the webhook host to resolve to the Compose private network. - # Note: the allowlist takes the origin only, without the /events path. + # The allowlist takes the origin only, without the /events path. RUSTFS_OUTBOUND_ALLOW_ORIGINS: "http://logstash:8080" logstash: image: docker.elastic.co/logstash/logstash:8.15.0 # ... ``` -The endpoint keeps its full path (`/events`); the allowlist entry is the origin -(`http://logstash:8080`) only. - -## Upgrade checklist (beta.10 → beta.11+, or OIDC beta.11 → beta.12+) - -1. List every outbound endpoint whose hostname resolves to a loopback, private, - shared, or reserved address: notification webhooks, audit webhooks, OIDC - providers, tiering endpoints, and Keystone auth URLs. -2. Add each one to `RUSTFS_OUTBOUND_ALLOW_ORIGINS` as an exact - `scheme://host:port` origin (no path). -3. Ensure the endpoint is reachable directly — for webhook and audit targets, - proxies are disabled and redirects are not followed. -4. Restart RustFS; the policy is read at startup. -5. Confirm delivery, and check the logs for `... is not allowed` messages if a - target still fails to activate. +The endpoint keeps its full path (`/events`); the allowlist entry is the origin only. Restart RustFS after changing the variable — the policy is read at startup — and check the logs for `is not allowed` messages if a target still fails to activate. diff --git a/docs/operations/pool-metadata-recovery.md b/docs/operations/pool-metadata-recovery.md index 9b45d116d..ad594acc2 100644 --- a/docs/operations/pool-metadata-recovery.md +++ b/docs/operations/pool-metadata-recovery.md @@ -1,8 +1,9 @@ # Pool metadata upgrade and recovery -`pool.bin` is cluster state. Do not delete or copy it independently on a live -node. Version 3 adds a deployment identity, epoch, durable generation, and a -recoverable prepare/commit record on every pool. +**Use this when:** upgrading a cluster to `pool.bin` V3, a node cannot rejoin after a metadata-drive replacement, or startup reports `pool.bin` as incompatible, corrupt, or recovery required. +**Source of truth:** `crates/ecstore/src/core/pools.rs` (`pool.bin` / `pool.bin.identity` reader and writer, the `RUSTFS_POOL_META_V3_WRITE` and `RUSTFS_POOL_META_V3_FLEET_CONFIRMED` gates). + +`pool.bin` is cluster state. Do not delete or copy it independently on a live node. Version 3 adds a deployment identity, epoch, durable generation, and a recoverable prepare/commit record on every pool. ## Compatibility matrix @@ -12,59 +13,30 @@ recoverable prepare/commit record on every pool. | V2-capable binary | read/write while mixed | read/write after the V2 fleet gate | reject | | V3-capable binary | read/migrate | read/migrate | read/write; never downgrade | -Leave `RUSTFS_POOL_META_V3_WRITE` or -`RUSTFS_POOL_META_V3_FLEET_CONFIRMED` disabled while any running process lacks -V3 support. Both must be `true` before an existing cluster migrates. A fresh -deployment can initialize directly at V3. Once a committed V3 generation is -observed, rollback to a V1/V2-only binary is not supported. -Repairing a missing identity on an existing V1/V2 snapshot does not cross the -V3 gate; the identity is committed as initialized while `pool.bin` stays on its -observed legacy version. +Leave `RUSTFS_POOL_META_V3_WRITE` and `RUSTFS_POOL_META_V3_FLEET_CONFIRMED` disabled while any running process lacks V3 support. Both must be `true` before an existing cluster migrates. A fresh deployment can initialize directly at V3. Once a committed V3 generation is observed, rollback to a V1/V2-only binary is not supported. Repairing a missing identity on an existing V1/V2 snapshot does not cross the V3 gate; the identity is committed as initialized while `pool.bin` stays on its observed legacy version. -Unknown fields are not ignored. An unsupported version or field layout is -reported as **incompatible** and is never overwritten. A truncated or invalid -payload is **corrupt** and may be repaired only from a verified committed -replica. Conflicting identities, epochs, or transactions at the same generation -are **recovery required** and need an operator-selected source. +| Startup verdict | Cause | Handling | +| --- | --- | --- | +| **incompatible** | Unsupported version or field layout (unknown fields are not ignored) | Never overwritten | +| **corrupt** | Truncated or invalid payload | Repaired only from a verified committed replica | +| **recovery required** | Conflicting identities, epochs, or transactions at the same generation | Needs an operator-selected source | ## Partial writes -A V3 update first conditionally writes a pending generation containing the last -committed snapshot, then conditionally replaces it with the committed record. -During initial bootstrap, `pool.bin.identity` remains `initialized=false` and -carries a unique fresh-bootstrap nonce until that committed V3 record is -verified. Restarting from an initial prepare record finishes generation 1; it -never rewrites the record as V1 or V2. -On restart: +A V3 update first conditionally writes a pending generation containing the last committed snapshot, then conditionally replaces it with the committed record. During initial bootstrap, `pool.bin.identity` remains `initialized=false` and carries a unique fresh-bootstrap nonce until that committed V3 record is verified. Restarting from an initial prepare record finishes generation 1; it never rewrites the record as V1 or V2. On restart: - prepare-only replicas expose their previous committed snapshot; - one committed replica makes that transaction authoritative; - remaining pending or older replicas are repairable by the next fenced save; - two different committed transactions at one generation stop startup. -Do not hand-edit a pending record or select a replica only because it is in pool -zero. Preserve all copies when escalating recovery. +Do not hand-edit a pending record or select a replica only because it is in pool zero. Preserve all copies when escalating recovery. ## Disk replacement and metadata erasure 1. Keep a quorum of nodes online and verify the cluster is ready. 2. Stop the lagging node before replacing or erasing its metadata drive. -3. Restore storage formats and the `pool.bin.identity` marker from the same - deployment before rejoining it. -4. Start the node and wait for it to load the verified committed generation and - repair its replicas before touching another node. +3. Restore storage formats and the `pool.bin.identity` marker from the same deployment before rejoining it. +4. Start the node and wait for it to load the verified committed generation and repair its replicas before touching another node. -An initialized identity with every `pool.bin` missing is recovery required. -Existing storage formats with neither identity nor `pool.bin` are also recovery -required. Format creation alone is not fresh-cluster proof. Only the elected -first topology node may create a durable `initialized=false` bootstrap identity -with a fresh-bootstrap nonce, and only after every configured disk explicitly -responds that it is unformatted. -An unreachable peer, a non-elected distributed node, or an existing format is -not sufficient proof. All-missing `pool.bin` replicas are accepted only by the -same startup that proved the fresh topology and persisted that pending identity. -When every `pool.bin` is missing, a later startup must recover even if the -pending identity survived. This prevents a wiped or lagging node from rebuilding -empty state and overwriting the cluster. Runtime reload, rebalance activation, -and rebalance worker admission all fail closed and latch the same recovery gate -until the node is restarted with readable metadata. +An initialized identity with every `pool.bin` missing is recovery required, as are existing storage formats with neither identity nor `pool.bin`. Format creation alone is not fresh-cluster proof: only the elected first topology node may create a durable `initialized=false` bootstrap identity with a fresh-bootstrap nonce, and only after every configured disk explicitly responds that it is unformatted. An unreachable peer, a non-elected distributed node, or an existing format is not sufficient proof. All-missing `pool.bin` replicas are accepted only by the same startup that proved the fresh topology and persisted that pending identity; when every `pool.bin` is missing, a later startup must recover even if the pending identity survived. This prevents a wiped or lagging node from rebuilding empty state and overwriting the cluster. Runtime reload, rebalance activation, and rebalance worker admission all fail closed and latch the same recovery gate until the node is restarted with readable metadata. diff --git a/docs/operations/presigned-multipart-size-limit.md b/docs/operations/presigned-multipart-size-limit.md deleted file mode 100644 index 4769b6247..000000000 --- a/docs/operations/presigned-multipart-size-limit.md +++ /dev/null @@ -1,55 +0,0 @@ -# Presigned multipart total-size limit - -RustFS V2 supports an optional capability on a signed or SigV4-presigned -`CreateMultipartUpload` request: - -```text -x-rustfs-max-total-object-size= -``` - -The backend must include the parameter before calculating the SigV4 -signature. It is part of the canonical query and cannot be added, removed, or -changed by the browser. RustFS stores the verified limit in the multipart -upload session and applies it to every `UploadPart` and to -`CompleteMultipartUpload`. - -Backend pseudocode (the custom query must be present before signing): - -```text -uri = "/photos/archive.zip?uploads" -uri += "&x-rustfs-max-total-object-size=104857600" -presigned_url = sigv4_presign("POST", uri, credentials) -# Return presigned_url to the browser. Never append the parameter afterwards. -``` - -The resulting flow is: - -1. The backend signs `CreateMultipartUpload?...&x-rustfs-max-total-object-size=104857600`. -2. RustFS verifies the SigV4 request and persists the limit with the upload ID. -3. The browser uploads parts using the returned upload ID. -4. RustFS rejects a part whose declared logical size would exceed the remaining - budget and rejects completion if the server-side part metadata exceeds the - limit. - -The limit is measured in logical object bytes (`actual_size`), not erasure, -encryption, or compression bytes. Replacing an existing part uses replacement -semantics: the old part size is removed before the new part size is admitted. -Unknown-length parts are rejected for capped sessions rather than buffered -without a bound. Capped parts are admitted under an upload-wide write lock -before temporary shards are created and use a per-upload staging permit to -bound local in-flight data. The distributed lock is released while the body is -read and reacquired for the final check/rename, so Complete and Abort are not -blocked behind a slow upload. The normal request-body stall timeout releases -the staging permit when a client stops sending. - -The parameter is accepted only on `CreateMultipartUpload`. Supplying it on -`UploadPart`, `CompleteMultipartUpload`, `AbortMultipartUpload`, listing, or -copy operations returns `InvalidRequest`; those requests use the persisted -session state. A multipart upload created without this parameter remains -unlimited for backward compatibility. The V1 single-request capability -(`x-rustfs-max-content-length`) is independent and is not a multipart limit. - -Because enforcement happens in the multipart data plane, every node that may -receive requests for a capped upload must run the V2 implementation. During a -rolling upgrade, route capped uploads only to upgraded nodes; older nodes treat -the internal metadata as unknown and cannot enforce the limit. diff --git a/docs/operations/presigned-put-size-limit.md b/docs/operations/presigned-put-size-limit.md deleted file mode 100644 index 85f0a746b..000000000 --- a/docs/operations/presigned-put-size-limit.md +++ /dev/null @@ -1,35 +0,0 @@ -# Presigned PutObject size limit - -RustFS V1 supports an optional, RustFS-specific capability on a SigV4 -presigned `PutObject` URL: - -```text -x-rustfs-max-content-length= -``` - -The backend that creates the URL must add this query parameter to the request -URI before calculating the SigV4 presign. It is part of the canonical query; -adding, removing, or changing it after signing invalidates the signature. A -browser can then upload with a plain `PUT` and does not need a custom size -header. - -RustFS validates the capability after SigV4 authentication and enforces it on -the decoded request body. A declared `Content-Length` above the limit is -rejected before storage. If the body produces more bytes than the limit while -streaming, RustFS returns `EntityTooLarge` and does not publish the object. - -The V1 contract is deliberately narrow: - -- The parameter is accepted only on a SigV4 presigned `PutObject` request. -- Duplicate, case-variant, malformed, negative, or overflowing values return - `InvalidRequest`. -- Requests without the parameter, including ordinary authenticated or - anonymous `PUT`, keep the existing behavior. -- The parameter on `CopyObject`, multipart, `GET`, `HEAD`, `DELETE`, bucket, or - other operations returns `InvalidRequest`. -- Unknown-length and SigV4 streaming-chunked uploads remain unsupported by the - existing PutObject admission contract and are not enabled by this feature. - -This capability is per request; it is not a cumulative multipart-upload cap. -Multipart session limits are planned for V2 under a separate query/API -contract. diff --git a/docs/operations/presigned-size-limits.md b/docs/operations/presigned-size-limits.md new file mode 100644 index 000000000..922d19c1c --- /dev/null +++ b/docs/operations/presigned-size-limits.md @@ -0,0 +1,57 @@ +# Presigned upload size limits + +**Use this when:** a backend issues SigV4-presigned upload URLs to browsers and must cap how much a client can upload with one URL — per request (`PutObject`) or per multipart upload. +**Source of truth:** `rustfs/src/auth.rs` (`RUSTFS_MAX_CONTENT_LENGTH_QUERY`, `RUSTFS_MAX_TOTAL_OBJECT_SIZE_QUERY`, `parse_presigned_put_max_content_length`, `parse_presigned_multipart_max_total_object_size`); enforcement in `rustfs/src/app/object/put.rs` (`MaxContentLengthStream`), `rustfs/src/app/multipart_usecase.rs` (`multipart_max_total_object_size`), and `crates/ecstore/src/set_disk/ops/multipart.rs` (`multipart_size_limit_from_metadata`, `admitted_multipart_size`). + +Both limits are RustFS-specific query parameters carried inside the SigV4 canonical query. + +## Shared signing rule + +1. The backend appends the parameter to the request URI **before** computing the SigV4 presigned signature. It is part of the canonical query, so adding, removing, or changing it afterwards invalidates the signature; the browser cannot alter it. +2. RustFS parses the parameter only after the request has been accepted as SigV4-signed (the `VerifiedPresignedRequest` / `VerifiedSigV4Request` request markers). The same query string on an unsigned request is rejected with `InvalidRequest`. +3. The value is an unsigned 64-bit integer. Duplicate, case-variant, malformed, negative, or overflowing values return `InvalidRequest`. +4. Requests that do not carry the parameter, including ordinary authenticated or anonymous uploads, keep their existing behavior. + +| | V1 per-request | V2 per-upload | +| --- | --- | --- | +| Query parameter | `x-rustfs-max-content-length=` | `x-rustfs-max-total-object-size=` | +| Accepted on | SigV4 presigned `PutObject` only | `CreateMultipartUpload` only (signed or presigned SigV4) | +| Any other operation carrying it | `InvalidRequest` (`CopyObject`, multipart, `GET`, `HEAD`, `DELETE`, bucket operations) | `InvalidRequest` (`UploadPart`, `CompleteMultipartUpload`, `AbortMultipartUpload`, listing, copy — these read the persisted session state instead) | +| What is measured | Decoded request body bytes of that one request | Logical object bytes (`actual_size`) summed across the upload's parts; not erasure, encryption, or compression bytes | +| Where the limit lives | The request only | Multipart session metadata (`SUFFIX_MAX_TOTAL_OBJECT_SIZE`), written at create time | +| Over-limit result | `EntityTooLarge`; the object is not published | `EntityTooLarge` on the offending `UploadPart`, and on `CompleteMultipartUpload` if the recorded parts exceed the limit | + +## V1: `x-rustfs-max-content-length` + +```text +uri = "/photos/avatar.png" +uri += "?x-rustfs-max-content-length=10485760" +presigned_url = sigv4_presign("PUT", uri, credentials) +# Return presigned_url to the browser. Never append the parameter afterwards. +``` + +- A declared `Content-Length` above the limit is rejected before storage. A body that streams more bytes than the limit is cut off with `EntityTooLarge` and nothing is published. +- Not combinable with archive auto-extraction (`x-amz-meta-snowball-auto-extract`): `InvalidRequest`. +- Unknown-length and SigV4 streaming-chunked uploads stay outside the existing PutObject admission contract; this parameter does not enable them. +- Per request only: it is neither a cumulative cap across several PUTs nor a multipart limit. + +## V2: `x-rustfs-max-total-object-size` + +```text +uri = "/photos/archive.zip?uploads" +uri += "&x-rustfs-max-total-object-size=104857600" +presigned_url = sigv4_presign("POST", uri, credentials) +# Return presigned_url to the browser. Never append the parameter afterwards. +``` + +1. RustFS verifies the SigV4 request and persists the limit with the upload ID. +2. The browser uploads parts with the returned upload ID; part requests carry no custom parameter. +3. Each `UploadPart` (and each `UploadPartCopy` into the upload) is admitted only if the upload's running logical total plus this part fits the budget. Replacing an existing part number uses replacement semantics: the old part's size is released before the new size is admitted. +4. `CompleteMultipartUpload` re-sums the recorded parts and rejects the completion if they exceed the limit. + +Properties of a capped upload: + +- Unknown-length or negative-length parts are rejected with `UnexpectedContent` rather than buffered without a bound. +- Capped parts are admitted under an upload-wide write lock before temporary shards are created, and hold a per-upload staging permit that bounds local in-flight data. The lock is released while the body is read and reacquired for the final check and rename, so `Complete` and `Abort` are not blocked behind a slow upload. The request-body stall timeout releases the staging permit when a client stops sending. +- An upload created without the parameter stays unlimited. +- Enforcement runs in the multipart data plane on every node. During a rolling upgrade, route capped uploads only to nodes that carry the V2 implementation; a node without it treats the internal metadata as unknown and cannot enforce the limit. diff --git a/docs/operations/rebalance-stored-representation-impact.md b/docs/operations/rebalance-stored-representation-impact.md index 5a984e7d3..285389181 100644 --- a/docs/operations/rebalance-stored-representation-impact.md +++ b/docs/operations/rebalance-stored-representation-impact.md @@ -1,50 +1,35 @@ # Rebalance Stored-Representation Impact Guide -This guide covers the historical data-movement read defect tracked by -[`rustfs/backlog#1850`](https://github.com/rustfs/backlog/issues/1850). It is an -impact-assessment and read-only triage guide. It does not repair, rewrite, -migrate, delete, or quarantine any object. +**Use this when:** a deployment ran pool rebalance (or, in a narrower window, decommission) on an affected release and you must assess whether compressed or server-side-encrypted objects were copied as plaintext under their original metadata. Read-only triage only; this guide repairs nothing. -The defect affected data movement when the source reader returned logical -plaintext but the target writer preserved the source's stored-representation -metadata and sizes. Compressed objects could therefore be copied as plaintext -under compression metadata. Server-managed encrypted objects could be copied as -plaintext under encryption metadata. The forward rebalance fix reached `main` -in commit -[`e11fcfbd`](https://github.com/rustfs/rustfs/commit/e11fcfbd087f8a8dae2c0f2c62bc0f6e40e3f10a) -through [PR #6057](https://github.com/rustfs/rustfs/pull/6057). +**Source of truth:** `crates/ecstore/src/services/rebalance/migration.rs` and `crates/ecstore/src/core/pools.rs` (`raw_data_movement_read` on the source read options), `crates/ecstore/src/object_api/readers.rs` (raw stored-range read path), `crates/ecstore/src/data_movement/mod.rs` (metadata, part-size, ETag and index preservation), `crates/filemeta/examples/dump_fileinfo.rs` (evidence decoder). Tracked as `rustfs/backlog#1850`. -Upgrading prevents this defect in later rebalance runs. It does not validate or -repair copies produced by an earlier run. +## Mechanism + +The migration pipeline is a stored-representation copier: it preserves the source ETag and internal metadata, divides the stream using stored `part.size` values, and carries the decoded compression index. The affected rebalance read options supplied only the version ID and lock setting, so the normal GET read plan decompressed or decrypted the stream first. The target write could therefore complete while its bytes no longer matched the metadata describing them: compressed objects became plaintext under compression metadata, SSE-S3 and SSE-KMS objects became plaintext under encryption metadata. + +Historical rebalance cleanup deleted the source entry only after every version in it was reported moved. A target write accepted as a successful move could therefore be followed by source deletion even though a later GET of the target fails. Conversely, a source-read failure prevented the version from being counted as moved and blocked normal source cleanup. + +The forward fix (commit [`e11fcfbd`](https://github.com/rustfs/rustfs/commit/e11fcfbd087f8a8dae2c0f2c62bc0f6e40e3f10a), [PR #6057](https://github.com/rustfs/rustfs/pull/6057)) sets `raw_data_movement_read: true` for rebalance source reads. Upgrading prevents the defect in later runs; it does not validate or repair copies produced by an earlier run. ## Immediate Operator Decision Treat a deployment as exposed when both conditions are true: -1. it ran rebalance in an affected build, or decommission in the narrower - historical window described below; and +1. it ran rebalance in an affected build, or decommission in the narrower historical window below; and 2. the operation could have selected compressed, SSE-S3, or SSE-KMS objects. For an exposed deployment: -- preserve old pool media, snapshots, replicas, and backups before any pool is - removed, reformatted, reused, or returned; -- stop destructive cleanup and do not use another rebalance or decommission run - as a repair mechanism; +- preserve old pool media, snapshots, replicas, and backups before any pool is removed, reformatted, reused, or returned; +- stop destructive cleanup and do not use another rebalance or decommission run as a repair mechanism; - inventory and validate candidates with read-only operations; -- handle SSE-S3 and SSE-KMS candidates as a confidentiality incident as well as - a data-integrity incident; -- restore only from a separately verified source under an incident-specific - recovery plan. +- handle SSE-S3 and SSE-KMS candidates as a confidentiality incident as well as a data-integrity incident; +- restore only from a separately verified source under an incident-specific recovery plan. ## Affected Versions -The release boundaries below were verified by tag ancestry. Commit -[`a236b0d0`](https://github.com/rustfs/rustfs/commit/a236b0d01d40a152309446a553756ea991c9f901) -introduced the merged rebalance and decommission implementation. Commit -[`2f25cf60`](https://github.com/rustfs/rustfs/commit/2f25cf606e5ca814fe992be6327a91e31fe066b3) -introduced the raw stored-representation read mode and wired it into -decommission. Commit `e11fcfbd` wired the same mode into rebalance. +Boundaries were verified by tag ancestry. Commit [`a236b0d0`](https://github.com/rustfs/rustfs/commit/a236b0d01d40a152309446a553756ea991c9f901) introduced the merged rebalance and decommission implementation. Commit [`2f25cf60`](https://github.com/rustfs/rustfs/commit/2f25cf606e5ca814fe992be6327a91e31fe066b3) introduced the raw stored-representation read mode and wired it into decommission. Commit `e11fcfbd` wired the same mode into rebalance. | Release or commit range | Rebalance | Decommission | Operator classification | | --- | --- | --- | --- | @@ -53,31 +38,9 @@ decommission. Commit `e11fcfbd` wired the same mode into rebalance. | `1.0.0-beta.9` through `1.0.0-rc.1`, from `2f25cf60` up to but excluding `e11fcfbd` | Decoded read | Raw stored-representation read | Rebalance requires assessment; decommission is not affected by this defect | | `1.0.0-rc.2` and later, at or after `e11fcfbd` | Raw stored-representation read | Raw stored-representation read | Forward-fixed; earlier copies still require assessment | -Preview tags follow the commit they reference. In particular, the `rc.1` -preview is affected and the `rc.2` preview contains the forward fix. For custom -or untagged builds, compare the deployed commit with the three commit boundaries -rather than inferring behavior from a version string. +Preview tags follow the commit they reference: the `rc.1` preview is affected and the `rc.2` preview contains the forward fix. For custom or untagged builds, compare the deployed commit with the three commit boundaries rather than inferring behavior from a version string. -The historical decommission result is narrower than the rebalance result but is -not empty. Before `2f25cf60`, decommission used the same ordinary decoded reader. -From `1.0.0-beta.9` onward it explicitly used `raw_data_movement_read: true`. -Any code change or automated remediation for the earlier decommission window is -outside this report and requires a separate issue. - -## Why The Copy Could Be Accepted - -The migration pipeline is a stored-representation copier. It preserves the -source ETag and internal metadata, uses stored `part.size` values to divide the -stream, and carries the decoded compression index. The affected rebalance read -options supplied only the version ID and lock setting, so the normal GET read -plan decompressed or decrypted the stream first. A target write could therefore -complete while its bytes no longer matched the metadata that described them. - -Historical rebalance cleanup ran only after every version in an entry was -reported moved. It then deleted the source entry. A target write accepted as a -successful move could therefore be followed by source deletion even though a -later GET of the target would fail. Conversely, a source-read failure prevented -the version from being counted as moved and prevented normal source cleanup. +The decommission exposure is narrower than the rebalance exposure but not empty: before `2f25cf60` decommission used the same ordinary decoded reader. Any code change or automated remediation for that earlier decommission window is outside this guide and requires a separate issue. ## Object Classification @@ -90,32 +53,21 @@ the version from being counted as moved and prevented normal source cleanup. | SSE-C | The migration request did not have the customer key, so the normal read failed closed | Migration failure and possible incomplete progress; no successful corrupting copy is expected from this path | Medium; confirm the source was retained | | Any compressed and encrypted combination | Multiple stored-representation assumptions were violated | Confidentiality exposure and data corruption | Critical | -The classification is specific to this defect. A low-risk classification does -not certify an object against unrelated corruption. +The classification is specific to this defect. A low-risk classification does not certify an object against unrelated corruption. ## Read-Only Assessment Workflow -### 1. Establish The Operation Window +### 1. Establish the operation window -Record the exact RustFS version and commit for every node that participated. -Collect the authenticated rebalance status response, decommission status when -applicable, service logs, deployment change records, and release history. +Record the exact RustFS version and commit for every node that participated. Collect the authenticated rebalance status response, decommission status when applicable, service logs, deployment change records, and release history. -Persisted rebalance metadata records the run ID, participating pools, start and -end state, bucket lists, counters, and the last bucket/object progress value. It -does not persist a complete per-object movement ledger. Status metadata can -prove that a run occurred and narrow time, pool, and bucket scope, but it cannot -by itself enumerate every moved object. +Persisted rebalance metadata records the run ID, participating pools, start and end state, bucket lists, counters, and the last bucket/object progress value. It does not persist a per-object movement ledger: status metadata can prove that a run occurred and narrow time, pool, and bucket scope, but it cannot enumerate every moved object. -If no reliable operation record remains, assume that every object version in a -bucket present during the affected deployment interval is a candidate until -other evidence narrows the set. +If no reliable operation record remains, assume that every object version in a bucket present during the affected deployment interval is a candidate until other evidence narrows the set. -### 2. Build A Candidate Inventory +### 2. Build a candidate inventory -Use read-only S3 list and list-object-versions operations for the buckets in -scope. Preserve bucket, key, version ID, last-modified time, size, ETag, storage -class, and any client-side content digest. Join that list with: +Use read-only S3 list and list-object-versions operations for the buckets in scope. Preserve bucket, key, version ID, last-modified time, size, ETag, storage class, and any client-side content digest. Join that list with: - upload records that identify compression settings or SSE mode; - KMS audit history and application catalogs; @@ -123,20 +75,13 @@ class, and any client-side content digest. Join that list with: - rebalance/decommission timestamps and source/target pool records; - server access logs showing successful or failed reads after movement. -Do not use ETag equality as proof of content integrity. The migration writer -preserved the source ETag, including for a malformed target copy, and multipart -or encrypted ETags are not general-purpose content hashes. +Do not use ETag equality as proof of content integrity. The migration writer preserved the source ETag, including for a malformed target copy, and multipart or encrypted ETags are not general-purpose content hashes. -### 3. Classify Stored Metadata On Evidence Copies +### 3. Classify stored metadata on evidence copies -When API and application records cannot classify a candidate, copy `xl.meta` -from each relevant shard disk to a restricted evidence location and inspect the -copy on an offline host. Do not edit or decode metadata in place on a live data -path. Keep the evidence copies under the same access controls as the object. +When API and application records cannot classify a candidate, copy `xl.meta` from each relevant shard disk to a restricted evidence location and inspect the copy on an offline host. Do not edit or decode metadata in place on a live data path. Keep the evidence copies under the same access controls as the object. -The existing `rustfs-filemeta` example can decode an evidence copy. It prints -metadata values, some of which are sensitive encryption material, so redact -metadata values before they reach a terminal or report: +The `rustfs-filemeta` example decodes an evidence copy. It prints metadata values, some of which are sensitive encryption material, so redact values before they reach a terminal or report: ```bash cargo run --quiet -p rustfs-filemeta --example dump_fileinfo -- /evidence/object/xl.meta | @@ -145,119 +90,51 @@ cargo run --quiet -p rustfs-filemeta --example dump_fileinfo -- /evidence/object Use the output only as a screen: -- either the `x-rustfs-internal-compression` or - `x-minio-internal-compression` key marks a compressed representation; -- `actual-size`, per-part `size`/`actual_size`, and compression-index totals - should be arithmetically consistent; +- either the `x-rustfs-internal-compression` or `x-minio-internal-compression` key marks a compressed representation; +- `actual-size`, per-part `size`/`actual_size`, and compression-index totals should be arithmetically consistent; - SSE-C customer-algorithm/MD5 markers identify SSE-C; - KMS key-ID/context markers identify SSE-KMS; -- a managed encryption envelope without SSE-C or KMS markers identifies an - SSE-S3 candidate. +- a managed encryption envelope without SSE-C or KMS markers identifies an SSE-S3 candidate. -Never include encryption metadata values in tickets, logs, chat, or assessment -reports. Metadata consistency is necessary but not sufficient: the defect -preserved metadata, so plausible sizes and a decodable index do not prove that -the stored bytes match it. +Never include encryption metadata values in tickets, logs, chat, or assessment reports. Metadata consistency is necessary but not sufficient: the defect preserved metadata, so plausible sizes and a decodable index do not prove that the stored bytes match it. -### 4. Validate Logical Content Without Mutation +### 4. Validate logical content without mutation -For each high- or critical-risk candidate, perform a complete authenticated GET -of the exact version into a restricted validation sink. Supply the customer key -only for an authorized SSE-C check. Record the status, byte count, and a -cryptographic digest calculated by the validation client. Compare it with a -digest from an independently trusted source, backup, replica, or application -record. +For each high- or critical-risk candidate, perform a complete authenticated GET of the exact version into a restricted validation sink. Supply the customer key only for an authorized SSE-C check. Record the status, byte count, and a cryptographic digest calculated by the validation client. Compare it with a digest from an independently trusted source, backup, replica, or application record. Interpret the result conservatively: -- a GET decode/decrypt error, unexpected EOF, or short byte count is a strong - affected-copy signal, but may also have another corruption cause; +- a GET decode/decrypt error, unexpected EOF, or short byte count is a strong affected-copy signal, but may also have another corruption cause; - a matching independent cryptographic digest validates that logical version; -- a successful GET without an independent digest proves readability, not - identity; +- a successful GET without an independent digest proves readability, not identity; - a matching ETag alone is inconclusive; -- an SSE-S3/KMS candidate moved in the affected window remains a confidentiality - incident until storage-level review excludes plaintext target copies and - derivative snapshots or backups. +- an SSE-S3/KMS candidate moved in the affected window remains a confidentiality incident until storage-level review excludes plaintext target copies and derivative snapshots or backups. -Storage-level confirmation for managed-SSE candidates may expose plaintext and -sealed-key material. It must be performed only by the incident/security owner on -offline evidence copies. Do not print, upload, or serve raw shard bytes, and do -not bypass RustFS to return them to an application. +Storage-level confirmation for managed-SSE candidates may expose plaintext and sealed-key material. It must be performed only by the incident/security owner on offline evidence copies. Do not print, upload, or serve raw shard bytes, and do not bypass RustFS to return them to an application. -### 5. Record Confidence And Outcome +### 5. Record confidence and outcome Record one result for every candidate version: -- `confirmed-good`: full logical bytes match an independent digest; -- `confirmed-affected`: target decode/decrypt/length evidence and a trusted - source establish the mismatch, or authorized storage review confirms - plaintext under managed-SSE metadata; -- `suspected`: the version and operation window match, but proof is incomplete; -- `not-applicable`: evidence proves the object was plain and uncompressed or was - never selected by an affected operation; -- `unrecoverable-pending-source`: affected or suspected, with no verified source - yet found. +| Result | Meaning | +| --- | --- | +| `confirmed-good` | Full logical bytes match an independent digest. | +| `confirmed-affected` | Target decode/decrypt/length evidence and a trusted source establish the mismatch, or authorized storage review confirms plaintext under managed-SSE metadata. | +| `suspected` | The version and operation window match, but proof is incomplete. | +| `not-applicable` | Evidence proves the object was plain and uncompressed or was never selected by an affected operation. | +| `unrecoverable-pending-source` | Affected or suspected, with no verified source yet found. | -Retain the evidence used for each decision. Do not collapse object versions with -the same key into one result. +Retain the evidence used for each decision. Do not collapse object versions with the same key into one result. -## Source Retention And Recovery Limits +## Source Retention and Recovery Limits -Successful historical migration could be followed by source-entry deletion. -Therefore, neither successful rebalance status nor absence from the old source -pool proves that the target bytes are sound. Recovery is possible only from a -separately verified source, such as: +Successful historical migration could be followed by source-entry deletion, so neither successful rebalance status nor absence from the old source pool proves that the target bytes are sound. Recovery is possible only from a separately verified source: - retained source-pool media or a snapshot taken before cleanup; - an independently validated replica; - an external backup; - the original application or upstream source with a trusted digest. -SSE-C normally failed before the target copy was accepted because the migration -read had no customer key. That failure prevented normal source cleanup, but -operators must verify the exact version on retained source media rather than -assuming it is present. +SSE-C normally failed before the target copy was accepted because the migration read had no customer key. That failure prevented normal source cleanup, but operators must verify the exact version on retained source media rather than assuming it is present. -If no verified source exists, mark the version unrecoverable for this incident. -Do not edit `xl.meta`, rewrite shard files, clear encryption/compression markers, -or overwrite the object in place. Those actions can destroy evidence, violate -retention/versioning policy, or turn a visible read failure into silent data -substitution. Any restoration or replacement procedure needs its own reviewed, -rollback-aware plan. - -## Release Guidance - -Release notes for `1.0.0-rc.2` and later should state: - -> Rebalance now copies the stored object representation for compressed and -> encrypted objects. Deployments that ran rebalance on versions from -> `1.0.0-alpha.91` through `1.0.0-rc.1` should preserve old pool media and run -> the read-only assessment in this guide. Upgrading prevents new copies from -> this defect but does not repair historical copies. Deployments that ran -> decommission from `1.0.0-alpha.91` through `1.0.0-beta.8` require the same -> assessment. SSE-S3 and SSE-KMS candidates require security incident handling. - -Do not recommend rerunning rebalance as remediation. Do not remove or repurpose -old pool media until high- and critical-risk candidates have a recorded outcome -and the incident owner has accepted the recovery limits. - -## Evidence Audit - -The conclusions above are grounded in these repository facts: - -- `crates/ecstore/src/services/rebalance/migration.rs` now sets both - `data_movement` and `raw_data_movement_read` for rebalance source reads; -- `crates/ecstore/src/core/pools.rs` sets the same flags for decommission source - reads; -- `crates/ecstore/src/object_api/readers.rs` returns the stored byte range before - compression or encryption transforms when `raw_data_movement_read` is set; -- `crates/ecstore/src/data_movement/mod.rs` preserves stored part sizes, ETags, - indexes, and internal metadata during migration; -- the historical `a236b0d0` rebalance and decommission readers both used normal - read options, while `2f25cf60` changed only decommission to the raw mode; -- the historical rebalance entry deleted its source prefix only after all - versions were counted as moved; -- the tag ancestry boundaries are `1.0.0-alpha.91`, `1.0.0-beta.9`, and - `1.0.0-rc.2` for the implementation, decommission raw-read fix, and rebalance - raw-read fix respectively. +If no verified source exists, mark the version unrecoverable for this incident. Do not edit `xl.meta`, rewrite shard files, clear encryption/compression markers, or overwrite the object in place. Those actions can destroy evidence, violate retention/versioning policy, or turn a visible read failure into silent data substitution. Any restoration or replacement procedure needs its own reviewed, rollback-aware plan. diff --git a/docs/operations/replication-check.md b/docs/operations/replication-check.md index c623a1300..d102d30db 100644 --- a/docs/operations/replication-check.md +++ b/docs/operations/replication-check.md @@ -1,29 +1,25 @@ # Replication target check -`GET /BUCKET?replication-check` is a signed S3 extension for validating every -replication target referenced by a bucket replication configuration. +**Use this when:** you are about to call, automate, or debug `GET /BUCKET?replication-check`, or need to explain why a `GET` wrote and deleted objects on a replication target. + +**Source of truth:** `rustfs/src/admin/router.rs` (`REPLICATION_CHECK_PROBE_PREFIX`, `REPLICATION_CHECK_ERROR_MAX_BYTES`, the `replication-check` route handler). + +`GET /BUCKET?replication-check` is a signed S3 extension that validates every replication target referenced by a bucket replication configuration. ## Active mutation warning Despite using `GET`, this operation is **not read-only**. On each target it: -1. writes an 8-byte object under `.rustfs.sys/replication-check//`; +1. writes an 8-byte object under `.rustfs.sys/replication-check//` (`REPLICATION_CHECK_PROBE_PREFIX`); 2. creates a replicated delete marker; 3. permanently deletes the probe object version; and -4. enumerates that exact probe key and attempts to delete every remaining - object version and delete marker. +4. enumerates that exact probe key and attempts to delete every remaining object version and delete marker. -Callers should obtain operator confirmation before sending the request. Probe -keys use a reserved namespace and two independent random UUIDs. Before writing, -the server verifies that no version or delete marker exists at the exact key, -then uses an atomic `If-None-Match: *` write so it cannot overwrite a key created -concurrently by an application. +Obtain operator confirmation before sending the request. Probe keys use a reserved namespace and two independent random UUIDs. Before writing, the server verifies that no version or delete marker exists at the exact key, then uses an atomic `If-None-Match: *` write so it cannot overwrite a key created concurrently by an application. ## Response contract -The route returns HTTP 200 with JSON after all configured targets have been -checked. `Status` is `FAILED` when any target or cleanup phase failed; successful -target results remain present when another target fails. +The route returns HTTP 200 with JSON after all configured targets have been checked. `Status` is `FAILED` when any target or cleanup phase failed; successful target results remain present when another target fails. ```json { @@ -55,23 +51,15 @@ target results remain present when another target fails. } ``` -Phase states are `OK`, `FAILED`, or `SKIPPED`. Errors are single-line, bounded -to 512 bytes, and omit remote messages, endpoints, credentials, signatures, and -authorization material. A cleanup failure is always explicit; it is never -reported as a successful check. +| Field | Contract | +| --- | --- | +| `Phases.*.Status` | `OK`, `FAILED`, or `SKIPPED`. | +| `Error` | Single line, bounded to `REPLICATION_CHECK_ERROR_MAX_BYTES` (512 bytes); omits remote messages, endpoints, credentials, signatures, and authorization material. | +| `Cleanup` | A cleanup failure is always explicit; it is never reported as a successful check. | +| `Code` | Appears only on failures callers are expected to branch on (currently `BucketRemoteTargetVersionMismatch`). Go decoders ignore the unknown key. | -`VersionFidelity` pins the version-identity contract on **both** write paths: -the probe PUT carries a source version id (header plus `?versionId=` query, -the exact shape live replication uses) and the target must answer with the -same id, and a second probe repeats it through CreateMultipartUpload -> -UploadPart -> CompleteMultipartUpload, where the target fixes the version at -initiate and only reports it on completion. A target can adopt PutObject ids -and still mint its own for multipart, which would leave multipart deletes and -heals addressing a version that never existed; the failure message names the -path that drifted. Targets that -mint their own version ids break every version-addressed operation that -follows (version deletes, heal re-drives), so the phase fails with the -machine-readable extension key `"Code": "BucketRemoteTargetVersionMismatch"`, -the later mutation phases are skipped, and cleanup still removes the probe via -the version id the target actually assigned. `Code` only appears on failures -that callers are expected to branch on; Go decoders ignore the unknown key. +## VersionFidelity phase + +`VersionFidelity` pins the version-identity contract on both write paths. The probe PUT carries a source version id (header plus `?versionId=` query, the exact shape live replication uses) and the target must answer with the same id; a second probe repeats the check through CreateMultipartUpload -> UploadPart -> CompleteMultipartUpload, where the target fixes the version at initiate and only reports it on completion. A target can adopt PutObject ids and still mint its own for multipart; the failure message names the path that drifted. + +A target that mints its own version ids breaks every version-addressed operation that follows (version deletes, heal re-drives). The phase therefore fails with `"Code": "BucketRemoteTargetVersionMismatch"`, the later mutation phases are skipped, and cleanup still removes the probe via the version id the target actually assigned. diff --git a/docs/operations/replication-object-size-limits.md b/docs/operations/replication-object-size-limits.md index bbb17df29..40ce8ec0d 100644 --- a/docs/operations/replication-object-size-limits.md +++ b/docs/operations/replication-object-size-limits.md @@ -1,5 +1,8 @@ # Replication object size and shape limits (generic S3 targets) +**Use this when:** an object fails to replicate to an S3-compatible target with `EntityTooLarge`/`EntityTooSmall`, or you need to know whether a large or oddly-chunked object is replicable before relying on it. +**Source of truth:** `crates/ecstore/src/bucket/replication/` (transport selection and part replay), `crates/replication/` (target client), `crates/config/src/constants/` (`RUSTFS_OBS_LOGGER_LEVEL`). + What RustFS can and cannot replicate to a generic S3 target (AWS S3, Wasabi, MinIO, or any other S3-compatible endpoint configured as a bucket replication target), and how a rejected object shows up in the log. @@ -97,5 +100,4 @@ underneath this summary. - [Replication target check](replication-check.md) — validate a target's configuration, versioning, and version fidelity before relying on it. -- [Presigned PUT size limit](presigned-put-size-limit.md) -- [Presigned multipart size limit](presigned-multipart-size-limit.md) +- [Presigned size limits](presigned-size-limits.md) — per-request and per-upload caps a backend can put on presigned uploads. diff --git a/docs/operations/reverse-proxy.md b/docs/operations/reverse-proxy.md index 93bbfe746..bb6b2a882 100644 --- a/docs/operations/reverse-proxy.md +++ b/docs/operations/reverse-proxy.md @@ -1,70 +1,38 @@ # Running RustFS behind a reverse proxy -RustFS speaks plain S3 over HTTP/1.1 and HTTP/2 and works behind reverse -proxies (Caddy, Nginx, HAProxy) and CDNs (Cloudflare). Most proxy problems are -**not** RustFS storage bugs — the same request sent directly to `:9000` -succeeds, while the proxied request fails. This page documents the request -semantics RustFS expects from the proxy layer and gives known-good -configurations. +**Use this when:** a request succeeds against `http://:9000` directly but fails, hangs, or resets through Caddy, Nginx, HAProxy, or Cloudflare. -> Rule of thumb: if a request works against `http://:9000` directly but -> fails through the proxy, the fault is in the proxy/CDN request forwarding, not -> in RustFS object handling. Use the checklist below to find which forwarding -> behavior broke. +**Source of truth:** `crates/config/src/constants/tls.rs` (`DEFAULT_HTTP1_HEADER_READ_TIMEOUT`, `DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT`); the `put_object_body_read_stalled` log event. + +RustFS speaks plain S3 over HTTP/1.1 and HTTP/2. Most proxy problems are not RustFS storage bugs: if the same request works directly against `:9000`, the fault is in proxy/CDN request forwarding. Use the checklist below to find which forwarding behavior broke. ## What RustFS requires from the proxy -S3 clients sign requests with AWS SigV4. RustFS (via `s3s`) re-derives the -signature from the forwarded request, and streams the request body to storage. -For this to succeed the proxy must forward the request **byte-for-byte** with -respect to the signed material and the body: +S3 clients sign requests with AWS SigV4. RustFS (via `s3s`) re-derives the signature from the forwarded request and streams the request body to storage, so the proxy must forward the signed material and the body byte-for-byte: -1. **Do not alter the body.** No transparent compression, no re-encoding, no - truncation. If the client sent `Content-Length: N`, exactly `N` body bytes - must reach RustFS. If fewer bytes arrive, RustFS waits for the rest per the - HTTP spec and the request appears to hang until the client aborts. -2. **Do not rewrite signed headers.** `Host` and any `x-amz-*` / signed headers - must reach RustFS unchanged. Rewriting `Host` is fine only if the client - signed with that same host. -3. **Preserve `Content-Length`; avoid re-chunking large bodies.** Some CDNs - drop `Content-Length` and switch to `Transfer-Encoding: chunked`, or buffer - the whole request body before forwarding — both change the timing and - framing RustFS sees. -4. **Keep upstream idle keep-alive shorter than RustFS's, or vice-versa** (see - next section) so the proxy never reuses a connection RustFS has already - closed. -5. **Do not strip `ETag`** from responses (breaks multipart completion). +| Requirement | Why | +| --- | --- | +| Do not alter the body (no compression, re-encoding, truncation). | If the client sent `Content-Length: N`, exactly `N` body bytes must arrive; with fewer, RustFS waits for the rest and the request appears to hang until the client aborts. | +| Do not rewrite signed headers (`Host`, `x-amz-*`). | Rewriting `Host` is fine only if the client signed with that same host; otherwise `SignatureDoesNotMatch`. | +| Preserve `Content-Length`; do not re-chunk or buffer large bodies. | Switching to `Transfer-Encoding: chunked` or buffering the whole body changes the framing and timing RustFS sees. | +| Keep the proxy's upstream idle keep-alive shorter than RustFS's timeout (next section). | Otherwise the proxy reuses a connection RustFS has already closed. | +| Do not strip `ETag` from responses. | Breaks multipart completion. | -## Idle keep-alive: the #1 cause of `socket hang up` on writes +## Idle keep-alive: the main cause of `socket hang up` on writes -RustFS closes **idle** upstream HTTP/1.1 keep-alive connections after -`RUSTFS_HTTP1_HEADER_READ_TIMEOUT` seconds (default **75s**; see -`crates/config/src/constants/tls.rs`). Reverse proxies keep a pool of upstream -connections and reuse them. If the proxy's upstream idle-keepalive window is -**longer** than RustFS's timeout, the proxy can pick a connection that RustFS -has already FIN'd, write a request onto the dead socket, and the client sees: +RustFS closes idle upstream HTTP/1.1 keep-alive connections after `RUSTFS_HTTP1_HEADER_READ_TIMEOUT` seconds (`DEFAULT_HTTP1_HEADER_READ_TIMEOUT`, 75). Reverse proxies pool and reuse upstream connections. If the proxy's upstream idle-keepalive window is longer than RustFS's timeout, the proxy can pick a connection RustFS has already FIN'd, write a request onto the dead socket, and the client sees: -``` +```text TimeoutError: socket hang up # ECONNRESET AbortError: Request aborted ``` -This is most visible on large `PutObject` uploads because: +This is most visible on large `PutObject` uploads: `PUT` is non-idempotent, so proxies will not transparently retry it, and a larger body keeps the connection in use longer, widening the race window, so small uploads on the same path often succeed. -- `PUT` is non-idempotent, so proxies will **not** transparently retry it; and -- a larger body keeps the connection in use longer, widening the race window, - so small uploads on the same path often succeed. +Fix by making the two windows agree (doing both is safest): -### Fix — make the two windows agree - -Pick **either** side; doing both is safest: - -- **RustFS side:** keep `RUSTFS_HTTP1_HEADER_READ_TIMEOUT` (default 75s) *above* - the proxy's upstream idle-keepalive. To harden slowloris protection on a - directly-exposed node instead, lower it — but then also lower the proxy - keepalive below it. -- **Proxy side:** lower the proxy's upstream idle-keepalive below RustFS's - timeout, or disable upstream keep-alive entirely. +1. RustFS side: keep `RUSTFS_HTTP1_HEADER_READ_TIMEOUT` above the proxy's upstream idle-keepalive. To harden slowloris protection on a directly exposed node instead, lower it, and then also lower the proxy keepalive below it. +2. Proxy side: lower the proxy's upstream idle-keepalive below RustFS's timeout, or disable upstream keep-alive entirely. ## Known-good Caddy configuration @@ -127,47 +95,23 @@ location / { ## Cloudflare (orange-cloud) caveats -Cloudflare's proxy (orange cloud) may **buffer the entire request body** before -forwarding, and can rewrite requests to `Transfer-Encoding: chunked`, dropping -the client's `Content-Length`. Symptoms match this pattern exactly: tiny uploads -succeed, larger uploads fail with `socket hang up`. - -- For large object writes, prefer **DNS-only (grey cloud)** for the S3 endpoint, - or a Cloudflare plan/tunnel configuration that does not buffer/re-chunk the - request body. -- Force `Accept-Encoding: identity` so nothing in the path negotiates - compression (see issues #609, #1492). -- Ensure `Content-Length` reaches RustFS; disable chunked re-encoding in tunnel - settings (see issue #934). +Cloudflare's proxy may buffer the entire request body before forwarding and can rewrite requests to `Transfer-Encoding: chunked`, dropping the client's `Content-Length`. The symptom is exactly the pattern above: tiny uploads succeed, larger uploads fail with `socket hang up`. For large object writes prefer DNS-only (grey cloud) for the S3 endpoint, or a plan/tunnel configuration that does not buffer or re-chunk the body. The `Accept-Encoding` and `Content-Length` rows in the issue table below are the Cloudflare-specific failures seen so far. ## Diagnosis checklist -Run each step and note where behavior diverges: +1. Bypass the proxy. Send the failing request to `http://:9000` directly. Success confirms the fault is in the proxy/CDN path. +2. Bypass the CDN, keep the proxy. Point the proxy straight at the origin (Cloudflare grey cloud / direct DNS). If it now works, the CDN was buffering or re-chunking the body. +3. Check idle reuse. Intermittent failures that correlate with upload size are almost always the keep-alive mismatch. Lower the proxy keepalive (or disable it) and retry. +4. Check for a truncated body. If the upload hangs indefinitely rather than resetting, the proxy is forwarding a partial body and then going silent without closing the connection. RustFS bounds this wait with `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT` (`DEFAULT_HTTP_REQUEST_BODY_READ_TIMEOUT`, 300; `0` disables) and on timeout logs `put_object_body_read_stalled` with the received/expected byte counts. +5. Compare bytes. Confirm the proxy forwards exactly `Content-Length` body bytes with no compression or transformation. +6. Confirm signed headers survive. `Host` and `x-amz-*` must reach RustFS unchanged; a `SignatureDoesNotMatch` (rather than a hang) points here. -1. **Bypass the proxy.** Send the failing request to `http://:9000` - directly. Success here confirms the fault is in the proxy/CDN path. -2. **Bypass the CDN, keep the proxy.** Point the proxy straight at the origin - (Cloudflare grey cloud / direct DNS). If it now works, the CDN was - buffering/re-chunking the body. -3. **Check idle reuse.** If failures are intermittent and correlate with upload - size, it is almost always the keep-alive mismatch above. Lower the proxy - keepalive (or disable it) and retry. - - If instead the upload **hangs indefinitely** (rather than resetting), the - proxy is likely forwarding a *partial* body and then going silent without - closing the connection. RustFS bounds this wait with - `RUSTFS_HTTP_REQUEST_BODY_READ_TIMEOUT` (default 300s; `0` disables) and, on - timeout, logs a `put_object_body_read_stalled` event with the - received/expected byte counts — grep the server log for it to confirm a - truncated-body forwarding problem. -4. **Compare bytes.** Confirm the proxy forwards exactly `Content-Length` body - bytes with no compression/transformation. -5. **Confirm signed headers survive.** `Host` and `x-amz-*` headers must reach - RustFS unchanged; a `SignatureDoesNotMatch` (rather than a hang) points here. +## Known failure signatures -## Related issues - -- #3076 — Large single-request PutObject fails behind Caddy (this document) -- #609 — Bucket inaccessible via Cloudflare proxied DNS (`Accept-Encoding`) -- #1492 — SigV4 `SignatureDoesNotMatch` on Cloudflare tunnel (`Accept-Encoding`) -- #934 — Console fails behind Cloudflare tunnels (chunked / `Content-Length`) -- #1766 — Large multipart upload fails through Nginx (`ETag` stripping) +| Symptom | Forwarding fault | Issue | +| --- | --- | --- | +| Large single-request PutObject fails behind Caddy | Upstream idle keep-alive longer than RustFS's timeout | #3076 | +| Bucket inaccessible via Cloudflare proxied DNS | `Accept-Encoding` negotiation / body transformation | #609 | +| SigV4 `SignatureDoesNotMatch` on Cloudflare tunnel | `Accept-Encoding` header rewritten | #1492 | +| Console fails behind Cloudflare tunnels | Chunked re-encoding drops `Content-Length` | #934 | +| Large multipart upload fails through Nginx | `ETag` stripped from responses | #1766 | diff --git a/docs/operations/rolling-restart.md b/docs/operations/rolling-restart.md index 253327b97..ce185e2a2 100644 --- a/docs/operations/rolling-restart.md +++ b/docs/operations/rolling-restart.md @@ -1,153 +1,76 @@ # Restarting a multi-node RustFS cluster -How to restart nodes of an erasure-coded multi-node cluster without losing -availability, what to expect when several nodes are down at once (sequential -cold start), and how to read the degraded-mode signals. Written for the -failure pattern reported in rustfs/rustfs#4304. +**Use this when:** restarting or upgrading nodes of an erasure-coded multi-node cluster, bringing a cluster back after several nodes were down at once, or interpreting `503` degraded-mode responses during startup. -> Upgrading the binary or container image does not change the on-disk data -> format unless an explicitly enabled feature documents a version floor. -> Replacing the executable and restarting does not run a migration step on -> startup. +**Source of truth:** `crates/config/src/constants/health.rs` (`DEFAULT_STARTUP_READINESS_MAX_WAIT_SECS`), `rustfs/src/server/readiness.rs` (readiness responses and `Retry-After`), the `iam_bootstrap_retry_failed` log event. -> [!WARNING] -> The release that switches local SSE wrapped DEKs from the legacy -> `base64(nonce):base64(ciphertext)` representation to the versioned JSON -> envelope is a deliberate exception. Do not run that release together with -> an older RustFS version: older nodes cannot read objects written with the -> JSON envelope. Freeze every source of object mutation, including client -> writes and background lifecycle or replication work, upgrade every node, -> and then resume traffic. Downgrading or rolling back after new encrypted -> objects are written is not supported. +Upgrading the binary or container image does not change the on-disk data format unless an explicitly enabled feature documents a version floor. Replacing the executable and restarting does not run a migration step on startup. -> [!WARNING] -> `RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE` remains inactive unless -> `RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED` is also `true`. Enable -> both only after every node that can read or write object metadata supports -> the `part-checksums` sidecar and the fleet has adopted that version as its -> rollback floor. Leave either setting disabled throughout a mixed-version -> rolling upgrade. Once rebalance or decommission has migrated a legacy -> checksummed multipart object with both settings enabled, rolling back to an -> older build is not supported: older readers ignore the sidecar and can -> report an object checksum in place of the requested part checksum. +## Version floors that break mixed-version fleets -> [!WARNING] -> Writing pool metadata version 2 remains inactive unless both -> `RUSTFS_POOL_META_V2_WRITE=true` and -> `RUSTFS_POOL_META_V2_FLEET_CONFIRMED=true`. Leave either setting disabled -> until every node that can read or write `pool.bin` supports version 2. Once a node -> observes or writes version 2 it will not downgrade the file, and older -> binaries or rollback builds cannot read it. Unresolved decommission entries -> fail closed instead of being written in the version 1 format. +Each row is a feature whose activation makes older binaries unable to read what newer ones write. Keep every gate in its inactive state throughout a mixed-version rolling upgrade. -> [!WARNING] -> Pool metadata version 3 remains inactive on an existing cluster unless both -> `RUSTFS_POOL_META_V3_WRITE=true` and -> `RUSTFS_POOL_META_V3_FLEET_CONFIRMED=true`. V3 adds durable generations and a -> recoverable cross-pool commit protocol. Once committed, V1/V2-only binaries -> cannot rejoin. Follow [Pool metadata upgrade and recovery](pool-metadata-recovery.md) -> for the compatibility matrix and disk-replacement order. +| Feature gate | Activation | Consequence once active | Owning doc | +| --- | --- | --- | --- | +| Local SSE wrapped-DEK JSON envelope | The release that replaces the legacy `base64(nonce):base64(ciphertext)` representation with the versioned JSON envelope | Older nodes cannot read objects written with the JSON envelope. Freeze every source of object mutation (client writes, lifecycle, replication), upgrade every node, then resume traffic. Downgrading after new encrypted objects are written is not supported. | [compat-cleanup-register.md](../architecture/compat-cleanup-register.md) (`sse-local-dek-json-v1`), [minio-file-format-compat.md](../architecture/minio-file-format-compat.md) | +| Data-movement part checksums sidecar | `RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_WRITE=true` and `RUSTFS_DATA_MOVEMENT_PART_CHECKSUMS_FLEET_CONFIRMED=true` (inactive unless both) | Enable only after every node that reads or writes object metadata supports the `part-checksums` sidecar and the fleet has adopted that version as its rollback floor. Once rebalance or decommission has migrated a legacy checksummed multipart object with both enabled, rollback is not supported: older readers ignore the sidecar and can report an object checksum in place of the requested part checksum. | This page | +| Pool metadata version 2 | `RUSTFS_POOL_META_V2_WRITE=true` and `RUSTFS_POOL_META_V2_FLEET_CONFIRMED=true` (inactive unless both) | Once a node observes or writes version 2 it never downgrades `pool.bin`; older binaries and rollback builds cannot read it. Unresolved decommission entries fail closed instead of being written in the version 1 format. | [pool-metadata-recovery.md](pool-metadata-recovery.md) | +| Pool metadata version 3 | `RUSTFS_POOL_META_V3_WRITE=true` and `RUSTFS_POOL_META_V3_FLEET_CONFIRMED=true` (inactive on an existing cluster unless both) | Adds durable generations and a recoverable cross-pool commit protocol. Once committed, V1/V2-only binaries cannot rejoin. | [pool-metadata-recovery.md](pool-metadata-recovery.md) (compatibility matrix, disk-replacement order) | ## TL;DR -- **Rolling restart (no downtime):** restart **one node at a time**, and wait - for the restarted node to report `200` on `/health/ready` before touching - the next one. The remaining nodes keep serving traffic. -- **Sequential cold start (several nodes down):** nodes started before the - cluster has quorum come up in **degraded mode** — the process stays alive, - answers `503` with the blocking reason, and recovers **automatically** as - soon as enough peers are online. Do not restart-loop them; just keep - starting the remaining nodes. +- Rolling restart (no downtime): restart one node at a time and wait for the restarted node to report `200` on `/health/ready` before touching the next one. The remaining nodes keep serving traffic. +- Sequential cold start (several nodes down): nodes started before the cluster has quorum come up in degraded mode. The process stays alive, answers `503` with the blocking reason, and recovers automatically as soon as enough peers are online. Do not restart-loop them; keep starting the remaining nodes. ## Why a single node cannot serve alone -Erasure coding shards every object (including internal metadata such as IAM -users, groups, and policies under `.rustfs.sys`) across the drives of a set. -Reading an object back needs a **read quorum** of shards online. With the -drives of one set spread over several nodes, one node alone can never satisfy -the read quorum — this is a mathematical property of erasure coding, not a -bug. The cluster becomes readable once enough nodes are up (for internal -configuration objects, which are written with maximum parity, that is -typically about half the nodes of a set). +Erasure coding shards every object, including internal metadata such as IAM users, groups, and policies under `.rustfs.sys`, across the drives of a set. Reading an object back needs a read quorum of shards online. With the drives of one set spread over several nodes, one node alone can never satisfy the read quorum; this is a property of erasure coding, not a bug. The cluster becomes readable once enough nodes are up (for internal configuration objects, written with maximum parity, typically about half the nodes of a set). -Distributed locking similarly needs a majority of nodes' lock RPC endpoints. -The startup path no longer takes namespace locks while loading IAM -(rustfs/rustfs#4363), so IAM recovery depends only on the storage read -quorum. +Distributed locking similarly needs a majority of nodes' lock RPC endpoints. The startup path does not take namespace locks while loading IAM, so IAM recovery depends only on the storage read quorum. ## Rolling restart procedure -For each node, in any order, **one at a time**: +For each node, in any order, one at a time: 1. Restart the node (upgrade the binary/image first if this is an upgrade). -2. Wait until the node reports ready: +2. Wait until the node reports ready; a ready node returns `200` with `"ready": true` in the JSON body: ```bash curl -fsS http://:9000/health/ready ``` - A ready node returns `200` with `"ready": true` in the JSON body. 3. Only then move on to the next node. -While one node is down, the rest of the cluster keeps quorum and serves all -traffic. If you take a second node down before the first is back, some -erasure sets may lose write or even read quorum and requests start failing — -this is the situation to avoid. +While one node is down, the rest of the cluster keeps quorum and serves all traffic. Taking a second node down before the first is back can cost some erasure sets their write or even read quorum; that is the situation to avoid. ## Sequential cold start (multiple nodes down) -When the whole cluster (or several nodes) went down — power loss, host -maintenance, crash-looping deployment — and nodes are brought back one at a -time: +When the whole cluster (or several nodes) went down and nodes are brought back one at a time: -1. **Early nodes come up degraded.** The process does not exit. S3 requests - receive `503 Service Unavailable` with a `Retry-After: 5` header, an - `x-rustfs-readiness-pending` header, and a body naming the blocking - dependency: +1. Early nodes come up degraded. The process does not exit. S3 requests receive `503 Service Unavailable` with a `Retry-After: 5` header, an `x-rustfs-readiness-pending` header, and a body naming the blocking dependency: - - `storage_quorum` — waiting for enough nodes/disks for the erasure read - quorum; - - `iam` — storage is up, IAM cache is still loading; - - `startup_finalization` — last startup steps are being published. + | Blocking dependency | Meaning | + | --- | --- | + | `storage_quorum` | Waiting for enough nodes/disks for the erasure read quorum. | + | `iam` | Storage is up; the IAM cache is still loading. | + | `startup_finalization` | Last startup steps are being published. | -2. **Logs say what the node waits for.** The IAM recovery loop retries with - backoff and logs `event="iam_bootstrap_retry_failed"` with an actionable - `hint` field (for example, "storage read quorum not met yet; waiting for - enough cluster nodes/disks to come online"). After repeated failures the - log level escalates from WARN to ERROR — this still does not kill the - process. - -3. **Recovery is automatic.** As soon as enough peers are online for the - storage read quorum, the pending nodes finish IAM bootstrap on the next - retry and flip `/health/ready` to `200` on their own. No manual restart is - needed, and restarting them does not speed anything up. - -4. **Check readiness detail while waiting.** `/health/ready` (and - `/minio/health/ready`) return per-dependency detail during degradation: +2. Logs say what the node waits for. The IAM recovery loop retries with backoff and logs `event="iam_bootstrap_retry_failed"` with an actionable `hint` field (for example, "storage read quorum not met yet; waiting for enough cluster nodes/disks to come online"). After repeated failures the level escalates from WARN to ERROR; this still does not kill the process. +3. Recovery is automatic. As soon as enough peers are online for the storage read quorum, the pending nodes finish IAM bootstrap on the next retry and flip `/health/ready` to `200` on their own. Restarting them does not speed anything up. +4. Check readiness detail while waiting. `/health/ready` (and `/minio/health/ready`) return per-dependency detail during degradation; the `details` object shows `storage` / `iam` / `lock` readiness and `degradedReasons` lists machine-readable causes such as `storage_quorum_unavailable` or `lock_quorum_unavailable`: ```bash curl -s http://:9000/health/ready | jq ``` - The `details` object shows `storage` / `iam` / `lock` readiness, and - `degradedReasons` lists machine-readable causes such as - `storage_quorum_unavailable` or `lock_quorum_unavailable`. - ## Tuning -- `RUSTFS_STARTUP_READINESS_MAX_WAIT_SECS` (default `120`): how long startup - waits for full readiness before continuing in degraded mode with background - recovery. Raising it delays the listener during genuinely slow starts; - lowering it surfaces degraded mode sooner. Recovery retries continue - regardless of this limit. +| Variable | Default | Effect | +| --- | --- | --- | +| `RUSTFS_STARTUP_READINESS_MAX_WAIT_SECS` | `120` (`DEFAULT_STARTUP_READINESS_MAX_WAIT_SECS`) | How long startup waits for full readiness before continuing in degraded mode with background recovery. Raising it delays the listener during genuinely slow starts; lowering it surfaces degraded mode sooner. Recovery retries continue regardless of this limit. | -## What is *not* normal +## What is not normal -- A node process **exiting** with a fatal IAM/lock error during startup — - that fatal path was removed after v1.0.0-beta.5 (rustfs/rustfs#4304); - upgrade if you still see it. -- A node stuck degraded **after** the whole cluster is back: check network - reachability between nodes (peer RPC ports) and per-node clocks, then - inspect `degradedReasons` and the `hint` field of the IAM retry logs. -- A node shown offline in the console with no log output — tracked - separately, see rustfs/backlog#888. +- A node process exiting with a fatal IAM/lock error during startup. That fatal path was removed after v1.0.0-beta.5 (rustfs/rustfs#4304); upgrade if you still see it. +- A node stuck degraded after the whole cluster is back: check network reachability between nodes (peer RPC ports) and per-node clocks, then inspect `degradedReasons` and the `hint` field of the IAM retry logs. +- A node shown offline in the console with no log output is tracked separately (rustfs/backlog#888). diff --git a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md deleted file mode 100644 index 9ad160a7d..000000000 --- a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md +++ /dev/null @@ -1,568 +0,0 @@ -# RustFS heal & scanner vs MinIO — comprehensive parity analysis (v2, 2026-08-16) - -> English | [中文版](rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md) - -- Date: 2026-08-16 (based on that day's `main` code; audit HEAD ≈ `a118d7e4f`) -- Scope: `crates/heal` (src 19,560 lines + tests 2,274 lines), `crates/scanner` (src ~26,000 lines + tests), `crates/data-usage`, the heal/heal_walk/bitrot_self_verify and config parts of `crates/ecstore`, `crates/heal-contracts/src/heal_channel.rs`, `crates/madmin` (heal/scanner wire types), `rustfs/src` (startup wiring, admin handlers, cluster RPC) -- Parity baseline: minio/minio master (HEAD `7aac2a2c5b`; the repo has entered maintenance mode with master frozen, i.e. its final state) -- Method: four parallel audit tracks (heal crate / scanner crate / ecstore integration layer / MinIO source study), with key conclusions verified by hand one by one (points marked "verified first-hand" below were checked against the source directly) -- This document supersedes `docs/rustfs-heal-scanner-vs-minio-parity-assessment.md` (2026-06-15, v1). Since v1 there have been more than 80 heal/scanner commits (the full automatic drive-replacement healing chain, the resume state machine, making usage convergence authoritative, cluster-level heal coordination, ILM restore semantics, etc.), so v1's feature inventory and gap judgments are comprehensively outdated; v1 conclusions such as "bloom filter missing" were verified this round to be **misjudgments** (see §5.4). - ---- - -## 0. Conclusion summary - -1. **Overall verdict: the core functional chains of heal and scanner are complete.** Object-level heal (quorum arbitration + ETag fallback + bitrot Deep verification + dangling handling), erasure set deep scans (per-set disk-walk union enumeration), per-version resumable scans (schema'd persistence layer + CAS atomic publish + crash-window backfill), automatic drive-replacement healing (readiness validation + identity fencing + durable intent + completion proof), the scanner cycle loop (leader lock + persisted leader-epoch fence), data usage statistics (bucket-level/cluster-level, primary + backup + observed snapshots, epoch/cycle anti-rollback), the full ILM action set (expiry/transition/noncurrent/free-version/delete-marker cleanup), and the admin Start/Query/Cancel protocol (clientToken semantics aligned with madmin) — all of these are implemented and carry regression tests. There are **no empty implementations / early-return stubs** inside the two crates; every exceptional path has logs + metrics + error semantics. -2. **The main gaps concentrate on "entry points and the observability surface", not on the repair algorithms themselves**: the MRF/ECDecode/Metadata task executors are implemented but have no production trigger entry (`HealEvent` is entirely unwired); `CheckAbandonedParts` is `NotImplemented` at all three ecstore layers; the heal/scanner trace channels are missing; scanner excess S3 events are missing; madmin client methods are missing (only wire types exist); heal byte-level progress/ETA is not implemented. -3. **Important corrections to the v1 understanding**: the bloom filter has been **removed** from current MinIO master (`.bloomcycle.bin` stores only a cycle count), so RustFS's current state matches MinIO; the MinIO scanner is likewise a **cluster-level leader singleton**, and RustFS's leader.lock model is the same shape as MinIO's; RustFS's ETag majority-fallback arbitration is already implemented (`crates/ecstore/src/set_disk/ops/heal.rs:525-567,679`, verified first-hand) — the arbitration gap v1 worried about does not exist. -4. **RustFS exceeds MinIO in several places**: the remote_scanner RPC protocol (remote peers scan locally instead of the leader reading remote drives across the network), the persisted leader-epoch CAS fence, cycle budgets and per-set/per-disk concurrency gates, the pending-heal ledger, the durable replacement intent + completion proof state machine, foreground pressure gating (mainline throttle), and the cluster heal control coordinator + envelope replay protection. -5. Gap severity tally: 8 P1 items (behavioral/operational alignment gaps), 9 P2 items (completeness), 3 P3 items (cleanup/low risk), and 7 items of "not pursuing parity by design". Full list in §6. - ---- - -## 1. Architecture overview - -### 1.1 RustFS's three-layer architecture - -RustFS splits the heal/scanner functionality that MinIO keeps inside the `cmd/` monolith into three layers plus two standalone crates: - -| Layer | Location | Responsibilities | -|---|---|---| -| Primitives layer | `crates/ecstore/src/set_disk/ops/heal.rs` (~3,240 lines), `ops/heal_walk.rs`, `ops/bitrot_self_verify.rs`; upper wrappers `store/heal.rs`, `store/heal_walk.rs`, `core/sets.rs` | Object/bucket/format/replacement-drive format repair, disk-walk union enumeration, write-path bitrot self-verification; the `rustfs_storage_api::HealOperations` contract is implemented by `SetDisks`/`Sets`/`ECStore` (`crates/storage-api/src/object.rs:503-519`) | -| heal runtime | `crates/heal` | Process-level HealManager (priority queue/scheduler/auto disk scanner/resumable resume), HealChannelProcessor (consumes the global heal channel), drive-replacement recovery state machine | -| scanner runtime | `crates/scanner` | Data usage scanning, ILM evaluation and enqueueing, heal candidate production, replication usage statistics, remote scanner RPC | -| Shared protocol | `crates/heal-contracts/src/heal_channel.rs` (~776 lines) | Start/Query/Cancel command channel, `HealOpts`/`HealScanMode`/`HealRequestSource`/`HealAdmission*` shared types, `HealResultItem` (madmin) | -| Shared data | `crates/data-usage` | `DataUsageEntry/Info`, histograms, `hash_path`; produced by the scanner, consumed by ecstore/admin | - -Startup chain (wiring verified first-hand): - -1. `rustfs/src/startup_services.rs:93` → `init_background_service_runtime(store)`. -2. `rustfs/src/startup_background.rs:41-81`: create the global heal service cancel token; read `RUSTFS_SCANNER_ENABLED` (alias `RUSTFS_ENABLE_SCANNER`, default true) and `RUSTFS_HEAL_ENABLED` (alias `RUSTFS_ENABLE_HEAL`, default true); **the heal manager is initialized whenever either heal or scanner is enabled** (heal candidates produced by the scanner need a consumer; with both off, the heal channel is not initialized and `send_heal_request` reports "Heal channel not initialized"). -3. `crates/heal/src/lib.rs:142-216`: atomic initialization inside an owned task (a caller cancel cannot leave a half-initialized manager behind, `lib.rs:123-131`; `GLOBAL_HEAL_RUNTIME_INIT` mutex single-flight) → `HealManager::start()` → `rustfs_common::heal_channel::init_heal_channels()` → spawn `HealChannelProcessor::start_with_receipts`. -4. `crates/heal/src/heal/manager.rs:1301-1356` `HealManager::start`: `start_scheduler()` (`manager.rs:2394-2461`, interval default 10s + `Notify` event-driven wakeup) → `process_unclean_shutdown()` (`manager.rs:1362-1695`) → when `enable_auto_heal` (default true), `start_auto_disk_scanner()` (`manager.rs:2464-2999`). -5. After the server is ready, `rustfs/src/startup_lifecycle.rs:150-152`: when `enable_scanner`, `init_data_scanner(token, store)` (`crates/scanner/src/scanner.rs:1293-1372`). -6. Graceful shutdown: `rustfs/src/startup_shutdown.rs:308` `shutdown_ahm_services()` (cancel token); `:414` `clear_unclean_shutdown_markers()`. - -### 1.2 MinIO's corresponding structure (final master state) - -| MinIO file | Responsibilities | -|---|---| -| `cmd/admin-heal-ops.go` | Manual admin heal sequence (healSequence, clientToken/forceStart/forceStop) | -| `cmd/global-heal.go` | Resident background heal queue (newBgHealSequence, token fixed `0000-…`, never ends) + `healErasureSet` (full-object heal per set) | -| `cmd/background-heal-ops.go` | healRoutine worker pool (`_MINIO_HEAL_WORKERS`, default GOMAXPROCS/2) consuming healTask | -| `cmd/mrf.go` | MRF (Most Recent Fail) queue (capacity 100,000), persisted at process exit to `.minio.sys/buckets/.heal/mrf/list.bin` with startup replay | -| `cmd/background-newdisks-heal-ops.go` | Automatic resync for new/replaced drives (monitorLocalDisksAndHeal 10s polling + healFreshDisk + healingTracker) | -| `cmd/erasure-healing.go` / `erasure-healing-common.go` | Object-level heal core (~800 lines), listAndHeal | -| `cmd/data-scanner.go` | Scanner loop (globalLeaderLock cluster singleton) + folderScanner + applyActions | -| `cmd/erasure.go` (nsScanner) / `erasure-server-pool.go` | NSScanner three-layer structure | -| `cmd/bucket-lifecycle.go` | ILM executor (expiry/transition worker pools) | -| `cmd/xl-storage.go` | DiskInfo.Healing, CheckParts/VerifyFile, CleanAbandonedData, RenameData healing branch | -| `cmd/prepare-storage.go` | waitForFormatErasure new-drive startup handshake | - -### 1.3 Architecture-level differences (design trade-offs, not defects) - -1. **heal queue model**: MinIO funnels every heal (scanner sampling/MRF/admin/new-disk resync) into a single channel + a fixed worker pool (new-disk resync additionally has a per-drive worker pool); RustFS is a multi-policy scheduler built from a priority heap + dedup-merge + capacity-tiered dropping + per-set bulkhead + foreground pressure gating (`manager.rs:3003-3420`). RustFS is more expressive, at the cost of an observability question around "duplicate requests being merged" (already pointed out in v1; the current `HealAdmissionReceipt` canonical task_id + alias mechanism answers it, `manager.rs:1759-1846`). -2. **scanner remote-drive access**: the MinIO leader transparently reads and writes remote-node drives through the disk abstraction layer; the RustFS leader pushes scan execution down to the remote peer to run locally via the remote_scanner RPC (`crates/scanner/src/remote_scanner.rs`), with only results and progress heartbeats sent back. Both are cluster single-leader. RustFS's approach saves the leader↔remote metadata read amplification, at the cost of maintaining a separate RPC protocol (HMAC per-frame authentication, session replay cache, fence re-validation, `remote_scanner.rs:52-61,405-496,1024-1065`). -3. **heal state persistence**: MinIO uses a single file `.healing.bin` (msgp healingTracker, reset whenever the diskID mismatches); RustFS uses a schema'd multi-file layout (resume/checkpoint/intent/seal/proof, each CAS-published, `resume.rs:38-61`), with the crash window explicitly backfilled (`erasure_healer.rs:389-402`, `resume.rs:1027-1057`). -4. **write-path self-protection**: MinIO relies on background heal to converge after writes; RustFS, after the commit rename in PutObject/CompleteMultipartUpload, actively checks `convergence.needs_heal()` and immediately enqueues an object heal (`set_disk/ops/object.rs:2291-2306`, `ops/multipart.rs:2574-2589`), and additionally has read repair (`io_primitives.rs:1040-1160`). - ---- - -## 2. Heal implemented-feature panorama - -### 2.1 Task types (`HealType`, `crates/heal/src/heal/task.rs:85-111`) - -| Type | Semantics | Executor | Production trigger | -|---|---|---|---| -| `Cluster` | all buckets healed in turn (structure + optional recursive objects), in-batch retry ≤3 | `heal_cluster` task.rs:1420-1490 | channel: empty bucket means Cluster (channel.rs:576-577) | -| `Object{bucket,object,version_id}` | single object/version; when absent, rebuild per `recreate_missing` or error out | `heal_object` task.rs:855-1146 | admin, scanner, read-repair, write-path convergence, add_partial | -| `Bucket{bucket}` | bucket metadata/structure; `recursive` additionally walks all object versions | `heal_bucket` task.rs:1284-1418 + `heal_bucket_objects` task.rs:1508-1698 | admin (POST /v3/heal/{bucket}), scanner `build_bucket_heal_request` | -| `Prefix{bucket,prefix}` | recursive by prefix | `heal_prefix` task.rs:1492-1506 | channel: `recursive && prefix` non-empty (channel.rs:578-585) | -| `ErasureSet{buckets,set_disk_id}` | format repair + healing marker + per-bucket preprocessing + resumable per-version deep scan | `heal_erasure_set` task.rs:2158-2642 | admin (pool/set params), auto disk scanner, unclean shutdown, renew_disk, durable replacement recovery | -| `Metadata{bucket,object}` | metadata only (Deep, does not rebuild data) | `heal_metadata` task.rs:1700-1859 | **no production trigger** (§6 HS-01) | -| `MRF{meta_path}` | failure-path-driven Deep repair (recursive+update_parity) | `heal_mrf` task.rs:1861-1992 | **no production trigger** (only `HealEvent` can generate it, unwired) | -| `ECDecode{bucket,object,version_id}` | EC decode rebuild (Deep+recreate+update_parity), Urgent priority | `heal_ec_decode` task.rs:1994-2156 | **no production trigger** (only `HealEvent` can generate it, unwired) | - -Priorities `Low/Normal/High/Urgent` (task.rs:168-179); state machine `Pending/Running/Retrying/Completed/Failed/Cancelled/Timeout` (task.rs:225-241). - -### 2.2 Trigger-path panorama (beyond admin) - -| Channel | source | Priority | Evidence | -|---|---|---|---| -| Scanner periodic sampling (1/1024, `RUSTFS_HEAL_OBJECT_SELECT_PROB`) | Scanner | Low | `scanner_folder.rs:2117-2136`, `:1150`; `remove_corrupted=HEAL_DELETE_DANGLING(true)`, `recreate_missing=false` (`common/heal_channel.rs:24`, `scanner_folder.rs:510-511`) | -| Scanner metadata corruption (get_size failure classified HealMetadata) | Scanner | High | `scanner_folder.rs:2147-2208`, `:1244-1260` | -| Scanner abandoned children (present in cache, absent on disk, list_path_raw quorum verification) | Scanner | High (bucket-level + object-level) | `scanner_folder.rs:2528-2792` | -| Scanner pending-heal ledger retry (persisted after rejection by a full heal channel, ≤128 per bucket per round, 10k cap) | Scanner | original priority | `scanner_folder.rs:1721-1763`, `:99-100` | -| auto disk scanner (unformatted drive confirmed via replacement_readiness / `runtime_state=="returning"` drive / durable-intent re-entry) | AutoHeal | Low | `manager.rs:2464-2999` | -| unclean shutdown recovery (startup reads the `unclean-shutdown` marker → ErasureSet heal for all local sets) | AutoHeal | Low | `manager.rs:1362-1695` | -| write-path convergence (after PutObject/CompleteMultipartUpload, `convergence.needs_heal()`) | Internal | Normal | `set_disk/ops/object.rs:2291-2306`, `ops/multipart.rs:2574-2589` | -| partial-object heal (add_partial) | Internal | Normal | `set_disk/ops/object.rs:5808-5825` | -| stale data-directory cleanup leftover enqueue | Internal | Normal | `set_disk/core/io_primitives.rs:3880-3907` | -| read repair (metadata_read_error / missing_shards / decode_error, TTL dedup cache) | ReadRepair | Low | `set_disk/read.rs:407,995,1079` → `submit_read_repair_heal` (`io_primitives.rs:1105-1160`), `recreate_missing=true` | -| drive reconnect hits UnformattedDisk → send_heal_disk | AutoHeal | Normal | `set_disk/ops/locking.rs:339-347` | -| Admin API (incl. cluster coordinator routing) | Admin | High | `rustfs/src/admin/handlers/heal.rs:174-212`, `:771-930` | -| cluster RPC heal (peer invocation) | — | — | `rustfs/src/storage/rpc/node_service/heal.rs`, `ecstore/src/cluster/rpc/peer_s3_client.rs:296,1209` | - -Note: MinIO's MRF channel (read-path immediate delivery on missing/corrupt parts + queue persistence + shutdown replay, `cmd/mrf.go`, `erasure-object.go:395-410,800-812`) is **partially replaced** in RustFS by read-repair + write-path convergence; the three executors `HealType::MRF`/`ECDecode`/`Metadata` have no production entry (see §6 HS-01 for details). - -### 2.3 Object-level heal semantics (ecstore `set_disk/ops/heal.rs`) - -Flow (`heal_object_with_explicit_version_regen` from :426): - -1. Take the object write lock (unless `no_lock`); an `object` ending with `/` goes through object-directory heal (`heal_object_dir_locked` :1587-1717: dangling determination + `remove` deletion + missing-volume rebuild). -2. `read_all_fileinfo` reads xl.meta from all disks; all-not-found is treated as already deleted and returns. -3. **quorum arbitration + ETag fallback** (verified first-hand): `list_online_disks` treats the mod-time quorum as authoritative; when quorum fails it falls back to ETag majority arbitration (`:525-567` `filter_by_etag`/`quorum_etag`); `pick_valid_fileinfo` picks the canonical metadata; the cannotHeal determination for "number of bad-meta disks > parity" is waived when the ETag agrees across all disks (`:679`). Matches MinIO's dual arbitration in `filterDisksByETag`. -4. `disks_with_all_parts` (:562-572) validates parts per `scan_mode`: **Normal only stats (CheckParts semantics), Deep does full bitrot verification (VerifyFile semantics)**; when a Normal scan detects `FileCorrupt` it automatically escalates to Deep and retries once (`:2022-2031`, same shape as MinIO erasure-healing.go:1101-1106); a no-parity object (EC:0) with a bitrot failure is judged unrecoverable (`:700-726`). -5. `should_heal_object_on_disk` (:606-650) classifies each disk as missing/corrupt/offline/outdated → rebuild: per-part bitrot reader/writer (using per-part checksum + algorithm), write into a temporary volume then rename to commit (`HEAL_RENAME_INCOMPLETE` retry semantics :24); dangling-deletion safety check `dangling_delete_safety` (:1488); **orphan data-directory reclamation `reclaim_orphan_data_dirs_best_effort` (:1428)** — this part covers the main scenarios of MinIO's `CleanAbandonedData` (but there is no standalone `CheckAbandonedParts` API, see §6 HS-02). -6. Versioned objects: enumerate "every version" (`storage.rs:1494-1530`); the delete-marker path is decided by `latest_meta.deleted` (`storage.rs:262-277` comment); regression tests `tests/heal_b5_versioned_regression_test.rs:282,334`. -7. Explicit-version rebuild `try_regenerate_explicit_version_meta` (:1318); cleanup of local leftovers of transitioned objects. -8. The write path additionally has shard-level bitrot self-verification `verify_written_bitrot_shards` (`ops/bitrot_self_verify.rs:45-129`, HighwayHash256S, verifying freshly written shards right before the final rename, serving the EC:0 no-parity case) — **note this is not background bitrot patrol**; background patrol is carried by scanner bitrot_cycle-driven Deep heal. - -heal-crate-side wrapper (`task.rs:855-1146`): existence check (transient errors become `TransientSkip` to avoid false failures :551-569); scanner synthetic-directory normalization (:1148-1180); `recreate_missing` rebuild (:1183-1282); data-usage-cache object-lock timeout exemption (:571-653); not-found → treated_as_deleted success (:1012-1029); results `HealResultItem` keep at most 1024 entries + truncated flag (:50,845-852). - -Recursive walk (`heal_bucket_objects` task.rs:1508-1698): paginated enumeration of all versions including delete markers, transient-error exponential-backoff retry ≤3 (2^n + jitter :620-627), failure-sample log truncation ≤5 entries, aggregated `BatchHealFailure`. - -### 2.4 erasure set heal and resumable scans - -`heal_erasure_set` (task.rs:2158-2642) runs in four phases (4-step progress tracking): - -1. **Replacement intent and recovery-drive selection** (AutoHeal only + non-empty heal_endpoints): reuse the drive holding the durable intent / exclude the target endpoints and pick surviving drives; already-completed generations get an idempotent CleanupPending wrap-up. -2. **Format repair**: `heal_replacement_format(dry_run, pool, set, targets)` (`storage.rs:1372-1384`, trait default fail-closed); per-target-drive results must all be ok (`erasure_healer.rs:97-102`) + identity-fence re-check (task.rs:2410-2420). -3. **healing marker**: write an owner CAS marker `{set_disk_id}:{task_id}` to the target drive (`mod.rs:80-229`, CAS + rollback + unique concurrent owner), which makes `DiskInfo.healing` true (assignment chain verified first-hand `set_disk/mod.rs:4988`). -4. **Per-bucket preprocessing + resumable deep scan**: `ErasureSetHealer::heal_erasure_set` (`erasure_healer.rs:242-278`). - -`ErasureSetHealer` scan details (benchmarked against MinIO `healErasureSet`; the `heal_walk.rs:15-23` module comment explicitly cites MinIO `global-heal.go`'s listPathRaw + objQuorum=1 + mergeXLV2Versions): - -- **Enumerator choice (backlog#920)**: Deep or AutoHeal → per-set **disk-walk union enumeration** `list_versions_for_heal_page_disk_walk` ("exists on any drive" means sub-quorum reconstructible; `storage.rs:1559-1644`, page bounds 1,000 objects/10,000 versions, `dw1:` cursor); ordinary requests go through read-quorum `list_object_versions`. -- **Resume cursor**: the authoritative cursor is an opaque continuation token (`v1:` = marker JSON, `dw1:` = disk-walk key; the two namespaces are mutually exclusive against misreads, `storage.rs:81-260`); after each completed page, persist the cursor first, then clear the dedup set (`erasure_healer.rs:922-927`). -- **In-page concurrency**: FuturesUnordered + Semaphore, default `RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY=8`, Deep/AutoHeal forces 1 (`erasure_healer.rs:105-142`). -- **per-version dedup**: `compose_key` length-prefix injection encoding (`resume.rs:281-288`). -- **Error classification**: truly absent (FileNotFound etc.) → Absent (counted as success); infrastructure-transient (quorum/DiskNotFound/SlowDown etc.) → Transient (counted as skipped); everything else Failed (`erasure_healer.rs:148-182`; the comment cites backlog#856/#799 B7: offline drives must not be recorded healed/absent). -- **Loop protection**: abort when an empty page is truncated or the page-tail version identity does not advance (:933-949). -- **Completion determination**: if any of failed/skipped/failed_buckets is >0, do not mark complete; `schedule_retry()` resets both the resume and checkpoint layers (:561-626; backlog#855/B6/#1033: a skip round must not be marked complete). -- **Replacement-drive commit proof**: physical read-back on the target endpoints `replacement_targets_have_version` (`ops/heal.rs:340-412`); unconfirmed → transient skip. - -### 2.5 Automatic drive-replacement healing (replacement recovery) - -- **Identification** (`replacement_readiness.rs:25-73`): `replacement_mount_lease_root()` exists, canonicalize succeeds, is a mount point, the physical device id is non-empty, disjoint from the root device, and shares no physical device with sibling drives (Linux uses /proc/self/mountinfo mount-id+dev+ino). The non-root mount check has a regression test (`manager.rs:3549`). -- **State machine** (`resume.rs:63-73`): `Intent → Rebuilding → (write proof) Verified → CleanupPending → cleanup`; `Abandoned` is a terminal state; state transitions write the persistence layer first, then mutate (`save_state_strict`). -- **Persistence** (`resume.rs:38-61`, schema ResumeState=5/Checkpoint=5/proof=1): `{task_id}_ahm_resume_state.json`, `_ahm_checkpoint.json`, and intent/seal/completion_proof under the `buckets/ahm-replacement/` namespace; torn write + no seal is recognizable and rebuilt atomically (:1316-1338); CAS publish, refuses to overwrite a concurrently valid proof (:1512-1585). -- **Recovery**: both unclean shutdown and the periodic scan recover unfinished/pending-cleanup replacement generations from surviving drives (`manager.rs:1435-1640,2663-2815`); multi-generation conflict / validation failure → freeze that set (`replacement_recovery_blocked_sets`, `manager.rs:69-87,2782-2815`). -- **External snapshot**: `current_replacement_recovery_snapshot` (`lib.rs:262-333`) merges local surviving-drive records; conflict → Unknown / non-definitive; admin `GET /v4/heal/replacement-recovery`. - -### 2.6 Scheduler (manager.rs) - -- Priority heap + FIFO within the same priority (:148-191,330-347); dedup key per type (:469-506); enqueue three-state dedup active→queued→retrying (:1759-1785); duplicates default to Merged and return the canonical task_id (`HealAdmissionReceipt`, :1821-1846) + client token alias (:1219-1246). -- Capacity: when the queue is full, best-effort sources (Scanner/AutoHeal/ReadRepair) or low-priority items get Dropped(QueueFull); Admin/Internal may evict queued lower-priority items (`push_displacing_lower_priority` :353-396); 80%/95% tiered pressure handling (:885-909). -- Concurrency: global `max_concurrent_heals` (default 4) + per-set bulkhead `max_concurrent_per_set` (default 1) (:3040-3073,3434-3447). -- Foreground pressure gating, mainline throttle: delay best-effort tasks when foreground read/write permit utilization is ≥80% (:919-1009,2999-3020). -- Timeout: task-level aggregate timeout (default 300s), remaining budget preserved across retries (task.rs:444-451, PR #6101). -- Recoverable retry: `is_recoverable_heal()` (error.rs:83-136) ≤3 attempts, 2^n backoff capped at 30s; retries hold ownership inside a standalone backoff task (:3235-3382). -- Completion states are retained for 10 minutes for querying (:42). - -### 2.7 Admin API and cluster coordination - -- Routes (`rustfs/src/admin/handlers/heal.rs:174-212`): `POST /rustfs/admin/v3/heal/`, `/heal/{bucket}`, `/heal/{bucket}/{prefix}` (the same POST distinguishes start/query/cancel by the query `clientToken/forceStart/forceStop`, aligned with mc admin heal semantics); `POST /v3/background-heal/status`; `GET /v4/heal/replacement-recovery`. Permission `HealAdminAction` (route_policy.rs:334-341). -- Cluster coordination (heal.rs:771-930 + `node_service.rs:514-606`): `heal_topology_fingerprint` + deterministic-by-topology coordinator-node selection + coordinator epoch; envelope validation + SHA256 digest replay protection; when the coordinator is not local, go through peer gRPC `heal_control`; `probe_heal_control` capability probe (rolling-upgrade scenario). -- Request: the body is `HealOpts` (`recursive/dryRun/remove/recreate/scanMode(0/1/2)/updateParity/nolock/pool/set`, serde camelCase, fields aligned with madmin.HealOpts); a root heal start requires `recursive=true` or a `pool+set` pair; body cap 1MB. -- Response: `HealStartSuccess{clientToken, clientAddress, startTime}`; `HealTaskStatus{summary, detail, startTime, settings, items, truncated, progress}` (summary ∈ running/finished/stopped/notFound); `BackgroundHealStatus` (bitrot start time/cycle/current mode + `disabled/uninitialized/idle/active/degraded` states — an unreachable peer is explicitly degraded rather than impersonating idle, issue #5850) + `healOperations` as a priority×source matrix + cluster progress. -- `HealResultItem`/`HealDriveInfo`/`HealItemType`/DriveState enums are JSON-compatible with madmin (`crates/madmin/src/heal_commands.rs:19-65`). -- A status payload over 8MiB is truncated by halving (channel.rs:37,73-104); path-token validation (wrong token rejected; an empty path matches Cluster only). - -### 2.8 heal metrics and logs - -Metrics: `rustfs_heal_admission_total{source,result,reason,context}`, `rustfs_heal_task_start_total`, `rustfs_heal_task_running{type,set}`, `rustfs_heal_queue_delay_seconds`, `rustfs_heal_scheduler_skip_total`, `rustfs_heal_mainline_throttle_total`, `rustfs_heal_page_concurrency_current{set}`, `rustfs_heal_candidate_enqueue/merge/drop/priority_reject_total`, `rustfs_heal_read_repair_dedup_total{reason}`, etc. All logs are structured event style (PR #5720); per-object logs are demoted to prevent storms (`demote_to_debug_when!`, #5716/#5719/#5727). - ---- - -## 3. Scanner implemented-feature panorama - -### 3.1 Loop, leader, immediate triggering - -- **Cluster single leader**: distributed ns write lock `leader.lock` (`scanner.rs:3156-3207`, timeout default 5s) + **persisted leader-epoch CAS fence**: the leader writes (cycle, leader_epoch) encoded as `RSCYC001` into `.bloomcycle.bin` using an ETag precondition (`scanner.rs:118,1850-1861,2177-2334`); usage snapshots additionally carry an epoch fence (:2087-2153). Lock lost → cancel the current cycle, converging within 30s (:108-111,2623-2642). -- One round executes immediately after the lock is acquired; cycle = `RUSTFS_SCANNER_CYCLE` > config cycle > start_delay > deployment default > speed tier (±10% jitter, floor 1s). -- **clean-idle exponential backoff**: consecutive fully-clean idle intervals double (capped at 24h; bitrot-cycle compression cap; disabled when a bucket has active lifecycle/replication rules, :383-456,1382-1512). -- **superseded/deferred backoff**: exponential backoff from 5s capped at 30min (:105-106,3432-3438); maintenance probing failures get an independent backoff (:459-505). -- **Immediate wakeup**: ① dirty-usage fast path — write-path put/delete/multipart/bucket operations call `record_dirty_usage_bucket` (`scanner_io.rs:222-235`; call sites include `rustfs/src/app/object_usecase.rs:6221`), bump the generation and Notify-wake the leader; dirty buckets are queued first (`scanner_io.rs:462-488`); ② maintenance-config changes (lifecycle/replication settings call `record_scanner_maintenance_change`); ③ runtime-config hot updates generation+Notify; ④ cluster activity snapshot changes. -- **Cluster coordination**: `probe_scanner_activity` gathers this node's and peers' `ScannerNodeActivity` (instance_id/namespace_generation/maintenance_generation/protocol_version/topology_digest/data_movement_active/dirty usage); the topology digest covers pools/sets/drives URLs; a mismatched protocol version refuses to share the cache lock (`scanner.rs:970-1068`); **cycles are deferred during data movement (rebalance/decommission)** (`scanner_io.rs:2226-2374`); at cycle end, per-peer RPC confirms the dirty-usage ack (`scanner.rs:2925-2952`). - -### 3.2 Traversal model - -- The main traversal is a **full directory walk** (tokio::fs::read_dir recursion, `scanner_folder.rs:1915-2234`), not via metacache; metacache/`list_path_raw` is used only for the abandoned-children cross-drive verification (:2528-2792). -- Three-level concurrency: leader → per-set (semaphore default 4) → per-disk bucket scans (default 4) → single-drive recursion; a cache lock per bucket per set `.scanner-cycle.lock.pool-N.set-M` (losing the lock cancels that bucket's scan; lock contention re-queues); single-scan admission per drive (local drives also go through the semaphore, `scanner_io.rs:3246-3274`). -- Bucket ordering: after shuffle, re-ordered as dirty → uncached → cached (`scanner_io.rs:2947-2949,462-488`); entries within a directory sorted by name + resume-hint rotation (`scanner_folder.rs:333-359`). -- **Resumable scanning**: `DataUsageScanCheckpoint{version,resume_after,reason}` persisted in the cache info (`data_usage_define.rs:68,293-307`); written on budget exhaustion/cancel; resumption has Used/Stale/NoHint metrics; the resume unit is a directory (no cross-cycle object-level pagination). -- Erasure semantics: finding `xl.meta` marks an object boundary with no descent; at most 64 UUID data-dir candidate entries probed; data without metadata → record failed + high-priority heal; symlink directories ignored / cycles skipped. -- Cooperative yielding: `yield_now` every N objects (default 128). - -### 3.3 Large-bucket skip strategy (benchmarked against MinIO compaction) - -1. Cache-currency reuse: if the bucket and scan plan are unchanged (name/source/snapshot_complete/plan digest/next_cycle/leader_epoch/cache_key_format all match), the whole bucket is skipped (`scanner_io.rs:1062-1109`). -2. compacted-directory 16-cycle rotation window: rescan only when `hash mod (next_cycle, 16)` hits, otherwise copy from the old cache (`scanner_folder.rs:74,2429-2442`). -3. compaction thresholds: children <500 or pure-object leaves compress into a single entry; subfolders ≥2500 (root 10000) pre-compressed; children ≥10000 reduced (:75-78,2314-2340,2846-2887). -4. failed-object TTL skip: 86400s / at most 10,000 entries (:88-91,1354-1381). - -Compared with MinIO master: MinIO's skip strategy is likewise hash-mod-16 cycles + a compaction threshold tree (500/10000/2500), and the **bloom filter has been removed from master**. RustFS's constants and structure share the same origin as MinIO's current state (MinIO does not adopt cross-drive dirty-generation prioritization; RustFS additionally has two more skip layers — plan digest and cache-currency validation). - -### 3.4 data usage statistics - -- Dimensions: per-directory entry (size/objects/versions/delete_markers/size histogram/version histogram/replication stats/failed_objects/per-tier stats/children/compacted, `data-usage/src/data_usage.rs:661-679`); per-object SizeSummary (incl. per-ARN replication-target stats and tier stats; tier classification: fully transitioned counts toward its tier, otherwise by storage class; free versions not counted); bucket-level `BucketUsageInfo`; cluster-level `DataUsageInfo` (incl. scanner_cycle/scanner_epoch fence + usage_snapshot_complete). -- Storage: per bucket per set `{bucket}/.usage-cache.bin` (primary + `.bkp` backup + CAS retry); the authoritative cluster snapshot `buckets/data-usage/data-usage.json` (`.bkp` synced every 10 cycles, legacy path compatible); stale snapshots rejected on write (triple epoch/cycle/last_update determination); observation snapshots superseded by a race are stored separately as `data-usage-observed.json`. -- Consumption: `replace_bucket_usage_memory_from_info` refreshes bucket-usage memory + two-level cache invalidation (`scanner.rs:4142-4152`) → bucket stats/quota/admin account_info/system; the write path overlays memory in real time; at startup, reading the snapshot detects a cold cache and skips startup delay. -- Incomplete multipart uploads are not counted (consistent with MinIO, which also does not scan the multipart bucket). - -### 3.5 ILM integration - -- Per object `ScannerItem::apply_actions` (`scanner_folder.rs:747-1032`): `Evaluator::new(lifecycle).with_lock_retention(...).with_replication_config(...).eval()` batch evaluation. -- Implemented actions (the full IlmAction set, `scanner-contracts/src/metrics.rs:34-45`): expiry deletes (Delete/DeleteRestored/DeleteRestoredVersion), all-versions deletes (DeleteAllVersions/DelMarkerDeleteAllVersions, stop further versions after handling), transition (Transition/TransitionVersion, tier list read at runtime), noncurrent batches (DeleteVersionAction → `enqueue_by_newer_noncurrent`), free-version cleanup (`enqueue_free_version`), object-lock retention constraints. **A one-to-one mapping onto MinIO's 9 ILM actions.** -- Execution model: the scanner is the "discover and enqueue" role (the expiry/transition queues live in ecstore `bucket_lifecycle_ops.rs`); actions are consumed by worker pools — the same shape as MinIO's globalExpiryState/globalTransitionState. -- AbortIncompleteMultipartUpload is not executed inside scanner/ILM (MinIO likewise: `internal/bucket/lifecycle/rule.go` has a FIXME, and it is actually carried by the `erasureSets.cleanupStaleUploads` global routine); in RustFS it is an independent ecstore background task `init_background_stale_multipart_upload_cleanup` (`bucket_lifecycle_ops.rs:3289-3320`) + on-demand at bucket deletion. -- Integration-test coverage: transition+restore, free-version, noncurrent, delete-marker, 0-day, background-scan expiry (`scanner/tests/lifecycle_integration_test.rs:1071-2095`). - -### 3.6 heal candidate production (scanner side) - -- Sampling: `hash mod_alt(next_cycle/prob_div, 1024/prob_div)`; when rescanning via the compacted branch, prob_div=16 gives an equivalent ×16 probability (the same compensation as MinIO, `scanner_folder.rs:125-127,2117-2122`). -- deep/normal: cycle-level `get_cycle_scan_mode` (bitrot_cycle default 30d, `scanner.rs:1626-1657`) → object-level with `HealScanMode::Deep`; fresh objects (modified within 60s) are demoted to Normal (:146-155); state persisted in `.background-heal.json` (`BackgroundHealInfo{bitrot_start_time,bitrot_start_cycle,current_scan_mode}`, same path and structure as MinIO). -- The scanner only enqueues, never executes inline (inline heal was removed; the compat flag only warns, `scanner_folder.rs:411-427`); `HealScanMode::Deep` is just a marker — the bitrot-verification read happens at the heal consumer (the ecstore Deep path). -- Metadata corruption → high-priority heal (`classify_get_size_failure` → HealMetadata); abandoned children → list_path_raw quorum verification + bucket-level/object-level high-priority heal; healing drives get sticky skipping (`should_heal` :1628-1648). -- pending-heal ledger: candidates rejected by a full heal channel are persisted into the cache info and retried next round. -- Replication heal: `queue_replication_heal` → the replication queue (going through the replication channel, not the heal channel); per-ARN replication usage statistics. - -### 3.7 remote_scanner RPC protocol (RustFS-specific) - -Requests ≤16KB msgpack (version/request_id/server_epoch/session_id/session_sequence/bucket/next_cycle/leader_epoch/scan_plan_digest/skip_healing/scan_mode/budget); frames ≤2MB, HMAC-SHA256 per-frame authentication (domain `rustfs-ns-scanner-frame-v3`); progress heartbeats 1s (250ms in budget mode); phase announcements Scanning→Persisting; RPC lifetime cap 24h, disconnect grace 2min; anti-replay session+sequence cache (capacity 65536); the server validates leader-fence and persisted-cycle consistency + fence re-validation every 5s; results Complete/Partial/NamespaceNotFound/CycleAhead; remote drives without v4-protocol support fall back to the leader scanning locally (`remote_scanner.rs` whole file; `scanner_io.rs:2750-2812`). - -### 3.8 Rate limiting / budgets / hot updates / observability - -- DynamicSleeper proportional backoff (speed tiers fastest/fast/default/slow/slowest, same five-tier parameters as MinIO); idle_mode master switch; an extra backoff capped at 250ms per request (10ms base) driven by foreground S3 read traffic. -- Cycle budget ScannerCycleBudget: max_duration/max_objects/max_directories (default 0 = unlimited); partial cycles still advance the cycle count. -- runtime_config with three-layer sources (env > config > default) and per-field source markers (Env/Config/ScannerCompatConfig/Default); admin `PUT /v3/config` hot update → generation+Notify takes effect immediately; `GET /v3/scanner/status` returns enabled/freshness(fresh/stale/unknown)/metrics/cycle_schedule/runtime_config; `GET /v3/ilm/expiry/status` returns expiry queue/workers/missed/blocked. -- Metrics: leader lock; cycle complete/partial/deferred/superseded; versions scanned; per-source (Usage/Lifecycle/BucketReplication/SiteReplication/Heal/Bitrot/Alerts) checked/executed/queued/missed; checkpoint set/used/stale; current path (per-disk+bucket in real time); cache save series; concurrency series; alerts (excess versions/version size/folders). - ---- - -## 4. Item-by-item parity versus MinIO - -### 4.1 heal trigger-channel comparison - -| MinIO channel | RustFS counterpart | Status | -|---|---|---| -| A. Manual admin heal (healSequence, clientToken/forceStart/forceStop) | heal channel Start/Query/Cancel + cluster coordinator + envelope replay protection | ✅ equivalent and enhanced (cluster routing); sequence-semantics differences in §6 HS-06 | -| B. Resident background heal queue (newBgHealSequence + healRoutine worker pool) | HealManager resident scheduler + priority queue + bulkhead | ✅ equivalent and enhanced | -| C. Automatic new/replaced-drive resync (monitorLocalDisksAndHeal 10s + healFreshDisk + healingTracker + waitForFormatErasure handshake) | auto disk scanner (10s) + replacement_readiness + durable intent/proof state machine + heal_replacement_format | ✅ equivalent and enhanced (identity fence + completion proof; MinIO's tracker is stronger on external visibility, see §6 HS-07) | -| D. MRF (100k queue + persisted list.bin + shutdown replay + read-path corrupt delivery) | read-repair (Low + TTL dedup) + write-path convergence heal carry it partially; the `HealType::MRF` executor has no production entry | ⚠️ partially equivalent (§6 HS-01) | -| E. Scanner sampled heal (1/1024 + compacted ×16 compensation) + abandoned children | the same sampling + ×16 compensation + abandoned children + pending-heal ledger | ✅ equivalent and enhanced (the ledger) | -| F. Read-path inline trigger → MRF (GetObject part missing/corrupt, metadata rebuild missingBlocks>0) | read repair (three entries: missing_shards/decode_error/metadata_read_error) | ✅ equivalent (enqueued into the heal queue rather than the MRF queue) | - -### 4.2 Object-level heal semantics comparison - -| Feature | MinIO | RustFS | Status | -|---|---|---|---| -| mod-time quorum arbitration | listOnlineDisks | same | ✅ | -| ETag majority fallback (clock drift) | filterDisksByETag | `filter_by_etag`/`quorum_etag` (heal.rs:525-567) | ✅ verified first-hand | -| cannotHeal ETag waiver | waived on all-consistent ETag retry | heal.rs:679 | ✅ | -| Normal=CheckParts (stat) / Deep=VerifyFile (bitrot) | yes | `disks_with_all_parts` by scan_mode (ops/heal.rs:562-572,978-1024) | ✅ | -| Normal detecting corrupt auto-escalates to one Deep retry | erasure-healing.go:1101-1106 | ops/heal.rs:2022-2031 | ✅ | -| dangling determination (not-found > parity) + deletion auditing | isObjectDangling/deleteIfDangling | `dangling_delete_safety` (:1488) + scanner HEAL_DELETE_DANGLING | ✅ (audit-tags details differ) | -| Orphan data-dir/inline cleanup (CleanAbandonedData) | CheckAbandonedParts (invoked explicitly on scanner sampling + admin Remove) | in-heal-path `reclaim_orphan_data_dirs_best_effort` (:1428); standalone API NotImplemented at all three layers | ⚠️ partially equivalent (§6 HS-02) | -| Versioned/delete-marker heal | HealObject versionID; nullVersionID special case | per-version enumeration + delete-marker latest heal (B5 regression) | ✅ | -| Object-level healing metadata marker (x-minio-healing, RenameData skips version cleanup) | yes | no object-level marker; relies on drive-level healing.bin + NSLock + rename semantics | ⚠️ evaluation item (§6 HS-12) | -| Distribution/Index consistency, three lines of defense | yes (manual modification rejected) | target-drive format results all-ok check + identity fence | ✅ (different granularity) | -| no-parity (EC:0) objects | bitrot treated as unrecoverable | judged unrecoverable (:700-726) + write self-verification | ✅ enhanced (write-path self-verification) | -| three-layer distribution inconsistency refuses heal | yes | heal_walk normalization + page-bound defense | ✅ (different implementation approach) | -| multipart orphan reconciliation | carried by CheckAbandonedParts | explicitly NotImplemented (carried by lifecycle cleanup) | ⚠️ §6 HS-02 | -| suspended/decommissioned pool handling | skipped via IsSuspended | deferral semantics (store/heal.rs:192-207, PR #5876) | ✅ | -| heal mutually exclusive with concurrent deletes | NSLock + healing marker | NSLock + write lock | ✅ | - -### 4.3 new-drive resync comparison - -| MinIO | RustFS | Status | -|---|---|---| -| waitForFormatErasure handshake waiting indefinitely on four classes of recoverable errors | startup drive resolution + renew_disk reconnect path | ✅ (different model: RustFS does not block at startup waiting for format) | -| HealFormat NSLock + errNoHealRequired + refFormat-mismatch rejection | `heal_format`/`heal_replacement_format` fail-closed + target-slot restriction (PR #1787 semantics) | ✅ enhanced | -| per (pool,set) distributed lock preventing concurrent resync | set-level queue dedup + bulkhead (manager.rs:2854-2889) | ✅ | -| brand-new-cluster detection (drives-to-heal == total drives does not trigger) | replacement_readiness (independent mount point / physical-device validation, non-root) | ✅ enhanced | -| healingTracker (.healing.bin: Bytes/Items counters, QueuedBuckets/HealedBuckets, Resume snapshot, RetryAttempts ≤4, HealID linkage, diskID-change reset) | resume/checkpoint schema'd persistence + durable intent/proof (per-task files, CAS) | ✅ equivalent and enhanced (crash-window backfill); but **external snapshot visibility** is weaker than MinIO's (§6 HS-07) | -| skip versions written after heal start (ModTime > Started) | no such filter | ⚠️ §6 HS-13 | -| skip ILM-expired versions (filterLifecycle) | no such filter | ⚠️ §6 HS-13 | -| worker count max(GOMAXPROCS,NR)/4 floor 4, heal:drive_workers override | in-page concurrency 8 (Deep/AutoHeal forced to 1) + per-set bulkhead | ✅ (different parameter model) | -| waitForLowHTTPReq yield per entry | mainline throttle (foreground-utilization gating) | ✅ enhanced | -| heal scope includes the two pseudo-buckets `.minio.sys/config` and `.minio.sys/buckets`; newest bucket first | ErasureSet task pre-processes per bucket (meta-bucket semantics carried by heal_bucket) | ✅ (no "newest first" ordering) | -| whole-failure retry ≤4 (resetHealing + errRetryHealing) | schedule_retry resets both layers + recoverable retry ≤3 | ✅ | - -### 4.4 scanner comparison - -| MinIO | RustFS | Status | -|---|---|---| -| cluster single leader (globalLeaderLock) | leader.lock + persisted leader-epoch CAS fence | ✅ enhanced (epoch fence against split-brain; MinIO has no persisted epoch) | -| `.bloomcycle.bin` stores only the cycle (bloom removed) | same path stores cycle+leader_epoch (RSCYC001) | ✅ aligned (v1 misjudgment corrected) | -| folderScanner hash-mod-16 + compaction (500/10000/2500) | same constants + plan digest + cache-currency validation + dirty-first | ✅ enhanced | -| ≤GOMAXPROCS parallel scans per drive; healing drives excluded | per-set/per-disk semaphores + sticky skip of healing drives | ✅ | -| scannerSleeper (factor 2/max 1s, speed tiers hot-swapped) | DynamicSleeper same + idle_mode + foreground-read backoff | ✅ enhanced | -| idle semantics: `scanner:idle_speed=on` (throttle only in idle windows, full speed when busy) | `RUSTFS_SCANNER_IDLE_MODE=true` (master switch for rate limiting) | ⚠️ opposite semantic direction, §6 HS-14 | -| applyActions order (heal→ILM→replication→alerts) | apply_actions same order (heal candidates→ILM→replication heal→alerts) | ✅ | -| ILM 9 actions + batch evaluation + DeletePrefixObject optimization | same 9 actions + batch evaluation + expiry queue | ✅ (whether DeleteAllVersions has the single-call optimization was not checked line by line) | -| abandoned children (listPathRaw minDisks=N/2 detects under-written drives) | list_path_raw + quorum verification + high-priority heal | ✅ | -| incomplete multipart independent routine (6h interval/24h expiry, rename into .trash) | ecstore independent background task (configurable interval/expiry) | ✅ (trash two-stage cleanup detail differences, §6 HS-18) | -| usage dimensions (size/objects/versions/DM/histograms/replication/tier/bucket level) | full coverage + cluster snapshot with triple anti-rollback | ✅ enhanced | -| prefix-level usage (loadPrefixUsageFromBackend, consumed by console) | the cache holds the directory tree but flattens only to bucket level | ❌ §6 HS-08 | -| excess events s3:ObjectManyVersions/LargeVersions/PrefixManyFolders + auditing | metrics alert_excess_* only (defaults 100/1TiB/65538 vs MinIO 100/1TB/50000) | ⚠️ §6 HS-04/HS-17 | -| scanner metrics v3 (bucket_scans/directories/objects/versions/last_activity) | full rustfs_scanner_* suite + freshness | ✅ (different naming scheme) | -| TraceScanner / realtime metrics (mc admin scanner status/trace) | no trace channel; /v3/scanner/status has its own structure | ⚠️ §6 HS-03 | - -### 4.5 admin/CLI/API surface comparison - -| MinIO | RustFS | Status | -|---|---|---| -| `POST /minio/admin/v3/heal/...` start/status/cancel | `POST /rustfs/admin/v3/heal/...` same three states | ✅ (different path prefix is expected) | -| `HealStartSuccess`/`HealTaskStatus`/`HealResultItem`/DriveState | same-named fields JSON-compatible | ✅ | -| `POST /v3/background-heal/status` (BgHealState aggregate) | same path + degraded semantics + operations matrix | ✅ enhanced (no MRF per-endpoint sub-state, because there is no MRF) | -| `GET /v3/healthinfo` per-drive `HealInfo *HealingDisk` | no equivalent healthinfo heal field (replacement-recovery v4 covers part of it) | ⚠️ §6 HS-07 | -| madmin client HealStart/HealStatus/BackgroundHealStatus/ScannerStatus methods | wire types only, no client methods | ❌ §6 HS-05 | -| mc admin heal --pool/--set, --scan-mode, --force-start/stop | HealOpts full field support (pool/set/scanMode/forceStart/forceStop) | ✅ (server-side ready; missing the mc-side entry, HS-05) | -| ErrHealAlreadyRunning / ErrHealOverlappingPaths typed errors | dedup-merge + eviction semantics; no typed overlap rejection | ⚠️ §6 HS-06 | -| result backpressure (maxUnconsumedItems=1000, 10s keep-alive streaming, 24h unconsumed abort) | snapshot-style query (1024 entries + 8MiB truncation + 10min retention) | ⚠️ §6 HS-06 | -| `mc support inspect`/healing-bin offline dump | none (inspect.rs exists but the healing dump is unconfirmed) | ⚠️ P3 | - -### 4.6 observability surface comparison - -| Dimension | MinIO | RustFS | Status | -|---|---|---|---| -| heal metrics | minio_heal_objects_total/heal_total/errors_total/time_last_activity + v3 drive_health 2=healing | full rustfs_heal_* suite (admission/queue delay/running/throttle/page concurrency) | ✅ (RustFS lacks an equivalent of the single drive_health=healing gauge; DiskInfo.healing is already assigned) | -| scanner metrics | v3 6 + realtime 18 items | full rustfs_scanner_* suite + per-source dimensions | ✅ | -| ILM metrics | v3 5 (expiry/transition pending/active/missed + versions_scanned) | ilm expiry status API + scanner per-source | ✅ (different metrics and API shape) | -| trace | TraceHealing/TraceScanner channels | none | ❌ §6 HS-03 | -| auditing | HealObject events, dangling-deletion audit, scanner:manyversions etc. | structured logs (event style) + metrics; no audit-log events | ⚠️ §6 HS-04 | -| progress | healingTracker Bytes/Items/QueuedBuckets/current object + usage-cache total baseline | HealProgress{scanned/healed/failed/bytes/current_object/percentage}; bytes_processed annotated as 0, estimated_completion_time always None | ⚠️ §6 HS-07 | - -### 4.7 configuration surface comparison (defaults) - -| MinIO | RustFS | Notes | -|---|---|---| -| `heal:bitrotscan` (default off; on=every cycle; Nm=N×30×24h) | `heal.bitrot_cycle` / `RUSTFS_SCANNER_BITROT_CYCLE_SECS` (default 30d=2592000s; 0/on=Deep every cycle, off=disabled) | ✅ same semantics (RustFS default 30d, MinIO default off — **different defaults**, RustFS more aggressive) | -| `heal:max_io=100`/`max_sleep=250ms` (waitForLowIO) | mainline throttle thresholds 80%/80%, max_sleep 250ms | ✅ same shape (different threshold model) | -| `heal:drive_workers` (default -1 auto) | in-page concurrency 8 + per-set 1 | ✅ same shape | -| `_MINIO_HEAL_WORKERS` (GOMAXPROCS/2) | `RUSTFS_HEAL_MAX_CONCURRENT_HEALS=4` + `_MAX_CONCURRENT_PER_SET=1` | ✅ | -| `_MINIO_AUTO_DRIVE_HEALING` (on) | `RUSTFS_HEAL_AUTO_HEAL_ENABLE=true` | ✅ | -| `_MINIO_SCANNER` (on) | `RUSTFS_SCANNER_ENABLED=true` | ✅ | -| `scanner:speed` five tiers (default=2x/1s/1m) | same five tiers, same names, same parameters | ✅ | -| `scanner:idle_speed` (on) | `RUSTFS_SCANNER_IDLE_MODE` (true) | ⚠️ semantic direction (HS-14) | -| `scanner:alert_excess_versions=100` | 100 | ✅ | -| `scanner:alert_excess_folders=50000` | 65538 (compatible with the PBS layout) | ⚠️ HS-17 | -| `ilm:expiration_workers=100`/`transition_workers=100` | ecstore expiry/transition worker pools (keys under the ilm subsystem) | ✅ (defaults not checked item by item) | -| `api:stale_upload_cleanup_interval=6h`/`expiry=24h` | ecstore background task, configurable via env | ✅ (defaults not checked item by item) | -| — (none) | `RUSTFS_HEAL_QUEUE_SIZE=10000`, `_TASK_TIMEOUT_SECS=300`, `_INTERVAL_SECS=10`, `_LOW_PRIORITY_MERGE/DROP`, `_PAGE_*`, `_SET_BULKHEAD`, `_MAINLINE_*`, `RUSTFS_SCANNER_CYCLE_MAX_*` budgets, `_MAX_CONCURRENT_SET/DISK_SCANS=4`, `_YIELD_EVERY_N_OBJECTS=128`, etc. | RustFS-specific (finer-grained) | - -### 4.8 Where RustFS exceeds MinIO - -1. remote_scanner RPC (scan execution pushed down to the remote peer locally, with HMAC authentication/replay cache/fence re-validation/disconnect grace). -2. Persisted leader-epoch CAS fence + usage-snapshot epoch/cycle anti-rollback (MinIO has only the lock, no persisted epoch). -3. Cycle budgets (max_duration/objects/directories) + partial-cycle advancement semantics. -4. per-set/per-disk scan concurrency gates + a cache lock per bucket per set. -5. pending-heal ledger (heal candidates are not lost when the heal channel is full). -6. Drive-replacement durable intent + completion proof state machine + identity fence (MinIO's healingTracker has no proof). -7. mainline throttle foreground pressure gating (driven by permit utilization). -8. Cluster heal control coordinator + envelope replay protection + explicit degraded fallback. -9. Write-path shard bitrot self-verification (the EC:0 case). -10. dirty-usage fast-path wakeup (immediate write-path notification + dirty buckets first). -11. heal runtime observability matrix (priority×source operations snapshot). -12. workload admission integration (the heal scheduler reads the foreground pressure snapshot). - ---- - -## 5. Gap and improvement list - -Severity definitions: P1 = behavioral/operational alignment gap (affects production operations or toolchain compatibility); P2 = completeness (the feature exists but is missing a corner); P3 = cleanup/low risk. Each item includes current-state evidence, MinIO behavior, impact, recommendation, and acceptance. - -### P1 (8 items) - -**HS-01 The MRF/ECDecode/Metadata heal task types have no production trigger; HealEvent unwired** -- Current state: the `HealType::MRF/ECDecode/Metadata` executors are complete (task.rs:1700-2156) but have no production trigger anywhere in the repo; `HealEvent`/`HealEventHandler` (event.rs:50-367) has zero references outside the crate (verified first-hand by grep); channel conversion produces only Cluster/Object/Bucket/Prefix/ErasureSet (channel.rs:566-601). -- MinIO: mrf.go has a standalone MRF queue (capacity 100k, drop-and-count when full), msgp persistence to `.heal/mrf/list.bin` at process exit + startup replay, 1s delay for enqueues <1s (waiting for network recovery), healSleeper rate limiting; on the read path, GetObject part missing/corrupt, metadata rebuild missingBlocks>0, partial Put success, DeleteObject, multipart, and the peer client add up to 7+ delivery points. -- Impact: RustFS's read-repair + write-path convergence covers the main scenarios, but lacks: ① an event-driven Urgent ECDecode rebuild entry (on ecstore decode failure there is currently only Low read-repair); ② a metadata-only heal entry (the scanner's HealMetadata classification exists but goes through ordinary object heal); ③ MRF queue persistence (unconsumed repair intents are lost on restart — partially mitigated by the scanner's pending-heal ledger). -- Recommendation: a pick-one-of-three decision — (a) wire HealEvent (emit events at ecstore decode-failure/metadata-corruption points) + implement a persistent retry ledger; (b) delete the MRF/ECDecode/Metadata dead code and keep only a documentation note; (c) keep the executors and demote HealEvent to an internal API. (a) is recommended, but first quantify whether read-repair already meets the response-time requirements for decode-failure scenarios. -- Acceptance: an e2e decode-failure → Urgent heal-request chain; replay of pending repair intents after restart; HealEvent ring-buffer metrics. - -**HS-02 CheckAbandonedParts NotImplemented at all three layers (missing standalone abandoned-data reconciliation entry)** -- Current state: `set_disk/ops/heal.rs:2052-2056`, `core/sets.rs:1144-1148`, `store/heal.rs:258-266` explicitly return `Err(NotImplemented)` at all three layers (verified first-hand); the comment reads "intentionally retained above the set layer until there is a concrete caller". -- MinIO: `CheckAbandonedParts` → per-drive `CleanAbandonedData`: read xl.meta → list UUID data-dirs + inline entries → diff against getDataDirs → delete surplus data-dirs/inline entries and rewrite xl.meta; invoked explicitly on scanner-sampled heals and admin heal Remove. -- Impact: RustFS's in-heal-path `reclaim_orphan_data_dirs_best_effort` (:1428) covers "reclaim orphan directories while healing", but ① there is no standalone trigger point (MinIO can also clean abandoned data before an object reaches the heal threshold); ② orphan inline-data entry cleanup is unconfirmed; ③ multipart orphan reconciliation is explicitly out of scope (a design decision, carried by lifecycle). -- Recommendation: evaluate promoting `reclaim_orphan_data_dirs_best_effort` to a fixed step of heal_object (if it is not already) + implement a real HealOperations::check_abandoned_parts (calling the same reclamation logic), or explicitly document "carried by lifecycle" and close the API surface. -- Acceptance: construct data-dir/inline orphans → cleaned after scanner sampling/admin heal; the three-layer API returns success or an explicitly documented NotSupported. - -**HS-03 heal/scanner trace channels missing** -- Current state: zero hits for TraceHealing/TraceScanner (verified first-hand by grepping the whole repo). -- MinIO: `madmin.TraceHealing` (mc admin trace --healing, FuncName=heal.Bucket/heal.Object/heal.CheckAbandonedParts, with dry/remove/mode/version-id/disks/bytes), `TraceScanner` (mc admin scanner trace, supports --filter-size/--response-duration). -- Impact: no way to observe in real time the latency and parameters of individual heal/scanner actions; troubleshooting can rely only on aggregated metrics and logs. -- Recommendation: instrument heal-channel execution and scanner folder/item handling, and hook them into the existing admin trace subscription surface (reuse the rustfs trace infrastructure if it exists; otherwise extend it per madmin TraceType). -- Acceptance: an mc-equivalent tool can subscribe to the heal/scanner trace stream. - -**HS-04 Scanner excess S3 events and auditing missing** -- Current state: only `rustfs_scanner_excess_*_total` metrics (versions 100 / version size 1TiB / folders 65538). -- MinIO: emits `s3:ObjectManyVersions` (>100 versions), `s3:ObjectLargeVersions` (cumulative >1TB), `s3:PrefixManyFolders` (>50000 subdirectories) events (UserAgent: Scanner) + scanner:manyversions/largeversions/manyprefixes auditing. -- Impact: users relying on event subscriptions for capacity governance (console/external auditing) receive no alerts. -- Recommendation: hook the scanner_folder alert points into notify event publishing (reusing the lifecycle event-channel semantics). -- Acceptance: after configuring bucket notifications, an over-threshold object triggers an event. - -**HS-05 madmin client methods missing** -- Current state: `crates/madmin/src/heal_commands.rs` has only wire types (HealDriveInfo/Infos/HealResultItem); no HealStart/HealStatus/BackgroundHealStatus/ScannerStatus client methods. -- MinIO: madmin-go provides the full client; mc admin heal/scanner/status/trace are all built on it. -- Impact: admin tools like mc cannot directly drive the RustFS heal/scanner admin surface; automated operations must hand-write HTTP. -- Recommendation: add the client following the madmin-go interface shape (the server side is ready; this is pure client work). -- Acceptance: complete the start→query→cancel flow with the madmin client. - -**HS-06 admin heal sequence semantics differ from MinIO** -- Current state: duplicate/overlapping requests are dedup-merged (returning the canonical task_id) or evicted; no ErrHealAlreadyRunning/ErrHealOverlappingPaths typed errors (verified first-hand: manager.rs:1309's already_running is an idempotent-startup guard, not an admin semantic); results are snapshot-style queries (1024 entries/8MiB truncation/10min retention), not MinIO's streaming increments (clientToken pulls increments + maxUnconsumedItems=1000 backpressure + 10s keep-alive + 24h unconsumed abort). -- Impact: mc admin heal's interaction model (long connection pulling increments) behaves against RustFS as multiple snapshot polls; automation scripts cannot easily distinguish "merged" from "newly started". -- Recommendation: ① incremental semantics: channel query supports item increments since the last clientToken (or a cursor); ② overlapping requests return a typed error code (or an explicit merged_into field in the receipt — the existing alias mechanism already provides the base); ③ verify forceStart's stop-old-then-start-new semantics. -- Acceptance: an madmin-compatible client polling in the MinIO style can retrieve the full item set. - -**HS-07 healing progress and drive-level healing state insufficiently visible externally** -- Current state: byte-recovery progress `progress.bytes_processed = 0 // set to 0 for now` (erasure_healer.rs:967); `HealProgress::estimated_completion_time` is always None and `HealStatistics::add_healed_objects` is never written (progress.rs:38,135-139 zero calls); healthinfo has no per-drive HealInfo equivalent (MinIO HealingDisk: BytesDone/Failed/Skipped, ObjectsTotal baseline, QueuedBuckets/HealedBuckets, Resume snapshot, current object); v3 metrics lack an equivalent of the single drive_health=2 (healing) gauge. -- Impact: during a drive rebuild (potentially hours to days) operations cannot answer "where are we / how much is left / when will it finish". -- Recommendation: ① accumulate bytes in erasure set heal (heal_object already yields the object size); ② read the object-total baseline from usage-cache (the same approach as MinIO); ③ expose a per-drive healing snapshot in admin healthinfo/background status (DiskInfo.healing already exists; add the aggregated exposure); ④ derive the ETA from baseline + rate. -- Acceptance: during a drive rebuild, admin shows byte progress and ETA; an mc info-equivalent output shows the Healing flag. - -**HS-08 prefix-level usage not exposed** -- Current state: the DataUsageCache holds the directory-tree entries (organized by hash_path), but `dui()` flattens only to the bucket name (data_usage_define.rs:858-915). -- MinIO: `loadPrefixUsageFromBackend` (30s cache) aggregates prefix usage from each set's `.usage-cache.bin`, consumed by console bucket-prefix statistics. -- Impact: console/front ends cannot show prefix-level usage; there is no API to locate "which prefix is using the space" in a large bucket. -- Recommendation: implement a prefix-flattening query API (the data is already in the cache; this is pure aggregation and exposure work). -- Acceptance: a ListBuckets/PrefixUsage API returns statistics matching the prefix filter. - -### P2 (9 items) - -**HS-09 get_disk_status always returns Ok (the only TODO)**: `crates/heal/src/heal/storage.rs:930-943` (verified first-hand). Currently no production caller (low risk). Recommendation: delete the method or wire it to the real ecstore disk status (the DiskStatus enum is already defined). - -**HS-10 About 1/3 of HealStorageAPI methods are dead code**: get_object_meta/get_object_data/put_object_data/delete_object/verify_object_integrity/ec_decode_rebuild/get_disk_status/format_disk/heal_bucket_metadata/get_object_size/get_object_checksum/list_objects_for_heal (the non-paginated version, with its own memory_heavy warning) all have 0 callers. Recommendation: clean up or wire them together with the HS-01 decision (dead interfaces mislead future maintainers into thinking a call path exists). - -**HS-11 bitrot self-test missing**: MinIO at startup runs bitrotSelfTest over known vectors for the four algorithms and exits Fatal on failure (guarding against silent data corruption). RustFS has no equivalent (verified first-hand by grep). Recommendation: at startup, run known-vector self-tests for HighwayHash256S and the other algorithms in use (low cost, high value). - -**HS-12 object-level healing metadata marker evaluation**: during heal, MinIO tags objects with `x-minio-healing:true`, and RenameData uses it to skip version cleanup/legacy purge (missing it lets heal and concurrent deletes destroy each other). RustFS has no object-level marker (verified first-hand by grep; object.rs has no healing branch) and relies on NSLock + rename semantics. Recommendation: audit whether the RustFS rename-commit path has a "heal commit racing concurrent delete/version cleanup" window; if not, document the difference, and if so, add a marker-equivalent mechanism. - -**HS-13 erasure set heal lacks "skip newly written / ILM-expired versions" filters**: MinIO resync skips versions with ModTime>tracker.Started (so heal does not chase the tail of new writes) and ILM-expired versions (so work is not wasted). RustFS's erasure_healer does not implement such filters (per-version dedup exists; time/ILM filters do not). Impact: a long tail on rebuild completion (the completion decision for a continuously written bucket is pushed out by new versions) and wasted heal work. Recommendation: add a started_at time filter at the disk-walk enumeration point + an evaluator pre-check. - -**HS-14 scanner idle semantics point the opposite way from MinIO**: MinIO `scanner:idle_speed=on` (default) means "throttle only when the cluster is idle, full speed when busy"; RustFS `RUSTFS_SCANNER_IDLE_MODE=true` (default) is a master switch for rate limiting (false = never sleep at all). The default behaviors may end up similar (both throttle), but the parameter semantics are not interchangeable; migration docs must state this explicitly; if mc config compatibility is the goal, a rename/re-semantization is needed. Recommendation: document the difference first, then evaluate aligning the semantics. - -**HS-15 alert_excess_folders default differs**: RustFS 65538 (compatible with the PBS/Proxmox layout, scanner_folder.rs:79) vs MinIO 50000. The behavioral difference is that the trigger threshold differs out of the box. Recommendation: document it (keeping 65538 has local rationale). - -**HS-16 single-node default-cycle hook not enabled**: `single_disk_default_cycle_secs(_features) -> None` is always empty (scanner.rs:1428-1430); single-node deployments get no dedicated default-cycle override. Recommendation: after deciding the single-node default-cycle policy, enable or delete the hook. - -**HS-17 DeleteAllVersions batch-optimization check**: MinIO uses the single DeletePrefix+DeletePrefixObject call instead of per-version fan-out. Whether RustFS's expiry-queue path has the same optimization was not verified line by line (integration tests cover behavioral correctness). Recommendation: check the `apply_expiry_rule` all-versions delete path; if there is no prefix single-call optimization, evaluate adding it. - -### P3 (3 items) - -**HS-18 trash/temp-directory two-stage cleanup detail check**: MinIO cleans `.minio.sys/tmp/.trash` (delete_cleanup_interval default 5m + deleteCleanupSleeper) and stale uploads are renamed into trash in two stages. RustFS has delete_tail_activity.rs and the stale multipart task; whether the two-stage semantics are fully aligned was not verified line by line. Recommendation: align or document. - -**HS-19 root-heal direct path is dead code**: `should_handle_root_heal_directly` is always false (admin/handlers/heal.rs:1200-1202, locked by a test); the store.heal_format direct branch is unreachable. Recommendation: delete the dead branch or restore the direct path as a fallback for cluster-coordination failure. - -**HS-20 compat flags and dead metrics cleanup**: `RUSTFS_SCANNER_INLINE_HEAL_ENABLE` (enabling only warns) + the dead `rustfs_scanner_inline_heal_total` metric + the scanner-domain code in `rustfs_common::metrics` awaiting layering migration (backlog #1843 already filed). Recommendation: clean up along with the layering migration. - -### Not pursuing parity by design (7 items, recorded to prevent later misreading as gaps) - -1. **bloom filter**: removed from MinIO master; RustFS reuses `.bloomcycle.bin` as the cycle/epoch fence, consistent with MinIO's current state. -2. **scanner cluster single leader**: both sides agree; RustFS additionally has the epoch fence. -3. **heal emits no S3 bucket notification**: both sides agree (heal results go through admin status). -4. **incomplete multipart not executed inside scanner/ILM**: both sides agree (independent background routine). -5. **inline heal removal**: a deliberate RustFS choice (the scanner only enqueues); MinIO's applyHealing inline path is not a parity target. -6. **heal-sequence resident keep-alive (10s blank write-back)**: RustFS's snapshot-query model differs; handling incremental semantics per HS-06 is enough — do not copy the streaming keep-alive. -7. **`.trash`/`tmp-old` path-name compatibility**: RustFS's layout constants are independent; no literal alignment with MinIO paths. - ---- - -## 6. Configuration defaults master table (RustFS) - -heal (env prefix `RUSTFS_HEAL_`, `crates/config/src/constants/heal.rs`, consumed at `manager.rs:724-800`): - -| Setting | Default | Hot update | -|---|---|---| -| AUTO_HEAL_ENABLE | true | no | -| QUEUE_SIZE | 10000 | no | -| INTERVAL_SECS | 10 | no (fixed at startup) | -| TASK_TIMEOUT_SECS | 300 | no | -| MAX_CONCURRENT_HEALS | 4 | no | -| MAX_CONCURRENT_PER_SET | 1 (≤min(global, value)) | no | -| LOW_PRIORITY_MERGE_ENABLE | true | no | -| LOW_PRIORITY_DROP_WHEN_FULL | true | no | -| PAGE_OBJECT_CONCURRENCY | 8 (Deep/AutoHeal forced to 1) | no | -| EVENT_DRIVEN_SCHEDULER_ENABLE | true | no | -| SET_BULKHEAD_ENABLE | true | no | -| PAGE_PARALLEL_ENABLE | true | no | -| MAINLINE_THROTTLE_ENABLE | true | no | -| MAINLINE_READ/WRITE_UTILIZATION_HIGH_PERCENT | 80/80 | no | -| MAINLINE_MAX_SLEEP_MS | 250 | no | -| (master switch) RUSTFS_HEAL_ENABLED | true | no | -| admin subsystem heal.bitrot_cycle | 30d | yes (via scanner runtime config) | - -scanner (admin subsystem `scanner`, `crates/config/src/constants/scanner.rs` + `ecstore/src/config/scanner.rs` + `runtime_config.rs:527-673`): - -| Key | env | Default | -|---|---|---| -| speed | RUSTFS_SCANNER_SPEED | default (2x/1s/60s) | -| delay / max_wait / cycle / start_delay | RUSTFS_SCANNER_* | derived/empty | -| cycle_max_duration/objects/directories | …_MAX_* | 0 (unlimited) | -| bitrot_cycle | …_BITROT_CYCLE_SECS | 2592000 (30d; 0/on=every cycle, off=disabled) | -| idle_mode | …_IDLE_MODE | true | -| cache_save_timeout | …_CACHE_SAVE_TIMEOUT_SECS | 14s | -| max_concurrent_set_scans / disk_scans | …_MAX_CONCURRENT_* | 4/4 | -| yield_every_n_objects | …_YIELD_EVERY_N_OBJECTS | 128 | -| alert_excess_versions / version_size / folders | …_ALERT_* | 100 / 1TiB / 65538 | - -scanner-internal env: `RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=16`, `RUSTFS_HEAL_OBJECT_SELECT_PROB=1024`, `RUSTFS_SCANNER_DEEP_VERIFY_COOLDOWN_SECS=60`, `RUSTFS_DATA_USAGE_FAILED_OBJECT_TTL_SECS=86400`/`_MAX=10000`, `RUSTFS_LOCK_ACQUIRE_TIMEOUT=5s`, `RUSTFS_SCANNER_ENABLED=true`, `RUSTFS_SCANNER_INLINE_HEAL_ENABLE=false` (compat warning). - -All 17 scanner keys support the env > config dual channel + admin PUT hot update (generation+Notify takes effect immediately); heal runtime parameters are currently env-only (no admin hot-update entry; the `Arc>` structure is already reserved). - ---- - -## 7. Related backlog / history index - -- Automatic drive-replacement healing series (closed loop): backlog #1786 (redundant false-green algorithm), #1787 (target-slot restriction), #1789 (binding resume and the healing marker to the replacement instance), #1791 (black-box/white-box acceptance matrix). -- #801 DiskInfo.healing never assigned (fixed and closed; the assignment chain now lives at `set_disk/mod.rs:4988`). -- #1651 Scanner metrics node/source/bucket-drive dimensions (OPEN; related to §3.8/§4.6 of this analysis). -- #1843 crates/common 83% scanner/heal domain code layering migration (OPEN; includes HS-20). -- Historical defects cited in code comments (now guarded with regression tests): #856/#799 B7 (offline drive falsely recorded healed), #855/B6/#1033 (a skip round must not be marked complete), #920 (sub-quorum union enumeration), #856 B5 (per-version resume), #5173 (bitrot trailing bytes), #5029 (stale-version merge at regression nodes). -- v1 parity document: `docs/rustfs-heal-scanner-vs-minio-parity-assessment.md` (superseded by this document); the landing playbook `docs/rustfs-heal-scanner-vs-minio-improvement-playbook.md` (some entries have since been overtaken by implementation). -- Drive-replacement deep analyses: `docs/new-disk-replacement-and-healing-deep-analysis-zh.md`, `docs/node-disk-identity-and-healing-analysis-zh.md`. - -## 8. Audit method and limitations - -- Four parallel audit tracks (heal crate file by file, scanner crate file by file, ecstore integration-layer wiring, MinIO master source study) + the main session verifying each key "missing" conclusion first-hand (the get_disk_status TODO, HealEvent's zero external references, .bloomcycle.bin having no bloom implementation, check_abandoned_parts NotImplemented at all three layers, the ETag fallback being implemented, zero trace-channel hits, the already_running semantics). -- Points not verified line by line (marked "unconfirmed / not checked line by line" in the text): the DeleteAllVersions prefix single-call optimization (HS-17), trash two-stage cleanup details (HS-18), ilm worker default comparisons, stale multipart default comparisons, mc CLI flag spellings (MinIO side). Of these, HS-17 and HS-18 completed line-by-line verification on 2026-08-19; conclusions in §9.2/§9.3. -- MinIO-side references follow its master `7aac2a2c5b`; RustFS-side line numbers follow the 2026-08-16 workspace — for later evolution, search by symbol name instead. - -## 9. Landing results (updated 2026-08-19) - -All 14 sub-issues derived from this audit (backlog #1865~#1878) are closed. This section is the final disposition record for the gap list HS-01~HS-20, and also the incremental baseline for the next parity re-audit. - -### 9.1 Landed (all PRs merged to main) - -- HS-01 MRF wiring + persistent repair ledger (#1865, PR #6189): decision (a) chosen. common MRF channel (bounded 8192, try_send never blocks) + heal mrf_queue (100k entries / 8MiB dual-capacity ring) + `buckets/.heal/mrf/journal.bin` CRC-persisted replay (torn tail truncated, deleted after replay) + three delivery points (read decode_error→Urgent ECDecode, scanner metadata corruption→High Metadata, add_partial→Normal) + `RUSTFS_HEAL_MRF_ENABLE` one-switch rollback. -- HS-02 abandoned parts/data-dir reconciliation (#1866, PR #6179): wired up the abandoned-check entry, retaining dry-run / reclaim counters. -- HS-03 heal/scanner trace channels (#1867, PR #6179): in-process trace bus + `/v3/trace` admin streaming subscription + heal task / abandoned-parts / scanner folder / ILM / heal-candidate trace producers. -- HS-04 scanner excess S3 events (#1868, PR #6176): the three events `s3:Scanner:ManyVersions/LargeVersions/BigPrefix` + 24h edge cooldown; the HS-15 threshold delta documented (`docs/operations/scanner-excess-alerts.md`). -- HS-05 madmin client phase 1 (#1869, PR #6166): SigV4 admin client heal/scanner methods; incremental-consumption methods await a follow-up (the protocol was already folded in by HS-06). -- HS-06 admin heal incremental semantics and typed overlap (#1870, PR #6206): `sinceSeq/nextSeq/minSeq` incremental cursor (wire additive; absent = full snapshot) + `RUSTFS_HEAL_OVERLAP_POLICY` (default merge unchanged; under minio_error, typed AlreadyRunning/OverlappingPaths rejections) + forceStart stops the old sequence before starting the new one. -- HS-07 healing progress visibility (#1871, PR #6179): data-usage total baseline + baseline/current/healed counters. -- HS-08 prefix usage (#1872, PR #6171): `GET /v3/usage/{bucket}`. -- HS-11 bitrot startup self-test (#1873, PR #6165). -- HS-13 heal skip filters (#1875, PR #6179): filter-hit versions are no longer counted as failures. -- HS-16 single-node cycle hook (#1878, PR #6250): removed the always-None hook; the decision record is in `docs/operations/heal-scanner-parity-notes-zh.md`. -- HS-09/10/19/20 dead-code cleanup batch (#1877, PR #6256): net −911 lines, zero behavior change; the `get_disk_status` TODO (the repo's only product TODO) cleared to zero; `ec_decode_rebuild`/`get_object_meta`, kept due to the HS-01 linkage, are retained with Reserved annotations (MRF currently executes via `heal_object`). - -### 9.2 Confirmed "already implemented / not a gap" after verification (audit-period misjudgment corrections, four in total) - -- bloom filter (corrected in §0): removed from MinIO master; both sides now agree. -- ETag fallback arbitration (corrected in §0): RustFS already has the implementation (`set_disk/ops/heal.rs`). -- HS-17 (#1876, closed after line-by-line verification on 2026-08-19): the DeleteAllVersions prefix single-call optimization is fully implemented in RustFS — `apply_expiry_on_non_transitioned_objects` sets `delete_prefix + delete_prefix_object` for the two `delete_all()` actions and then performs a single `delete_object` call (`bucket_lifecycle_ops.rs:5047-5056`); the SetDisks branch takes one write lock + one all-version quorum read + inline per-version object-lock checks (`set_disk/ops/object.rs:5566-5612`), aligned line by line with MinIO `expire.go`'s `applyExpiryOnNonTransitionedObjects`. The item §8 listed as "not verified line by line" now has a conclusion: the current state is already the optimized path; nothing to implement. -- HS-14 (#1878, checked alongside PR #6250): MinIO's "idle = throttle only when idle" was the behavior before 2024-01 minio/minio#18734 (`scannerIdleMode` is now a static config; `idle_speed=on` by default means always throttling per the speed tier — the "idle" naming is a historical leftover); RustFS's `RUSTFS_SCANNER_IDLE_MODE` points the same way as MinIO's current semantics, and additionally has a foreground-read backoff floor that MinIO lacks. The real migration traps (the variable must carry the `RUSTFS_` prefix, the `on/off` vs `true/false` vocabulary, `false` also turning off foreground protection) are documented in `docs/operations/heal-scanner-parity-notes-zh.md`. - -### 9.3 Audit-style conclusions (no code change needed) - -- HS-12 (#1874, PR #6183): the class of race MinIO defends against with `x-minio-healing` does not exist — every commit surface for the same (bucket, object) is mutually exclusive under the same object-level ns write lock, and the heal lock guard covers the whole rename commit; delivered 2 concurrency-invariant regression tests + the intersection matrix in `docs/operations/heal-concurrency-safety-notes-zh.md`. -- HS-18 (#1878, line-by-line verification on 2026-08-19): trash/tmp three-stage cleanup fully aligned — stale multipart isolation-cleanup is equivalent and safer (`delete_all_with_quorum` recursively deletes per drive, i.e. the `move_to_trash` rename into `.rustfs.sys/tmp/.trash`, plus lock + fence); trash draining is essentially equivalent (no per-entry sleeper throttling; the 5m cycle naturally rate-limits); tmp non-trash 24h reclamation is equivalent (RustFS's 5m is more timely than MinIO's 6h); the three cycle defaults 24h/6h/5m all align. The item §8 listed as "not verified line by line" now has a conclusion. - -### 9.4 Handed over to follow-ups (summarized in the backlog#1862 comment thread) - -HS-01 bitrot GET→MRF full-chain e2e, kill -9 journal replay e2e, queue-full RSS stress test (≤ budget+10%); HS-05/06 madmin incremental-consumption methods + single-source wire + embedded e2e + multi-round polling soak; HS-08 multi-drive scanner cycle e2e; HS-04 excess audit entries; HS-18 the stale-multipart crash-residue window below quorum (crashing mid-fan-out with already-cleaned drives > parity means FileNotFound is not in the ignore set, so convergence is unnatural; the fix needs a dedicated quorum variant). - -Recommendation for the next re-audit: trigger it after the next big heal/scanner feature lands, using this section as the incremental baseline. diff --git a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md b/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md deleted file mode 100644 index f3efcc2aa..000000000 --- a/docs/operations/rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16_zh.md +++ /dev/null @@ -1,568 +0,0 @@ -# RustFS heal / scanner 全量功能分析与 MinIO 对标（v2） - -> English version: [rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md](rustfs-heal-scanner-vs-minio-comprehensive-analysis-2026-08-16.md) - -- 日期：2026-08-16（基于 main 分支当日代码，审计时 HEAD ≈ `a118d7e4f`） -- 范围：`crates/heal`（src 19,560 行 + tests 2,274 行）、`crates/scanner`（src 约 26,000 行 + tests）、`crates/data-usage`、`crates/ecstore` 中 heal/heal_walk/bitrot_self_verify 与 config、`crates/heal-contracts/src/heal_channel.rs`、`crates/madmin`（heal/scanner wire 类型）、`rustfs/src`（startup wiring、admin handlers、集群 RPC） -- 对标基线：minio/minio master（HEAD `7aac2a2c5b`，仓库已进入维护模式，master 冻结，即最终态） -- 方法：四路并行审计（heal crate / scanner crate / ecstore 集成层 / MinIO 源码研究），关键结论逐条人工抽验（文内标注"已亲验"处为一手验证） -- 本文档取代 `docs/rustfs-heal-scanner-vs-minio-parity-assessment.md`（2026-06-15 v1）。v1 之后 heal/scanner 相关提交超过 80 个（换盘自动修复全链路、resume 状态机、usage 收敛权威化、集群级 heal 协调、ILM restore 语义等），v1 的功能清单与差距判断已全面过时；v1 中"bloom filter 缺失"等结论经本次核实为**误判**（详见 §5.4）。 - ---- - -## 0. 结论摘要 - -1. **总体判断：heal 与 scanner 的核心功能链路已经完整**。对象级 heal（quorum 仲裁 + ETag 兜底 + bitrot Deep 校验 + dangling 处理）、erasure set 深扫（per-set disk-walk 并集枚举）、按版本断点续扫（schema 化持久层 + CAS 原子发布 + 崩溃窗口补齐）、换盘自动修复（readiness 校验 + 身份围栏 + durable intent + completion proof）、scanner 周期循环（leader lock + 持久化 leader-epoch 围栏）、data usage 统计（桶级/集群级、主+备+观测快照、epoch/cycle 防回退）、ILM 全动作（expiry/transition/noncurrent/free-version/delete-marker 清理）、admin Start/Query/Cancel 协议（clientToken 语义对齐 madmin）——以上均有实现且带回归测试。两个 crate 内**没有空实现/早退桩**，异常路径全部有日志 + 指标 + 错误语义。 -2. **主要缺口集中在"入口与观测面"，而不是修复算法本身**：MRF/ECDecode/Metadata 三类任务执行体已实现但无生产触发入口（`HealEvent` 完全未接线）；`CheckAbandonedParts` 在 ecstore 三层全部 `NotImplemented`；heal/scanner trace 通道缺失；scanner 超限 S3 事件缺失；madmin 客户端方法缺失（只有 wire 类型）；heal 字节级进度/ETA 未实现。 -3. **与 v1 认知的重要修正**：bloom filter 在 MinIO 当前 master **已删除**（`.bloomcycle.bin` 只存 cycle 计数），RustFS 现状与 MinIO 一致；MinIO scanner 同样是**集群级 leader 单例**，RustFS 的 leader.lock 模型与 MinIO 同型；RustFS 的 ETag 多数派兜底仲裁已实现（`crates/ecstore/src/set_disk/ops/heal.rs:525-567,679`，已亲验），v1 担心的仲裁缺口不存在。 -4. **RustFS 在多处超出 MinIO**：remote_scanner RPC 协议（远端 peer 本地扫描而非 leader 跨网读远盘）、持久化 leader-epoch CAS 围栏、周期预算与 per-set/per-disk 并发闸、pending-heal 账本、durable replacement intent + completion proof 状态机、前台压力门控（mainline throttle）、集群 heal control coordinator + envelope 重放防护。 -5. 差距分级统计：P1（行为/运维对齐缺口）8 项，P2（完善性）9 项，P3（清理/低风险）3 项，"按设计不追平"7 项。完整清单见 §6。 - ---- - -## 1. 架构总览 - -### 1.1 RustFS 三层架构 - -RustFS 把 MinIO 在 `cmd/` 内单体的 heal/scanner 拆成三层 + 两个独立 crate： - -| 层 | 位置 | 职责 | -|---|---|---| -| 原语层 | `crates/ecstore/src/set_disk/ops/heal.rs`（~3,240 行）、`ops/heal_walk.rs`、`ops/bitrot_self_verify.rs`；上层封装 `store/heal.rs`、`store/heal_walk.rs`、`core/sets.rs` | 对象/桶/format/替换盘格式修复、disk-walk 并集枚举、写入路径 bitrot 自校验；由 `SetDisks`/`Sets`/`ECStore` 实现 `rustfs_storage_api::HealOperations` 契约（`crates/storage-api/src/object.rs:503-519`） | -| heal 运行时 | `crates/heal` | 进程级 HealManager（优先级队列/调度器/auto disk scanner/断点续传 resume）、HealChannelProcessor（消费全局 heal channel）、换盘替换恢复状态机 | -| scanner 运行时 | `crates/scanner` | 数据使用扫描、ILM 评估与入队、heal 候选生产、复制用量统计、remote scanner RPC | -| 共享协议 | `crates/heal-contracts/src/heal_channel.rs`（~776 行） | Start/Query/Cancel 命令通道、`HealOpts`/`HealScanMode`/`HealRequestSource`/`HealAdmission*` 共享类型、`HealResultItem`（madmin） | -| 共享数据 | `crates/data-usage` | `DataUsageEntry/Info`、直方图、`hash_path`；scanner 产生、ecstore/admin 消费 | - -启动链路（已亲验 wiring）： - -1. `rustfs/src/startup_services.rs:93` → `init_background_service_runtime(store)`。 -2. `rustfs/src/startup_background.rs:41-81`：创建全局 heal 服务取消令牌；读 `RUSTFS_SCANNER_ENABLED`（别名 `RUSTFS_ENABLE_SCANNER`，默认 true）与 `RUSTFS_HEAL_ENABLED`（别名 `RUSTFS_ENABLE_HEAL`，默认 true）；**只要 heal 或 scanner 任一开启就初始化 heal manager**（scanner 产生的 heal 候选需要消费端；两者都关时 heal channel 不初始化，`send_heal_request` 报 "Heal channel not initialized"）。 -3. `crates/heal/src/lib.rs:142-216`：owned task 内原子初始化（caller 取消不会遗留半初始化 manager，`lib.rs:123-131`；`GLOBAL_HEAL_RUNTIME_INIT` 互斥单飞）→ `HealManager::start()` → `rustfs_common::heal_channel::init_heal_channels()` → spawn `HealChannelProcessor::start_with_receipts`。 -4. `crates/heal/src/heal/manager.rs:1301-1356` `HealManager::start`：`start_scheduler()`（`manager.rs:2394-2461`，interval 默认 10s + `Notify` 事件驱动唤醒）→ `process_unclean_shutdown()`（`manager.rs:1362-1695`）→ `enable_auto_heal`（默认 true）时 `start_auto_disk_scanner()`（`manager.rs:2464-2999`）。 -5. server ready 后 `rustfs/src/startup_lifecycle.rs:150-152`：`enable_scanner` 时 `init_data_scanner(token, store)`（`crates/scanner/src/scanner.rs:1293-1372`）。 -6. 优雅停机：`rustfs/src/startup_shutdown.rs:308` `shutdown_ahm_services()`（取消令牌）；`:414` `clear_unclean_shutdown_markers()`。 - -### 1.2 MinIO 对应结构（master 最终态） - -| MinIO 文件 | 职责 | -|---|---| -| `cmd/admin-heal-ops.go` | 手动 admin heal 序列（healSequence、clientToken/forceStart/forceStop） | -| `cmd/global-heal.go` | 常驻后台 heal 队列（newBgHealSequence，token 固定 `0000-…`，永不结束）+ `healErasureSet`（逐 set 全量对象 heal） | -| `cmd/background-heal-ops.go` | healRoutine worker 池（`_MINIO_HEAL_WORKERS`，默认 GOMAXPROCS/2）消费 healTask | -| `cmd/mrf.go` | MRF（Most Recent Fail）队列（容量 100,000），进程退出时持久化 `.minio.sys/buckets/.heal/mrf/list.bin` 并启动回放 | -| `cmd/background-newdisks-heal-ops.go` | 新盘/换盘自动 resync（monitorLocalDisksAndHeal 10s 轮询 + healFreshDisk + healingTracker） | -| `cmd/erasure-healing.go` / `erasure-healing-common.go` | 对象级 heal 核心（~800 行）、listAndHeal | -| `cmd/data-scanner.go` | scanner 循环（globalLeaderLock 集群单例）+ folderScanner + applyActions | -| `cmd/erasure.go`（nsScanner）/ `erasure-server-pool.go` | NSScanner 三层结构 | -| `cmd/bucket-lifecycle.go` | ILM 执行器（expiry/transition worker 池） | -| `cmd/xl-storage.go` | DiskInfo.Healing、CheckParts/VerifyFile、CleanAbandonedData、RenameData healing 分支 | -| `cmd/prepare-storage.go` | waitForFormatErasure 新盘启动握手 | - -### 1.3 架构级差异（设计取舍，非缺陷） - -1. **heal 队列模型**：MinIO 所有 heal（scanner 抽样/MRF/admin/新盘 resync）汇入单 channel + 固定 worker 池（新盘 resync 另有 per-drive worker 池）；RustFS 是优先级堆 + 去重合并 + 容量分级丢弃 + per-set bulkhead + 前台压力门控的多策略调度器（`manager.rs:3003-3420`）。RustFS 表达力更强，代价是"重复请求被合并"的可观测性问题（v1 已指出，现有 `HealAdmissionReceipt` canonical task_id + alias 机制回应了它，`manager.rs:1759-1846`）。 -2. **scanner 远端盘访问**：MinIO leader 通过磁盘抽象层透明读写远端节点磁盘；RustFS leader 通过 remote_scanner RPC 把扫描执行下放到远端 peer 本地进行（`crates/scanner/src/remote_scanner.rs`），只回传结果与进度心跳。两者都是集群单 leader。RustFS 方案省 leader↔远端的元数据读放大，代价是需要维护独立 RPC 协议（HMAC 逐帧认证、会话重放缓存、fence 复验，`remote_scanner.rs:52-61,405-496,1024-1065`）。 -3. **heal 状态持久化**：MinIO 用单文件 `.healing.bin`（msgp healingTracker，diskID 不匹配即重置）；RustFS 用 schema 化多文件（resume/checkpoint/intent/seal/proof 各自 CAS 发布，`resume.rs:38-61`），崩溃窗口显式补齐（`erasure_healer.rs:389-402`、`resume.rs:1027-1057`）。 -4. **写路径自保护**：MinIO 写入后靠后台 heal 收敛；RustFS 在 PutObject/CompleteMultipartUpload 提交 rename 后主动检查 `convergence.needs_heal()` 并立即入队对象 heal（`set_disk/ops/object.rs:2291-2306`、`ops/multipart.rs:2574-2589`），另有读修复 read repair（`io_primitives.rs:1040-1160`）。 - ---- - -## 2. Heal 已实现功能全景 - -### 2.1 任务类型（`HealType`，`crates/heal/src/heal/task.rs:85-111`） - -| 类型 | 语义 | 执行体 | 生产触发方 | -|---|---|---|---| -| `Cluster` | 所有 bucket 依次 heal（结构 + 可选递归对象），批内重试 ≤3 | `heal_cluster` task.rs:1420-1490 | channel：bucket 为空即 Cluster（channel.rs:576-577） | -| `Object{bucket,object,version_id}` | 单对象/版本；不存在时按 `recreate_missing` 重建或报错 | `heal_object` task.rs:855-1146 | admin、scanner、read-repair、写路径收敛、add_partial | -| `Bucket{bucket}` | 桶元数据/结构；`recursive` 再遍历全部对象版本 | `heal_bucket` task.rs:1284-1418 + `heal_bucket_objects` task.rs:1508-1698 | admin（POST /v3/heal/{bucket}）、scanner `build_bucket_heal_request` | -| `Prefix{bucket,prefix}` | 按前缀递归 | `heal_prefix` task.rs:1492-1506 | channel：`recursive && prefix` 非空（channel.rs:578-585） | -| `ErasureSet{buckets,set_disk_id}` | format 修复 + healing 标记 + 逐桶预处理 + 可恢复逐版本深扫 | `heal_erasure_set` task.rs:2158-2642 | admin（pool/set 参数）、auto disk scanner、unclean shutdown、renew_disk、durable replacement 恢复 | -| `Metadata{bucket,object}` | 仅元数据（Deep、不重建数据） | `heal_metadata` task.rs:1700-1859 | **无生产触发方**（§6 HS-01） | -| `MRF{meta_path}` | 失败路径驱动的 Deep 修复（recursive+update_parity） | `heal_mrf` task.rs:1861-1992 | **无生产触发方**（仅 `HealEvent` 可生成，未接线） | -| `ECDecode{bucket,object,version_id}` | EC 解码重建（Deep+recreate+update_parity），Urgent 优先级 | `heal_ec_decode` task.rs:1994-2156 | **无生产触发方**（仅 `HealEvent` 可生成，未接线） | - -优先级 `Low/Normal/High/Urgent`（task.rs:168-179）；状态机 `Pending/Running/Retrying/Completed/Failed/Cancelled/Timeout`（task.rs:225-241）。 - -### 2.2 触发路径全景（admin 之外） - -| 通道 | source | 优先级 | 证据 | -|---|---|---|---| -| Scanner 周期抽样（1/1024，`RUSTFS_HEAL_OBJECT_SELECT_PROB`） | Scanner | Low | `scanner_folder.rs:2117-2136`、`:1150`；`remove_corrupted=HEAL_DELETE_DANGLING(true)`、`recreate_missing=false`（`common/heal_channel.rs:24`、`scanner_folder.rs:510-511`） | -| Scanner 元数据损坏（get_size 失败分类 HealMetadata） | Scanner | High | `scanner_folder.rs:2147-2208`、`:1244-1260` | -| Scanner abandoned children（缓存有、盘上无，list_path_raw quorum 核查） | Scanner | High（桶级+对象级） | `scanner_folder.rs:2528-2792` | -| Scanner pending-heal 账本重试（heal 通道满被拒后持久化，每桶每轮 ≤128 条、上限 10k） | Scanner | 原优先级 | `scanner_folder.rs:1721-1763`、`:99-100` | -| auto disk scanner（unformatted 盘经 replacement_readiness 确认 / `runtime_state=="returning"` 盘 / durable intent 重入） | AutoHeal | Low | `manager.rs:2464-2999` | -| unclean shutdown 恢复（启动读 `unclean-shutdown` 标记 → 全部本地 set ErasureSet heal） | AutoHeal | Low | `manager.rs:1362-1695` | -| 写路径收敛（PutObject/CompleteMultipartUpload 后 `convergence.needs_heal()`） | Internal | Normal | `set_disk/ops/object.rs:2291-2306`、`ops/multipart.rs:2574-2589` | -| 部分对象 heal（add_partial） | Internal | Normal | `set_disk/ops/object.rs:5808-5825` | -| 旧数据目录清理残留 enqueue | Internal | Normal | `set_disk/core/io_primitives.rs:3880-3907` | -| 读修复（metadata_read_error / missing_shards / decode_error，TTL 去重缓存） | ReadRepair | Low | `set_disk/read.rs:407,995,1079` → `submit_read_repair_heal`（`io_primitives.rs:1105-1160`），`recreate_missing=true` | -| 盘重连遇 UnformattedDisk → send_heal_disk | AutoHeal | Normal | `set_disk/ops/locking.rs:339-347` | -| Admin API（含集群 coordinator 路由） | Admin | High | `rustfs/src/admin/handlers/heal.rs:174-212`、`:771-930` | -| 集群 RPC heal（peer 调用） | — | — | `rustfs/src/storage/rpc/node_service/heal.rs`、`ecstore/src/cluster/rpc/peer_s3_client.rs:296,1209` | - -注意：MinIO 的 MRF 通道（读路径检出 part 缺失/损坏即时投递 + 队列持久化 + shutdown 回放，`cmd/mrf.go`、`erasure-object.go:395-410,800-812`）在 RustFS 由 read-repair + 写路径收敛**部分替代**；`HealType::MRF`/`ECDecode`/`Metadata` 三个执行体没有生产入口（详见 §6 HS-01）。 - -### 2.3 对象级 heal 语义（ecstore `set_disk/ops/heal.rs`） - -流程（`heal_object_with_explicit_version_regen` :426 起）： - -1. 取对象写锁（除非 `no_lock`）；`object` 以 `/` 结尾走对象目录 heal（`heal_object_dir_locked` :1587-1717：dangling 判定 + `remove` 删除 + 缺 volume 重建）。 -2. `read_all_fileinfo` 全盘读 xl.meta，全部 not-found 视为已删除返回。 -3. **quorum 仲裁 + ETag 兜底**（已亲验）：`list_online_disks` 以 mod-time quorum 为准；quorum 失效时回退 ETag 多数派仲裁（`:525-567` `filter_by_etag`/`quorum_etag`）；`pick_valid_fileinfo` 选 canonical 元数据；"meta 坏盘数 > parity" 的 cannotHeal 判定在 ETag 全盘一致时豁免（`:679`）。与 MinIO `filterDisksByETag` 双仲裁一致。 -4. `disks_with_all_parts`（:562-572）按 `scan_mode` 校验 part：**Normal 仅 stat（CheckParts 语义），Deep 做全量 bitrot 校验（VerifyFile 语义）**；Normal 扫描检出 `FileCorrupt` 自动升级 Deep 重试一次（`:2022-2031`，与 MinIO erasure-healing.go:1101-1106 同型）；无 parity 对象（EC:0）bitrot 失败判不可恢复（`:700-726`）。 -5. `should_heal_object_on_disk`（:606-650）逐盘分类 missing/corrupt/offline/outdated → 重建：per-part bitrot reader/writer（用 per-part checksum + 算法）、写临时卷后 rename 提交（`HEAL_RENAME_INCOMPLETE` 重试语义 :24）；dangling 删除安全检查 `dangling_delete_safety`（:1488）；**孤儿数据目录回收 `reclaim_orphan_data_dirs_best_effort`（:1428）**——这部分覆盖了 MinIO `CleanAbandonedData` 的主场景（但无独立 `CheckAbandonedParts` API，见 §6 HS-02）。 -6. 版本化对象：枚举"每个版本"（`storage.rs:1494-1530`）；delete-marker 路径由 `latest_meta.deleted` 决定（`storage.rs:262-277` 注释）；回归测试 `tests/heal_b5_versioned_regression_test.rs:282,334`。 -7. 显式版本重建 `try_regenerate_explicit_version_meta`（:1318）；transitioned 对象本地残留清理。 -8. 写入路径另有 shard 级 bitrot 自校验 `verify_written_bitrot_shards`（`ops/bitrot_self_verify.rs:45-129`，HighwayHash256S，最终 rename 前校验刚写出的 shard，服务 EC:0 无 parity 场景）——**注意这不是后台 bitrot 巡检**；后台巡检由 scanner bitrot_cycle 驱动 Deep heal 承担。 - -heal crate 侧包装（`task.rs:855-1146`）：存在性检查（瞬时错误转 `TransientSkip` 不误判失败 :551-569）；scanner 合成目录规范化（:1148-1180）；`recreate_missing` 重建（:1183-1282）；data-usage-cache 对象锁超时豁免（:571-653）；not-found → treated_as_deleted 成功（:1012-1029）；结果 `HealResultItem` 保留至多 1024 条 + truncated 标志（:50,845-852）。 - -递归遍历（`heal_bucket_objects` task.rs:1508-1698）：分页枚举全部版本含 delete marker、瞬时错误指数退避重试 ≤3（2^n + 抖动 :620-627）、失败样本日志截断 ≤5 条、聚合 `BatchHealFailure`。 - -### 2.4 erasure set heal 与断点续扫 - -`heal_erasure_set`（task.rs:2158-2642）四阶段（4 步进度跟踪）： - -1. **替换意图与恢复盘选择**（仅 AutoHeal + heal_endpoints 非空）：复用 durable intent 所在盘 / 排除目标端点选幸存盘；已完成代（CleanupPending）幂等收尾。 -2. **格式修复**：`heal_replacement_format(dry_run, pool, set, targets)`（`storage.rs:1372-1384`，trait 默认实现 fail-closed）；逐目标盘结果必须全 ok（`erasure_healer.rs:97-102`）+ 身份围栏复核（task.rs:2410-2420）。 -3. **healing 标记**：对目标盘写 owner CAS 标记 `{set_disk_id}:{task_id}`（`mod.rs:80-229`，CAS + 回滚 + 并发唯一 owner），使 `DiskInfo.healing` 为真（已亲验赋值链 `set_disk/mod.rs:4988`）。 -4. **逐桶预处理 + 可恢复深扫**：`ErasureSetHealer::heal_erasure_set`（`erasure_healer.rs:242-278`）。 - -`ErasureSetHealer` 扫描细节（对标 MinIO `healErasureSet`，`heal_walk.rs:15-23` 模块注释明确引用 MinIO `global-heal.go` 的 listPathRaw + objQuorum=1 + mergeXLV2Versions）： - -- **枚举器选择（backlog#920）**：Deep 或 AutoHeal → per-set **disk-walk 并集枚举** `list_versions_for_heal_page_disk_walk`（"任意盘上存在"即 sub-quorum 可重建；`storage.rs:1559-1644`，页界 1000 对象/10,000 版本，`dw1:` cursor）；普通请求走 read-quorum `list_object_versions`。 -- **续扫游标**：权威 cursor 为 opaque continuation token（`v1:`=marker JSON、`dw1:`=disk-walk key，两命名空间互斥防误读，`storage.rs:81-260`）；每完成一页先持久化 cursor 再清 dedup 集合（`erasure_healer.rs:922-927`）。 -- **页内并发**：FuturesUnordered + Semaphore，默认 `RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY=8`，Deep/AutoHeal 强制 1（`erasure_healer.rs:105-142`）。 -- **per-version dedup**：`compose_key` 长度前缀注入编码（`resume.rs:281-288`）。 -- **错误分类**：真缺席（FileNotFound 等）→ Absent（计成功）；基础设施瞬时（quorum/DiskNotFound/SlowDown 等）→ Transient（计 skipped）；其余 Failed（`erasure_healer.rs:148-182`，注释引 backlog#856/#799 B7：离线盘不得记 healed/absent）。 -- **防死循环**：空页 truncated 或页尾版本身份不前进即中止（:933-949）。 -- **完成判定**：failed/skipped/failed_buckets 任一 >0 不标记完成，`schedule_retry()` 复位 resume+checkpoint 两层（:561-626，backlog#855/B6/#1033：skip 轮不得标记完成）。 -- **替换盘提交证据**：目标端点物理回读 `replacement_targets_have_version`（`ops/heal.rs:340-412`），未确认 → transient skip。 - -### 2.5 换盘自动修复（replacement recovery） - -- **识别**（`replacement_readiness.rs:25-73`）：`replacement_mount_lease_root()` 存在、canonicalize 成功、是挂载点、物理设备 id 非空、与根设备不相交、不与兄弟盘共享物理设备（Linux 用 /proc/self/mountinfo mount-id+dev+ino）。非 root 挂载检查有回归测试（`manager.rs:3549`）。 -- **状态机**（`resume.rs:63-73`）：`Intent → Rebuilding →（写 proof）Verified → CleanupPending → 清理`；`Abandoned` 终态；跨状态迁移先写持久层再变更（`save_state_strict`）。 -- **持久化**（`resume.rs:38-61`，schema ResumeState=5/Checkpoint=5/proof=1）：`{task_id}_ahm_resume_state.json`、`_ahm_checkpoint.json`、`buckets/ahm-replacement/` 命名空间下 intent/seal/completion_proof；torn write + 无 seal 可识别并原子重建（:1316-1338）；CAS 发布、拒绝覆盖并发有效 proof（:1512-1585）。 -- **恢复**：unclean shutdown 与周期扫描都从幸存盘恢复未完成/待清理替换代（`manager.rs:1435-1640,2663-2815`）；多代冲突/校验失败 → 冻结该 set（`replacement_recovery_blocked_sets`，`manager.rs:69-87,2782-2815`）。 -- **对外快照**：`current_replacement_recovery_snapshot`（`lib.rs:262-333`）合并本地幸存盘记录，冲突 → Unknown/非 definitive；admin `GET /v4/heal/replacement-recovery`。 - -### 2.6 调度器（manager.rs） - -- 优先级堆 + 同优先级 FIFO（:148-191,330-347）；dedup key 按类型（:469-506）；入队三态查重 active→queued→retrying（:1759-1785）；重复默认 Merged 并返回 canonical task_id（`HealAdmissionReceipt`，:1821-1846）+ client token alias（:1219-1246）。 -- 容量：队列满时 best-effort 来源（Scanner/AutoHeal/ReadRepair）或低优先级被 Dropped(QueueFull)；Admin/Internal 可驱逐低优先级排队项（`push_displacing_lower_priority` :353-396）；80%/95% 压力分级（:885-909）。 -- 并发：全局 `max_concurrent_heals`（默认 4）+ per-set bulkhead `max_concurrent_per_set`（默认 1）（:3040-3073,3434-3447）。 -- 前台压力门控 mainline throttle：前台读/写 permit 利用率 ≥80% 时延迟 best-effort 任务（:919-1009,2999-3020）。 -- 超时：任务级聚合超时（默认 300s），跨重试保留剩余预算（task.rs:444-451，PR #6101）。 -- 可恢复重试：`is_recoverable_heal()`（error.rs:83-136）≤3 次、2^n 退避封顶 30s；retry 在独立 backoff task 中持有所有权（:3235-3382）。 -- 完成态保留 10 分钟供查询（:42）。 - -### 2.7 Admin API 与集群协调 - -- 路由（`rustfs/src/admin/handlers/heal.rs:174-212`）：`POST /rustfs/admin/v3/heal/`、`/heal/{bucket}`、`/heal/{bucket}/{prefix}`（同一 POST 按 query `clientToken/forceStart/forceStop` 区分 start/query/cancel，与 mc admin heal 语义对齐）；`POST /v3/background-heal/status`；`GET /v4/heal/replacement-recovery`。权限 `HealAdminAction`（route_policy.rs:334-341）。 -- 集群协调（heal.rs:771-930 + `node_service.rs:514-606`）：`heal_topology_fingerprint` + 按拓扑确定性选 coordinator 节点 + coordinator epoch；envelope 校验 + SHA256 digest 重放缓防重放；coordinator 非本机走 peer gRPC `heal_control`；`probe_heal_control` 能力探测（滚动升级场景）。 -- 请求：body 为 `HealOpts`（`recursive/dryRun/remove/recreate/scanMode(0/1/2)/updateParity/nolock/pool/set`，serde camelCase，与 madmin.HealOpts 字段对齐）；根 heal start 需 `recursive=true` 或 `pool+set` 成对；body 上限 1MB。 -- 响应：`HealStartSuccess{clientToken, clientAddress, startTime}`；`HealTaskStatus{summary, detail, startTime, settings, items, truncated, progress}`（summary ∈ running/finished/stopped/notFound）；`BackgroundHealStatus`（bitrot 起始时间/周期/当前模式 + `disabled/uninitialized/idle/active/degraded` 状态——peer 不可达显式 degraded 不冒充 idle，issue #5850 + `healOperations` 按优先级×来源矩阵 + 集群进度）。 -- `HealResultItem`/`HealDriveInfo`/`HealItemType`/DriveState 枚举与 madmin JSON 兼容（`crates/madmin/src/heal_commands.rs:19-65`）。 -- 状态 payload 超 8MiB 对折截断（channel.rs:37,73-104）；path-token 校验（错误 token 拒绝，空 path 仅匹配 Cluster）。 - -### 2.8 heal 指标与日志 - -指标：`rustfs_heal_admission_total{source,result,reason,context}`、`rustfs_heal_task_start_total`、`rustfs_heal_task_running{type,set}`、`rustfs_heal_queue_delay_seconds`、`rustfs_heal_scheduler_skip_total`、`rustfs_heal_mainline_throttle_total`、`rustfs_heal_page_concurrency_current{set}`、`rustfs_heal_candidate_enqueue/merge/drop/priority_reject_total`、`rustfs_heal_read_repair_dedup_total{reason}` 等。日志全部结构化 event style（PR #5720）；per-object 日志降级防风暴（`demote_to_debug_when!`，#5716/#5719/#5727）。 - ---- - -## 3. Scanner 已实现功能全景 - -### 3.1 循环、leader、立即触发 - -- **集群单 leader**：分布式 ns 写锁 `leader.lock`（`scanner.rs:3156-3207`，超时默认 5s）+ **持久化 leader-epoch CAS 围栏**：leader 用 ETag 前置条件向 `.bloomcycle.bin` 写 `RSCYC001` 编码的 (cycle, leader_epoch)（`scanner.rs:118,1850-1861,2177-2334`）；usage 快照再打 epoch fence（:2087-2153）。锁丢失 → 取消当前周期，30s 收敛（:108-111,2623-2642）。 -- 抢锁后立即执行一轮；周期 = `RUSTFS_SCANNER_CYCLE` > config cycle > start_delay > 部署默认 > 速度档位（±10% 抖动、下限 1s）。 -- **clean-idle 指数退避**：连续完整无脏周期间隔 ×2（封顶 24h；bitrot 周期压缩上限；桶有 lifecycle/replication 活动规则禁用，:383-456,1382-1512）。 -- **superseded/deferred 退避**：5s 起指数退避封顶 30min（:105-106,3432-3438）；维护探测失败独立退避（:459-505）。 -- **立即唤醒**：① dirty-usage 快路径——写路径 put/delete/multipart/bucket 操作调用 `record_dirty_usage_bucket`（`scanner_io.rs:222-235`；调用点 `rustfs/src/app/object_usecase.rs:6221` 等），自增 generation 并 Notify 唤醒 leader，脏桶优先排队（`scanner_io.rs:462-488`）；② 维护配置变更（lifecycle/replication 设置时 `record_scanner_maintenance_change`）；③ 运行时配置热更 generation+Notify；④ 集群活动快照变化。 -- **集群协调**：`probe_scanner_activity` 汇集本机+peer 的 `ScannerNodeActivity`（instance_id/namespace_generation/maintenance_generation/protocol_version/topology_digest/data_movement_active/dirty usage），拓扑摘要覆盖 pools/sets/drives URL，协议版本不齐拒绝共享缓存锁（`scanner.rs:970-1068`）；**数据迁移（rebalance/decommission）期间推迟周期**（`scanner_io.rs:2226-2374`）；周期结束逐 peer RPC 确认 dirty-usage ack（`scanner.rs:2925-2952`）。 - -### 3.2 遍历模型 - -- 主遍历是**全量目录 walk**（tokio::fs::read_dir 递归，`scanner_folder.rs:1915-2234`），不走 metacache；metacache/`list_path_raw` 仅用于 abandoned children 跨盘核查（:2528-2792）。 -- 三级并发：leader → per-set（信号量默认 4）→ per-disk 桶扫描（默认 4）→ 单盘递归；每桶每 set 缓存锁 `.scanner-cycle.lock.pool-N.set-M`（锁丢失取消该桶扫描，锁竞争重排队）；每盘单扫描准入（本地盘也走信号量，`scanner_io.rs:3246-3274`）。 -- 桶顺序：shuffle 后按 dirty → 未缓存 → 已缓存重排（`scanner_io.rs:2947-2949,462-488`）；目录内按名字排序 + resume 提示旋转（`scanner_folder.rs:333-359`）。 -- **断点续扫**：`DataUsageScanCheckpoint{version,resume_after,reason}` 持久于缓存 info（`data_usage_define.rs:68,293-307`）；预算耗尽/取消写入，恢复有 Used/Stale/NoHint 指标；续扫单位是目录（无跨周期对象级分页）。 -- erasure 语义：发现 `xl.meta` 即对象边界不下钻；UUID data-dir 候选最多探测 64 entry；有数据无元数据 → 记 failed + 高优 heal；symlink 目录忽略/环跳过。 -- 协作让出：每 N 对象（默认 128）`yield_now`。 - -### 3.3 大桶跳过策略（对标 MinIO compaction） - -1. 缓存当前性复用：桶与扫描计划未变（name/source/snapshot_complete/plan digest/next_cycle/leader_epoch/cache_key_format 全匹配）整桶跳过（`scanner_io.rs:1062-1109`）。 -2. compacted 目录 16 周期轮换窗口：`hash mod (next_cycle, 16)` 命中才重扫，否则从旧缓存拷贝（`scanner_folder.rs:74,2429-2442`）。 -3. compaction 阈值：子项 <500 或纯对象叶子压缩为单 entry；子文件夹 ≥2500（根 10000）预压缩；children ≥10000 归约（:75-78,2314-2340,2846-2887）。 -4. 失败对象 TTL 跳过：86400s/最多 10000 条（:88-91,1354-1381）。 - -与 MinIO master 对比：MinIO 的跳过策略同样是 hash-mod-16 周期 + compaction 阈值树（500/10000/2500），**bloom filter 已从 master 删除**。RustFS 的常量与结构与 MinIO 现状同源（MinIO 未采用跨盘 dirty-generation 优先，RustFS 额外多两层跳过——plan digest 与缓存当前性校验）。 - -### 3.4 data usage 统计 - -- 维度：每目录 entry（size/objects/versions/delete_markers/大小直方图/版本直方图/复制统计/failed_objects/per-tier stats/children/compacted，`data-usage/src/data_usage.rs:661-679`）；每对象 SizeSummary（含 per-ARN 复制目标统计、tier 统计，tier 分类：transitioned 完成记入其 tier 否则按 storage class，free version 不计）；桶级 `BucketUsageInfo`；集群级 `DataUsageInfo`（含 scanner_cycle/scanner_epoch 围栏 + usage_snapshot_complete）。 -- 存储：每桶每 set `{bucket}/.usage-cache.bin`（主 + `.bkp` 备份 + CAS 重试）；权威集群快照 `buckets/data-usage/data-usage.json`（每 10 周期同步 `.bkp`，legacy 路径兼容）；陈旧快照拒绝写入（epoch/cycle/last_update 三重判定）；被竞争 superseded 的观测快照另存 `data-usage-observed.json`。 -- 消费：`replace_bucket_usage_memory_from_info` 刷新桶用量内存 + 两层缓存失效（`scanner.rs:4142-4152`）→ bucket stats/quota/admin account_info/system；写路径内存实时叠加 overlay；启动读快照判断冷缓存跳过启动延迟。 -- 未完成 multipart 不参与统计（与 MinIO 一致，MinIO 也不扫 multipart 桶）。 - -### 3.5 ILM 集成 - -- 每对象 `ScannerItem::apply_actions`（`scanner_folder.rs:747-1032`）：`Evaluator::new(lifecycle).with_lock_retention(...).with_replication_config(...).eval()` 批量评估。 -- 已实现动作（IlmAction 全集，`scanner-contracts/src/metrics.rs:34-45`）：expiry 删除（Delete/DeleteRestored/DeleteRestoredVersion）、全版本删除（DeleteAllVersions/DelMarkerDeleteAllVersions，处理后停止后续版本）、transition（Transition/TransitionVersion，tier 列表运行时读取）、noncurrent 批量（DeleteVersionAction → `enqueue_by_newer_noncurrent`）、free-version 清理（`enqueue_free_version`）、object-lock retention 约束。**与 MinIO 的 9 个 ILM 动作一一对应**。 -- 执行模型：scanner 是"发现与入队"角色（expiry 队列/transition 队列在 ecstore `bucket_lifecycle_ops.rs`），动作由 worker 池消费——与 MinIO globalExpiryState/globalTransitionState 同型。 -- AbortIncompleteMultipartUpload 不在 scanner/ILM 内执行（MinIO 同样不在：`internal/bucket/lifecycle/rule.go` 有 FIXME，实际由 `erasureSets.cleanupStaleUploads` 全局例程承担）；RustFS 由 ecstore 独立后台任务 `init_background_stale_multipart_upload_cleanup`（`bucket_lifecycle_ops.rs:3289-3320`）+ 桶删除时 on-demand。 -- 集成测试覆盖：transition+restore、free-version、noncurrent、delete-marker、0-day、后台扫描过期（`scanner/tests/lifecycle_integration_test.rs:1071-2095`）。 - -### 3.6 heal 候选生产（scanner 侧） - -- 抽样：`hash mod_alt(next_cycle/prob_div, 1024/prob_div)`，进入 compacted 分支重扫时 prob_div=16 等效概率 ×16（与 MinIO 同款补偿，`scanner_folder.rs:125-127,2117-2122`）。 -- deep/normal：周期级 `get_cycle_scan_mode`（bitrot_cycle 默认 30d，`scanner.rs:1626-1657`）→ 对象级带 `HealScanMode::Deep`；新鲜对象（60s 内修改）降级 Normal（:146-155）；状态持久 `.background-heal.json`（`BackgroundHealInfo{bitrot_start_time,bitrot_start_cycle,current_scan_mode}`，与 MinIO 同路径同结构）。 -- scanner 只入队不内联执行（内联 heal 已移除，兼容旗标仅告警，`scanner_folder.rs:411-427`）；`HealScanMode::Deep` 只是标记，bitrot 校验读发生在 heal 消费端（ecstore Deep 路径）。 -- 元数据损坏 → 高优 heal（`classify_get_size_failure` → HealMetadata）；abandoned children → list_path_raw quorum 核查 + 桶级/对象级高优 heal；healing 盘粘性跳过（`should_heal` :1628-1648）。 -- pending-heal 账本：heal 通道满被拒持久化到缓存 info，下轮重试。 -- 复制 heal：`queue_replication_heal` → replication 队列（走 replication 通道而非 heal channel）；per-ARN 复制用量统计。 - -### 3.7 remote_scanner RPC 协议（RustFS 特有） - -请求 ≤16KB msgpack（version/request_id/server_epoch/session_id/session_sequence/bucket/next_cycle/leader_epoch/scan_plan_digest/skip_healing/scan_mode/budget）；帧 ≤2MB、HMAC-SHA256 逐帧认证（域 `rustfs-ns-scanner-frame-v3`）；进度心跳 1s（预算模式 250ms）；阶段播报 Scanning→Persisting；RPC 生命周期上限 24h、断连宽限 2min；防重放 session+sequence 缓存（容量 65536）；服务端校验 leader fence 与持久化 cycle 一致 + 每 5s fence 复验；结果 Complete/Partial/NamespaceNotFound/CycleAhead；不支持 v4 协议的远端盘回退 leader 本地扫描（`remote_scanner.rs` 全文件；`scanner_io.rs:2750-2812`）。 - -### 3.8 限速/预算/热更/观测 - -- DynamicSleeper 比例退避（速度档 fastest/fast/default/slow/slowest，同 MinIO 五档参数）；idle_mode 总闸；前台 S3 读流量每请求 10ms 封顶 250ms 额外退避。 -- 周期预算 ScannerCycleBudget：max_duration/max_objects/max_directories（默认 0=不限），partial 周期仍推进 cycle 计数。 -- runtime_config 三层来源（env > config > default）逐字段来源标记（Env/Config/ScannerCompatConfig/Default），admin `PUT /v3/config` 热更 → generation+Notify 即时生效；`GET /v3/scanner/status` 返回 enabled/freshness(fresh/stale/unknown)/metrics/cycle_schedule/runtime_config；`GET /v3/ilm/expiry/status` 返回 expiry 队列/worker/missed/blocked。 -- 指标：leader lock、周期 complete/partial/deferred/superseded、versions scanned、per-source（Usage/Lifecycle/BucketReplication/SiteReplication/Heal/Bitrot/Alerts）checked/executed/queued/missed、checkpoint set/used/stale、当前路径（per-disk+bucket 实时）、缓存 save 系列、并发系列、告警（excess versions/version size/folders）。 - ---- - -## 4. 与 MinIO 逐项对标 - -### 4.1 heal 触发通道对照 - -| MinIO 通道 | RustFS 对应 | 状态 | -|---|---|---| -| A. 手动 admin heal（healSequence，clientToken/forceStart/forceStop） | heal channel Start/Query/Cancel + 集群 coordinator + envelope 重放防护 | ✅ 等价且增强（集群路由）；序列语义差异见 §6 HS-06 | -| B. 常驻后台 heal 队列（newBgHealSequence + healRoutine worker 池） | HealManager 常驻调度器 + 优先级队列 + bulkhead | ✅ 等价且增强 | -| C. 新盘/换盘自动 resync（monitorLocalDisksAndHeal 10s + healFreshDisk + healingTracker + waitForFormatErasure 握手） | auto disk scanner（10s）+ replacement_readiness + durable intent/proof 状态机 + heal_replacement_format | ✅ 等价且增强（identity fence + completion proof；MinIO 的 tracker 面向对外可见性更强，见 §6 HS-07） | -| D. MRF（队列 100k + 持久化 list.bin + shutdown 回放 + 读路径 corrupt 投递） | read-repair（Low+TTL 去重）+ 写路径 convergence heal 部分承担；`HealType::MRF` 执行体无生产入口 | ⚠️ 部分等价（§6 HS-01） | -| E. Scanner 抽样 heal（1/1024 + compacted ×16 补偿）+ abandoned children | 同款抽样 + ×16 补偿 + abandoned children + pending-heal 账本 | ✅ 等价且增强（账本） | -| F. 读路径内联触发 → MRF（GetObject part 缺失/损坏、元数据重建 missingBlocks>0） | read repair（missing_shards/decode_error/metadata_read_error 三入口） | ✅ 等价（入 heal 队列而非 MRF 队列） | - -### 4.2 对象级 heal 语义对照 - -| 特性 | MinIO | RustFS | 状态 | -|---|---|---|---| -| mod-time quorum 仲裁 | listOnlineDisks | 同 | ✅ | -| ETag 多数派兜底（时钟漂移） | filterDisksByETag | `filter_by_etag`/`quorum_etag`（heal.rs:525-567） | ✅ 已亲验 | -| cannotHeal 的 ETag 豁免 | ETag 全一致豁免重试 | heal.rs:679 | ✅ | -| Normal=CheckParts（stat）/ Deep=VerifyFile（bitrot） | 是 | `disks_with_all_parts` 按 scan_mode（ops/heal.rs:562-572,978-1024） | ✅ | -| Normal 检出 corrupt 自动升 Deep 重试一次 | erasure-healing.go:1101-1106 | ops/heal.rs:2022-2031 | ✅ | -| dangling 判定（not-found > parity）+ 删除审计 | isObjectDangling/deleteIfDangling | `dangling_delete_safety`（:1488）+ scanner HEAL_DELETE_DANGLING | ✅（审计 tags 细节有差异） | -| 孤儿 data-dir/inline 清理（CleanAbandonedData） | CheckAbandonedParts（scanner 抽中 + admin Remove 时显式调用） | heal 路径内 `reclaim_orphan_data_dirs_best_effort`（:1428）；独立 API 三层 NotImplemented | ⚠️ 部分等价（§6 HS-02） | -| 版本化/delete-marker heal | HealObject versionID；nullVersionID 特判 | 逐版本枚举 + delete-marker latest heal（B5 回归） | ✅ | -| 对象级 healing 元数据标记（x-minio-healing，RenameData 跳过版本清理） | 有 | 无对象级标记；依赖盘级 healing.bin + NSLock + rename 语义 | ⚠️ 评估项（§6 HS-12） | -| Distribution/Index 一致性三处防线 | 有（manual modification 拒绝） | 目标盘格式结果全 ok 校验 + 身份围栏 | ✅（粒度不同） | -| 无 parity（EC:0）对象 | bitrot 不可恢复处理 | 判不可恢复（:700-726）+ 写入自校验 | ✅ 增强（写路径自校验） | -| 三层分布不一致拒绝 heal | 有 | heal_walk 归一化 + 页界防御 | ✅（实现方式不同） | -| multipart 孤儿对账 | CheckAbandonedParts 承担 | 显式 NotImplemented（由 lifecycle 清理承担） | ⚠️ §6 HS-02 | -| suspended/decommissioned pool 处理 | IsSuspended 跳过 | deferral 语义（store/heal.rs:192-207，PR #5876） | ✅ | -| heal 与并发删除互斥 | NSLock + healing 标记 | NSLock + 写锁 | ✅ | - -### 4.3 新盘 resync 对照 - -| MinIO | RustFS | 状态 | -|---|---|---| -| waitForFormatErasure 四类可恢复错误无限等待握手 | startup 盘解析 + renew_disk 重连路径 | ✅（模型不同：RustFS 不在启动时阻塞等待 format） | -| HealFormat NSLock + errNoHealRequired + refFormat 不一致拒绝 | `heal_format`/`heal_replacement_format` fail-closed + 目标槽位限定（PR #1787 语义） | ✅ 增强 | -| per (pool,set) 分布式锁防并发 resync | set 级队列去重 + bulkhead（manager.rs:2854-2889） | ✅ | -| 全新集群检测（待 heal 盘数==总盘数不触发） | replacement_readiness（独立挂载点/物理设备校验，非 root） | ✅ 增强 | -| healingTracker（.healing.bin：Bytes/Items 计数、QueuedBuckets/HealedBuckets、Resume 快照、RetryAttempts ≤4、HealID 联动、diskID 变更重置） | resume/checkpoint schema 化持久层 + durable intent/proof（per-task 文件，CAS） | ✅ 等价且增强（崩溃窗口补齐）；但**对外快照可见性**弱于 MinIO（§6 HS-07） | -| 跳过 heal 开始后新写入版本（ModTime > Started） | 无同款过滤 | ⚠️ §6 HS-13 | -| 跳过 ILM 已过期版本（filterLifecycle） | 无同款过滤 | ⚠️ §6 HS-13 | -| worker 数 max(GOMAXPROCS,NR)/4 下限 4，heal:drive_workers 覆盖 | 页内并发 8（Deep/AutoHeal 强制 1）+ per-set bulkhead | ✅（参数模型不同） | -| 每 entry waitForLowHTTPReq 让路 | mainline throttle（前台利用率门控） | ✅ 增强 | -| heal 范围含 `.minio.sys/config`、`.minio.sys/buckets` 两个伪桶；最新桶优先 | ErasureSet 任务逐 bucket 预处理（含 meta bucket 语义由 heal_bucket 承担） | ✅（顺序无"最新优先"） | -| 失败整体重试 ≤4 次（resetHealing + errRetryHealing） | schedule_retry 复位双层 + 可恢复重试 ≤3 | ✅ | - -### 4.4 scanner 对照 - -| MinIO | RustFS | 状态 | -|---|---|---| -| 集群单 leader（globalLeaderLock） | leader.lock + 持久化 leader-epoch CAS 围栏 | ✅ 增强（epoch 围栏防脑裂，MinIO 无持久化 epoch） | -| `.bloomcycle.bin` 只存 cycle（bloom 已删除） | 同路径存 cycle+leader_epoch（RSCYC001） | ✅ 对齐（v1 误判已修正） | -| folderScanner hash-mod-16 + compaction（500/10000/2500） | 同款常量 + plan digest + 缓存当前性校验 + dirty 优先 | ✅ 增强 | -| 每盘扫描并行 ≤GOMAXPROCS；healing 盘排除 | per-set/per-disk 信号量 + healing 盘粘性跳过 | ✅ | -| scannerSleeper（factor 2/max 1s，speed 档热更） | DynamicSleeper 同款 + idle_mode + 前台读退避 | ✅ 增强 | -| idle 语义：`scanner:idle_speed=on`（空闲时段才节流，忙时全速） | `RUSTFS_SCANNER_IDLE_MODE=true`（启用限速总闸） | ⚠️ 语义方向相反，§6 HS-14 | -| applyActions 顺序（heal→ILM→复制→告警） | apply_actions 同序（heal 候选→ILM→复制 heal→告警） | ✅ | -| ILM 9 动作 + 批量评估 + DeletePrefixObject 优化 | 同 9 动作 + 批量评估 + expiry 队列 | ✅（DeleteAllVersions 是否单调用优化未逐行核） | -| abandoned children（listPathRaw minDisks=N/2 发现漏写盘） | list_path_raw + quorum 核查 + 高优 heal | ✅ | -| incomplete multipart 独立例程（6h 间隔/24h 过期，rename 进 .trash） | ecstore 独立后台任务（可配间隔/过期） | ✅（trash 二段清理细节差异，§6 HS-18） | -| usage 维度（size/objects/versions/DM/直方图/复制/tier/bucket 级） | 全覆盖 + 集群快照三重防回退 | ✅ 增强 | -| prefix 级 usage（loadPrefixUsageFromBackend，console 消费） | 缓存内有目录树但仅 flatten 桶级 | ❌ §6 HS-08 | -| 超限事件 s3:ObjectManyVersions/LargeVersions/PrefixManyFolders + 审计 | 仅指标 alert_excess_*（默认 100/1TiB/65538 vs MinIO 100/1TB/50000） | ⚠️ §6 HS-04/HS-17 | -| scanner 指标 v3（bucket_scans/directories/objects/versions/last_activity） | rustfs_scanner_* 全套 + freshness | ✅（命名体系不同） | -| TraceScanner / realtime metrics（mc admin scanner status/trace） | 无 trace 通道；/v3/scanner/status 自有结构 | ⚠️ §6 HS-03 | - -### 4.5 admin/CLI/API 面对照 - -| MinIO | RustFS | 状态 | -|---|---|---| -| `POST /minio/admin/v3/heal/...` start/status/cancel | `POST /rustfs/admin/v3/heal/...` 同三态 | ✅（路径前缀不同属预期） | -| `HealStartSuccess`/`HealTaskStatus`/`HealResultItem`/DriveState | 同名字段 JSON 兼容 | ✅ | -| `POST /v3/background-heal/status`（BgHealState 聚合） | 同路径 + degraded 语义 + operations 矩阵 | ✅ 增强（MRF per-endpoint 子状态无，因无 MRF） | -| `GET /v3/healthinfo` 每 drive `HealInfo *HealingDisk` | 无同款 healthinfo heal 字段（replacement-recovery v4 承担部分） | ⚠️ §6 HS-07 | -| madmin 客户端 HealStart/HealStatus/BackgroundHealStatus/ScannerStatus 方法 | 仅 wire 类型，无客户端方法 | ❌ §6 HS-05 | -| mc admin heal --pool/--set、--scan-mode、--force-start/stop | HealOpts 全字段支持（pool/set/scanMode/forceStart/forceStop） | ✅（服务端就绪；缺 mc 侧入口，HS-05） | -| ErrHealAlreadyRunning / ErrHealOverlappingPaths 类型化错误 | 去重合并 + 驱逐语义；无类型化重叠拒绝 | ⚠️ §6 HS-06 | -| 结果 backpressure（maxUnconsumedItems=1000、10s 保活流式、24h 未消费 abort） | 快照式查询（1024 条 + 8MiB 截断 + 10min 保留） | ⚠️ §6 HS-06 | -| `mc support inspect`/healing-bin 离线 dump | 无（inspect.rs 存在但 healing dump 未确认） | ⚠️ P3 | - -### 4.6 观测面对照 - -| 维度 | MinIO | RustFS | 状态 | -|---|---|---|---| -| heal 指标 | minio_heal_objects_total/heal_total/errors_total/time_last_activity + v3 drive_health 2=healing | rustfs_heal_* 全套（admission/queue delay/running/throttle/page concurrency） | ✅（RustFS 缺 drive_health=healing 单一 gauge 等价物；DiskInfo.healing 已赋值） | -| scanner 指标 | v3 6 个 + realtime 18 项 | rustfs_scanner_* 全套 + per-source 维度 | ✅ | -| ILM 指标 | v3 5 个（expiry/transition pending/active/missed + versions_scanned） | ilm expiry status API + scanner per-source | ✅（指标与 API 形态不同） | -| trace | TraceHealing/TraceScanner 两通道 | 无 | ❌ §6 HS-03 | -| 审计 | HealObject 事件、dangling 删除审计、scanner:manyversions 等 | 结构化日志（event style）+ 指标；无 audit log 事件 | ⚠️ §6 HS-04 | -| 进度 | healingTracker Bytes/Items/QueuedBuckets/当前对象 + usage-cache 总量基线 | HealProgress{scanned/healed/failed/bytes/current_object/percentage}；bytes_processed 注释为 0、estimated_completion_time 恒 None | ⚠️ §6 HS-07 | - -### 4.7 配置面对照（默认值） - -| MinIO | RustFS | 备注 | -|---|---|---| -| `heal:bitrotscan`（默认 off；on=每轮；Nm=N×30×24h） | `heal.bitrot_cycle` / `RUSTFS_SCANNER_BITROT_CYCLE_SECS`（默认 30d=2592000s；0/on=每轮 Deep，off=禁用） | ✅ 同语义（RustFS 默认 30d，MinIO 默认 off——**默认值不同**，RustFS 更激进） | -| `heal:max_io=100`/`max_sleep=250ms`（waitForLowIO） | mainline throttle 阈值 80%/80%、max_sleep 250ms | ✅ 同型（阈值模型不同） | -| `heal:drive_workers`（默认 -1 自动） | 页内并发 8 + per-set 1 | ✅ 同型 | -| `_MINIO_HEAL_WORKERS`（GOMAXPROCS/2） | `RUSTFS_HEAL_MAX_CONCURRENT_HEALS=4` + `_MAX_CONCURRENT_PER_SET=1` | ✅ | -| `_MINIO_AUTO_DRIVE_HEALING`（on） | `RUSTFS_HEAL_AUTO_HEAL_ENABLE=true` | ✅ | -| `_MINIO_SCANNER`（on） | `RUSTFS_SCANNER_ENABLED=true` | ✅ | -| `scanner:speed` 五档（default=2x/1s/1m） | 同五档同名同参数 | ✅ | -| `scanner:idle_speed`（on） | `RUSTFS_SCANNER_IDLE_MODE`（true） | ⚠️ 语义方向（HS-14） | -| `scanner:alert_excess_versions=100` | 100 | ✅ | -| `scanner:alert_excess_folders=50000` | 65538（兼容 PBS 布局） | ⚠️ HS-17 | -| `ilm:expiration_workers=100`/`transition_workers=100` | ecstore expiry/transition worker 池（键见 ilm 子系统） | ✅（默认值未逐项核对） | -| `api:stale_upload_cleanup_interval=6h`/`expiry=24h` | ecstore 后台任务 env 可配 | ✅（默认值未逐项核对） | -| —（无） | `RUSTFS_HEAL_QUEUE_SIZE=10000`、`_TASK_TIMEOUT_SECS=300`、`_INTERVAL_SECS=10`、`_LOW_PRIORITY_MERGE/DROP`、`_PAGE_*`、`_SET_BULKHEAD`、`_MAINLINE_*`、`RUSTFS_SCANNER_CYCLE_MAX_*` 预算、`_MAX_CONCURRENT_SET/DISK_SCANS=4`、`_YIELD_EVERY_N_OBJECTS=128` 等 | RustFS 特有（更细粒度） | - -### 4.8 RustFS 超出 MinIO 的部分 - -1. remote_scanner RPC（扫描执行下放远端 peer 本地，含 HMAC 认证/重放缓存/fence 复验/断连宽限）。 -2. 持久化 leader-epoch CAS 围栏 + usage 快照 epoch/cycle 防回退（MinIO 仅锁，无持久 epoch）。 -3. 周期预算（max_duration/objects/directories）+ partial 周期推进语义。 -4. per-set/per-disk 扫描并发闸 + 每桶每 set 缓存锁。 -5. pending-heal 账本（heal 通道满不丢候选）。 -6. 换盘 durable intent + completion proof 状态机 + 身份围栏（MinIO healingTracker 无 proof）。 -7. mainline throttle 前台压力门控（permit 利用率驱动）。 -8. 集群 heal control coordinator + envelope 重放防护 + degraded 显式降级。 -9. 写路径 shard bitrot 自校验（EC:0 场景）。 -10. dirty-usage 快路径唤醒（写路径即时通知 + 脏桶优先）。 -11. heal 运行时可观测矩阵（优先级×来源 operations snapshot）。 -12. workload admission 联动（heal 调度器读前台压力快照）。 - ---- - -## 5. 差距与改进清单 - -分级定义：P1=行为/运维对齐缺口（影响生产运维或工具链兼容）；P2=完善性（功能在但缺一角）；P3=清理/低风险。每项含现状证据、MinIO 行为、影响、建议、验收方式。 - -### P1（8 项） - -**HS-01 MRF/ECDecode/Metadata 三类 heal 任务无生产触发入口，HealEvent 未接线** -- 现状：`HealType::MRF/ECDecode/Metadata` 执行体完整（task.rs:1700-2156）但全仓库无生产触发方；`HealEvent`/`HealEventHandler`（event.rs:50-367）crate 外零引用（已亲验 grep）；channel 转换只产生 Cluster/Object/Bucket/Prefix/ErasureSet（channel.rs:566-601）。 -- MinIO：mrf.go 独立 MRF 队列（容量 100k，满丢弃计数）、进程退出 msgp 持久化 `.heal/mrf/list.bin` + 启动回放、入队 <1s 延迟 1s（等网络恢复）、healSleeper 限速；读路径 GetObject part 缺失/损坏、元数据重建 missingBlocks>0、Put 部分成功、DeleteObject、multipart、peer client 共 7+ 投递点。 -- 影响：RustFS 的 read-repair + 写路径收敛覆盖了主场景，但缺少：① 事件驱动的 Urgent ECDecode 重建入口（ecstore 解码失败时目前仅 Low read-repair）；② Metadata-only heal 入口（scanner HealMetadata 分类存在但走普通对象 heal）；③ MRF 队列持久化（重启丢未消费修复意图——scanner pending-heal 账本部分缓解）。 -- 建议：三选一决策——(a) 接线 HealEvent（在 ecstore 解码失败/metadata 损坏点发事件）+ 实现持久化重试账本；(b) 删除 MRF/ECDecode/Metadata 死代码只保留文档说明；(c) 保留执行体、把 HealEvent 降级为内部 API。推荐 (a) 但需先量化 read-repair 是否已覆盖解码失败场景的响应时间要求。 -- 验收：解码失败 → Urgent heal 请求链路 e2e；重启后 pending 修复意图回放；HealEvent 环形缓冲指标。 - -**HS-02 CheckAbandonedParts 三层 NotImplemented（abandoned data 独立对账入口缺失）** -- 现状：`set_disk/ops/heal.rs:2052-2056`、`core/sets.rs:1144-1148`、`store/heal.rs:258-266` 三层显式 `Err(NotImplemented)`（已亲验），注释"intentionally retained above the set layer until there is a concrete caller"。 -- MinIO：`CheckAbandonedParts` → 每盘 `CleanAbandonedData`：读 xl.meta → 列 UUID data-dir + inline entries → 与 getDataDirs 差集 → 删多余 data-dir/inline 并重写 xl.meta；由 scanner 抽中 heal 与 admin heal Remove 时显式调用。 -- 影响：RustFS heal 路径内 `reclaim_orphan_data_dirs_best_effort`（:1428）覆盖"heal 时回收孤儿目录"，但 ① 无独立触发点（MinIO 在对象未到 heal 阈值时也能清 abandoned data）；② inline data 孤儿条目清理未确认；③ multipart 孤儿对账明确不做（设计决定，由 lifecycle 承担）。 -- 建议：评估把 `reclaim_orphan_data_dirs_best_effort` 提升为 heal_object 固定步骤（若尚非）+ 实现 HealOperations::check_abandoned_parts 真实现（调用同一回收逻辑），或明确文档化"由 lifecycle 承担"并关闭 API 面。 -- 验收：构造 data-dir/inline 孤儿 → scanner 抽样/admin heal 后被清理；三层 API 返回成功或显式 NotSupported 文档化。 - -**HS-03 heal/scanner trace 通道缺失** -- 现状：TraceHealing/TraceScanner 零命中（已亲验 grep 全仓库）。 -- MinIO：`madmin.TraceHealing`（mc admin trace --healing，FuncName=heal.Bucket/heal.Object/heal.CheckAbandonedParts，带 dry/remove/mode/version-id/disks/bytes）、`TraceScanner`（mc admin scanner trace，支持 --filter-size/--response-duration）。 -- 影响：无法实时观测单个 heal/scanner 动作的耗时与参数；排障只能靠指标聚合与日志。 -- 建议：在 heal channel 执行与 scanner folder/item 处理埋点，接入现有 admin trace 订阅面（若 rustfs 已有 trace 基建则复用，无则按 madmin TraceType 扩展）。 -- 验收：mc 等价工具能订阅 heal/scanner trace 流。 - -**HS-04 scanner 超限 S3 事件与审计缺失** -- 现状：仅 `rustfs_scanner_excess_*_total` 指标（versions 100/version size 1TiB/folders 65538）。 -- MinIO：发 `s3:ObjectManyVersions`（>100 版本）、`s3:ObjectLargeVersions`（累计 >1TB）、`s3:PrefixManyFolders`（>50000 子目录）事件（UserAgent: Scanner）+ scanner:manyversions/largeversions/manyprefixes 审计。 -- 影响：依赖事件订阅做容量治理的用户（console/外部审计）收不到告警。 -- 建议：scanner_folder 告警点接入 notify 事件发布（复用 lifecycle 事件通道语义）。 -- 验收：配置桶通知后超限对象触发事件。 - -**HS-05 madmin 客户端方法缺失** -- 现状：`crates/madmin/src/heal_commands.rs` 只有 wire 类型（HealDriveInfo/Infos/HealResultItem）；无 HealStart/HealStatus/BackgroundHealStatus/ScannerStatus 客户端方法。 -- MinIO：madmin-go 提供完整客户端；mc admin heal/scanner/status/trace 都建立在上面。 -- 影响：mc 等管理工具无法直接对接 RustFS heal/scanner 管理面；自动化运维只能手写 HTTP。 -- 建议：按 madmin-go 接口形状补客户端（服务端已就绪，纯客户端工作）。 -- 验收：用 madmin 客户端完成 start→query→cancel 全流程。 - -**HS-06 admin heal 序列语义与 MinIO 差异** -- 现状：重复/重叠请求被去重合并（返回 canonical task_id）或驱逐；无 ErrHealAlreadyRunning/ErrHealOverlappingPaths 类型化错误（已亲验：manager.rs:1309 的 already_running 是幂等启动保护，非 admin 语义）；结果为快照式查询（1024 条/8MiB 截断/10min 保留），非 MinIO 的流式增量（clientToken 拉增量 + maxUnconsumedItems=1000 backpressure + 10s 保活 + 24h 未消费 abort）。 -- 影响：mc admin heal 的交互模型（长连接拉增量）对 RustFS 表现为多次快照轮询；自动化脚本难以区分"已合并"与"新启动"。 -- 建议：① 增量语义：channel query 支持自上次 clientToken 起的 items 增量（或 cursor）；② 重叠请求返回类型化错误码（或 receipt 中显式 merged_into 字段——现有 alias 机制已有基础）；③ forceStart 先停旧再启新语义核对。 -- 验收：madmin 兼容客户端按 MinIO 模式轮询能取得全量 items。 - -**HS-07 healing 进度与盘级 healing 状态对外可见性不足** -- 现状：bytes 恢复进度 `progress.bytes_processed = 0 // set to 0 for now`（erasure_healer.rs:967）；`HealProgress::estimated_completion_time` 恒 None、`HealStatistics::add_healed_objects` 未写入（progress.rs:38,135-139 零调用）；healthinfo 无每盘 HealInfo 等价（MinIO HealingDisk：BytesDone/Failed/Skipped、ObjectsTotal 基线、QueuedBuckets/HealedBuckets、Resume 快照、当前 object）；v3 指标无 drive_health=2(healing) 单一 gauge 等价。 -- 影响：换盘重建（可能数小时~天）期间运维无法回答"进行到哪/还剩多少/预计何时完成"。 -- 建议：① erasure set heal 统计 bytes（heal_object 返回对象大小已可得）；② 从 usage-cache 读对象总量基线（MinIO 同款做法）；③ admin healthinfo/背景状态暴露每盘 healing 快照（DiskInfo.healing 已有，补聚合暴露）；④ ETA 由基线+速率推导。 -- 验收：换盘重建中 admin 可见 bytes 进度与 ETA；mc info 等价输出 Healing 标志。 - -**HS-08 prefix 级 usage 未暴露** -- 现状：DataUsageCache 内目录树 entry 存在（hash_path 组织），但 `dui()` 只 flatten 到桶名（data_usage_define.rs:858-915）。 -- MinIO：`loadPrefixUsageFromBackend`（30s cache）从每 set `.usage-cache.bin` 聚合 prefix usage，console 桶前缀统计消费。 -- 影响：console/前端无法展示前缀级用量；大桶定位"哪个前缀占空间"无 API。 -- 建议：实现 flatten 前缀查询 API（数据已在缓存内，纯聚合与暴露工作）。 -- 验收：ListBuckets/PrefixUsage API 返回与前缀过滤匹配的统计。 - -### P2（9 项） - -**HS-09 get_disk_status 恒返回 Ok（唯一 TODO）**：`crates/heal/src/heal/storage.rs:930-943`（已亲验）。当前无生产调用方（低风险）。建议：删除该方法或接 ecstore disk 状态真实现（DiskStatus 枚举已定义）。 - -**HS-10 HealStorageAPI 约 1/3 方法为死代码**：get_object_meta/get_object_data/put_object_data/delete_object/verify_object_integrity/ec_decode_rebuild/get_disk_status/format_disk/heal_bucket_metadata/get_object_size/get_object_checksum/list_objects_for_heal（非分页版，自带 memory_heavy 警告）均 0 调用方。建议：随 HS-01 决策一并清理或接线（死接口误导后续维护者以为存在调用路径）。 - -**HS-11 bitrot 自检缺失**：MinIO 启动时 bitrotSelfTest 对四算法已知向量自检失败即 Fatal（防静默数据损坏）。RustFS 无等价（已亲验 grep）。建议：启动时对 HighwayHash256S 等在用算法做已知向量自检（低成本高价值）。 - -**HS-12 对象级 healing 元数据标记评估**：MinIO heal 期间对象打 `x-minio-healing:true`，RenameData 据此跳过版本清理/legacy purge（漏掉会导致 heal 与并发删除互毁）。RustFS 无对象级标记（已亲验 grep object.rs 无 healing 分支），依赖 NSLock + rename 语义。建议：审计 RustFS rename 提交路径是否存在"heal 提交与并发 delete/version 清理竞争"窗口；若无则文档化差异，若有则补标记等价机制。 - -**HS-13 erasure set heal 无"跳过新写入/ILM 已过期版本"过滤**：MinIO resync 跳过 ModTime>tracker.Started 的版本（避免 heal 追新写入尾巴）与 ILM 已过期版本（避免白做）。RustFS erasure_healer 未实现同款过滤（按版本 dedup 有，时间/ILM 过滤无）。影响：重建尾部长尾（持续写入的桶 heal 完成判定被新版本推迟）与无效 heal 工作量。建议：disk-walk 枚举处加 started_at 时间过滤 + evaluator 预检。 - -**HS-14 scanner idle 语义方向与 MinIO 相反**：MinIO `scanner:idle_speed=on`（默认）= 集群空闲时才节流、忙时全速；RustFS `RUSTFS_SCANNER_IDLE_MODE=true`（默认）= 限速总闸（false=完全不休眠）。两者默认行为可能相近（都限速）但参数语义不可互换，迁移文档需显式说明；若追求 mc config 兼容需重命名/重语义。建议：先文档化差异，评估是否对齐语义。 - -**HS-15 alert_excess_folders 默认值差异**：RustFS 65538（兼容 PBS/Proxmox 布局，scanner_folder.rs:79）vs MinIO 50000。行为差异默认即触发阈值不同。建议：文档化（保留 65538 有本地理由）。 - -**HS-16 单机默认周期钩子未启用**：`single_disk_default_cycle_secs(_features) -> None` 恒空（scanner.rs:1428-1430），单机部署无专属默认周期覆盖。建议：决定单机默认周期策略后启用或删除钩子。 - -**HS-17 DeleteAllVersions 批量优化核对**：MinIO 用 DeletePrefix+DeletePrefixObject 单调用代替逐版本 fan-out。RustFS expiry 队列路径是否同款优化未逐行核实（集成测试覆盖行为正确性）。建议：核对 `apply_expiry_rule` 全版本删除路径，若无前缀单调用优化则评估补齐。 - -### P3（3 项） - -**HS-18 trash/临时目录二段清理细节核对**：MinIO `.minio.sys/tmp/.trash` 清理（delete_cleanup_interval 默认 5m + deleteCleanupSleeper）与 stale uploads rename-into-trash 二段式。RustFS 有 delete_tail_activity.rs 与 stale multipart 任务，二段语义是否完整对齐未逐行核实。建议：对照补齐或文档化。 - -**HS-19 root heal 直连死路径清理**：`should_handle_root_heal_directly` 恒 false（admin/handlers/heal.rs:1200-1202，测试锁定），store.heal_format 直连分支不可达。建议：删除死分支或恢复直连路径作为集群协调失败的降级。 - -**HS-20 兼容旗标与死指标清理**：`RUSTFS_SCANNER_INLINE_HEAL_ENABLE`（开启仅告警）+ `rustfs_scanner_inline_heal_total` 死指标 + `rustfs_common::metrics` 中 scanner 域代码分层迁移（backlog #1843 已登记）。建议：随分层迁移一并清理。 - -### 按设计不追平（7 项，记录以防后续误判为缺口） - -1. **bloom filter**：MinIO master 已删除；RustFS `.bloomcycle.bin` 复用为 cycle/epoch 围栏与 MinIO 现状一致。 -2. **scanner 集群单 leader**：双方一致；RustFS 额外有 epoch 围栏。 -3. **heal 不发 S3 bucket notification**：双方一致（heal 结果走 admin status）。 -4. **incomplete multipart 不在 scanner/ILM 内执行**：双方一致（独立后台例程）。 -5. **内联 heal 移除**：RustFS 有意为之（scanner 只入队），MinIO 的 applyHealing 内联路径不做对标。 -6. **heal 序列常驻保活（10s 空白回写）**：RustFS 快照式查询模型不同，按 HS-06 处理增量语义即可，不复制流式保活。 -7. **`.trash`/`tmp-old` 路径名兼容**：RustFS 布局常量独立，不逐字对齐 MinIO 路径。 - ---- - -## 6. 配置默认值总表（RustFS） - -heal（env 前缀 `RUSTFS_HEAL_`，`crates/config/src/constants/heal.rs`，消费于 `manager.rs:724-800`）： - -| 配置 | 默认 | 热更新 | -|---|---|---| -| AUTO_HEAL_ENABLE | true | 否 | -| QUEUE_SIZE | 10000 | 否 | -| INTERVAL_SECS | 10 | 否（启动时固定） | -| TASK_TIMEOUT_SECS | 300 | 否 | -| MAX_CONCURRENT_HEALS | 4 | 否 | -| MAX_CONCURRENT_PER_SET | 1（≤min(全局,值)） | 否 | -| LOW_PRIORITY_MERGE_ENABLE | true | 否 | -| LOW_PRIORITY_DROP_WHEN_FULL | true | 否 | -| PAGE_OBJECT_CONCURRENCY | 8（Deep/AutoHeal 强制 1） | 否 | -| EVENT_DRIVEN_SCHEDULER_ENABLE | true | 否 | -| SET_BULKHEAD_ENABLE | true | 否 | -| PAGE_PARALLEL_ENABLE | true | 否 | -| MAINLINE_THROTTLE_ENABLE | true | 否 | -| MAINLINE_READ/WRITE_UTILIZATION_HIGH_PERCENT | 80/80 | 否 | -| MAINLINE_MAX_SLEEP_MS | 250 | 否 | -| （总开关）RUSTFS_HEAL_ENABLED | true | 否 | -| admin 子系统 heal.bitrot_cycle | 30d | 是（经 scanner runtime config） | - -scanner（admin 子系统 `scanner`，`crates/config/src/constants/scanner.rs` + `ecstore/src/config/scanner.rs` + `runtime_config.rs:527-673`）： - -| 键 | env | 默认 | -|---|---|---| -| speed | RUSTFS_SCANNER_SPEED | default（2x/1s/60s） | -| delay / max_wait / cycle / start_delay | RUSTFS_SCANNER_* | 派生/空 | -| cycle_max_duration/objects/directories | …_MAX_* | 0（不限） | -| bitrot_cycle | …_BITROT_CYCLE_SECS | 2592000（30d；0/on=每轮，off=禁用） | -| idle_mode | …_IDLE_MODE | true | -| cache_save_timeout | …_CACHE_SAVE_TIMEOUT_SECS | 14s | -| max_concurrent_set_scans / disk_scans | …_MAX_CONCURRENT_* | 4/4 | -| yield_every_n_objects | …_YIELD_EVERY_N_OBJECTS | 128 | -| alert_excess_versions / version_size / folders | …_ALERT_* | 100 / 1TiB / 65538 | - -scanner 内部 env：`RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES=16`、`RUSTFS_HEAL_OBJECT_SELECT_PROB=1024`、`RUSTFS_SCANNER_DEEP_VERIFY_COOLDOWN_SECS=60`、`RUSTFS_DATA_USAGE_FAILED_OBJECT_TTL_SECS=86400`/`_MAX=10000`、`RUSTFS_LOCK_ACQUIRE_TIMEOUT=5s`、`RUSTFS_SCANNER_ENABLED=true`、`RUSTFS_SCANNER_INLINE_HEAL_ENABLE=false`（兼容告警）。 - -全部 17 个 scanner 键支持 env > config 双通道 + admin PUT 热更（generation+Notify 即时生效）；heal 运行时参数目前仅 env（无 admin 热更入口，`Arc>` 结构已预留）。 - ---- - -## 7. 相关 backlog / 历史索引 - -- 换盘自动修复系列（已闭环）：backlog #1786（冗余假绿算法）、#1787（目标槽位限定）、#1789（resume 与 healing marker 绑定 replacement 实例）、#1791（黑白盒验收矩阵）。 -- #801 DiskInfo.healing 从未赋值（已修复闭环，现 `set_disk/mod.rs:4988` 有赋值链）。 -- #1651 Scanner 指标节点/source/bucket-drive 维度（OPEN，本分析 §3.8/§4.6 相关）。 -- #1843 crates/common 83% scanner/heal 域代码分层迁移（OPEN，含 HS-20）。 -- 代码注释引用的历史缺陷（现已有防护与回归测试）：#856/#799 B7（离线盘误记 healed）、#855/B6/#1033（skip 不得标记完成）、#920（sub-quorum 并集枚举）、#856 B5（按版本续扫）、#5173（bitrot trailing bytes）、#5029（回归节点 stale 版本合并）。 -- v1 对标文档：`docs/rustfs-heal-scanner-vs-minio-parity-assessment.md`（本文取代）、落地手册 `docs/rustfs-heal-scanner-vs-minio-improvement-playbook.md`（部分条目已被后续实现超越）。 -- 换盘深度分析：`docs/new-disk-replacement-and-healing-deep-analysis-zh.md`、`docs/node-disk-identity-and-healing-analysis-zh.md`。 - -## 8. 审计方法与局限 - -- 四路并行审计（heal crate 逐文件、scanner crate 逐文件、ecstore 集成层 wiring、MinIO master 源码研究）+ 主会话对关键"缺失"结论逐条亲验（get_disk_status TODO、HealEvent 零外部引用、.bloomcycle.bin 无 bloom 实现、check_abandoned_parts 三层 NotImplemented、ETag 兜底已实现、trace 通道零命中、already_running 语义）。 -- 未逐行核实的点（已在文中标注"未确认/未逐行核"）：DeleteAllVersions 前缀单调用优化（HS-17）、trash 二段清理细节（HS-18）、ilm worker 默认值对照、stale multipart 默认值对照、mc CLI flag 逐字拼写（MinIO 侧）。其中 HS-17 与 HS-18 已于 2026-08-19 完成逐行核实，结论见 §9.2/§9.3。 -- MinIO 侧引用以其 master `7aac2a2c5b` 为准；RustFS 侧行号以 2026-08-16 工作区为准，后续演进请以符号名检索为准。 - -## 9. 落地结果（2026-08-19 更新） - -本审计衍生的 14 个子 issue（backlog #1865~#1878）已全部闭环。本节为差距清单 HS-01~HS-20 的最终处置记录，也是下一轮对标重审的增量基线。 - -### 9.1 已落地（PR 均已合并 main） - -- HS-01 MRF 接线 + 持久化修复账本（#1865，PR #6189）：决策选 (a)。common MRF channel（bounded 8192、try_send 永不阻塞）+ heal mrf_queue（100k 条 / 8MiB 双限环形）+ `buckets/.heal/mrf/journal.bin` CRC 持久化回放（torn tail 截断、回放后删除）+ 三投递点（read decode_error→Urgent ECDecode、scanner 元数据损坏→High Metadata、add_partial→Normal）+ `RUSTFS_HEAL_MRF_ENABLE` 一键回退。 -- HS-02 abandoned parts/data-dir 对账（#1866，PR #6179）：接通 abandoned 检查入口，保留 dry-run / reclaim 计数。 -- HS-03 heal/scanner trace 通道（#1867，PR #6179）：进程内 trace bus + `/v3/trace` admin 流式订阅 + heal task / abandoned-parts / scanner folder / ILM / heal-candidate trace producer。 -- HS-04 scanner 超限 S3 事件（#1868，PR #6176）：`s3:Scanner:ManyVersions/LargeVersions/BigPrefix` 三事件 + 24h 边沿冷却；HS-15 阈值差异文档化（`docs/operations/scanner-excess-alerts.md`）。 -- HS-05 madmin 客户端一期（#1869，PR #6166）：SigV4 admin 客户端 heal/scanner 方法；增量消费方法待 follow-up（协议已由 HS-06 并入）。 -- HS-06 admin heal 增量语义与类型化重叠（#1870，PR #6206）：`sinceSeq/nextSeq/minSeq` 增量游标（wire additive、缺省=全量快照）+ `RUSTFS_HEAL_OVERLAP_POLICY`（默认 merge 不变；minio_error 下 AlreadyRunning/OverlappingPaths 类型化拒绝）+ forceStart 先停旧再启新。 -- HS-07 healing 进度可见性（#1871，PR #6179）：data-usage 总量基线 + baseline/current/healed 计数。 -- HS-08 prefix usage（#1872，PR #6171）：`GET /v3/usage/{bucket}`。 -- HS-11 bitrot 启动自检（#1873，PR #6165）。 -- HS-13 heal 跳过过滤（#1875，PR #6179）：过滤命中版本不再计为失败。 -- HS-16 单机周期钩子（#1878，PR #6250）：删恒 None 钩子，决策记录见 `docs/operations/heal-scanner-parity-notes-zh.md`。 -- HS-09/10/19/20 死代码清理批（#1877，PR #6256）：净 −911 行零行为变更；`get_disk_status` TODO（全仓库唯一产品 TODO）清零；HS-01 联动的 `ec_decode_rebuild`/`get_object_meta` 保留并加 Reserved 注释（MRF 当前经 `heal_object` 执行）。 - -### 9.2 核对后确认"已实现 / 非缺口"（审计期误判修正，累计四例） - -- bloom filter（§0 已修正）：MinIO master 已删除，双方现状一致。 -- ETag 兜底仲裁（§0 已修正）：RustFS 已有实现（`set_disk/ops/heal.rs`）。 -- HS-17（#1876，2026-08-19 逐行核实后关闭）：DeleteAllVersions 前缀单调用优化 RustFS 已完整实现——`apply_expiry_on_non_transitioned_objects` 对 `delete_all()` 两 action 设 `delete_prefix + delete_prefix_object` 后单次 `delete_object`（`bucket_lifecycle_ops.rs:5047-5056`），SetDisks 分支一次写锁 + 一次全版本 quorum 读 + 内联逐版本 object-lock 检查（`set_disk/ops/object.rs:5566-5612`），与 MinIO `expire.go` 的 `applyExpiryOnNonTransitionedObjects` 逐行对齐。§8 原列"未逐行核实"的本项已有结论：现状即优化路径，无需实现。 -- HS-14（#1878，PR #6250 附带核对）：MinIO"idle=空闲才节流"是 2024-01 minio/minio#18734 之前的行为（`scannerIdleMode` 现为静态配置，`idle_speed=on` 默认即始终按速度档节流，"idle"命名是历史残留）；RustFS `RUSTFS_SCANNER_IDLE_MODE` 与 MinIO 当前语义方向一致，且另有 MinIO 没有的前台读退避下限。真实迁移陷阱（变量须 `RUSTFS_` 前缀、`on/off` vs `true/false` 词表、`false` 连前台保护一起关）已文档化于 `docs/operations/heal-scanner-parity-notes-zh.md`。 - -### 9.3 审计型结论（无需改代码） - -- HS-12（#1874，PR #6183）：不存在 MinIO 用 `x-minio-healing` 防御的那类竞争——所有同 (bucket, object) 提交面在同一把对象级 ns 写锁互斥，heal 锁 guard 覆盖 rename 提交全程；交付 2 个并发不变量回归测试 + `docs/operations/heal-concurrency-safety-notes-zh.md` 交点矩阵。 -- HS-18（#1878，2026-08-19 逐行核实）：trash/tmp 三段清理全对齐——stale multipart 隔离-清理等价且更安全（`delete_all_with_quorum` 逐盘递归删即 `move_to_trash` rename 进 `.rustfs.sys/tmp/.trash`，另有锁 + fence）、trash 排空基本等价（无逐条 sleeper 节流，5m 周期天然限频）、tmp 非 trash 24h 回收等价（RustFS 5m 比 MinIO 6h 更及时）；周期默认 24h/6h/5m 三项全对齐。§8 原列"未逐行核实"的本项已有结论。 - -### 9.4 移交 follow-up（汇总于 backlog#1862 评论区） - -HS-01 bitrot GET→MRF 全链路 e2e、kill -9 journal 回放 e2e、队列满压测 RSS（≤ 预算+10%）；HS-05/06 madmin 增量消费方法 + wire 单一来源化 + embedded e2e + 多轮轮询 soak；HS-08 多盘 scanner 周期 e2e；HS-04 超限审计条目；HS-18 低于 quorum 的 stale-multipart 崩溃残留窗口（扇出中途崩溃且已清盘数 > parity 时 FileNotFound 不在忽略集导致不自然收敛，修复需专用 quorum 变体）。 - -下一轮重审建议：跟随 heal/scanner 下一个大特性落地后触发，以本节为增量基线。 diff --git a/docs/operations/s3-tables-cutover-runbook.md b/docs/operations/s3-tables-cutover-runbook.md new file mode 100644 index 000000000..b817c68d5 --- /dev/null +++ b/docs/operations/s3-tables-cutover-runbook.md @@ -0,0 +1,60 @@ +# S3 Tables Durable Backing Cutover Runbook + +**Use this when:** moving a table-catalog warehouse from object-backed catalog state to the durable strong snapshot backing (`RUSTFS_TABLE_CATALOG_BACKING=durable-strong`), or rolling the strong snapshot format from version 1 to version 2. +**Source of truth:** the `{warehouse}/catalog/migration` routes registered in `rustfs/src/admin/handlers/table_catalog/routes.rs`; the env constants named below; claims and status labels in [docs/architecture/s3-tables-support-matrix.md](../architecture/s3-tables-support-matrix.md). + +## Preconditions + +| Requirement | Why | +|---|---| +| A principal with `GetTableCatalogAction` on each table bucket (preflight) and `admin:MigrateTableCatalog` (migration `POST` / `DELETE`). | The mutations are admin-gated. | +| Every catalog writer runs a release that recognizes the durable-backing migration fence. | An older writer does not see the persisted fence and can mutate the object-backed source after the snapshot inventory is captured. | +| An object-backed catalog backup, plus the current metadata pointer and version token for representative tables. | Recovery after a failed cutover is an operator-selected restore, not a restart against the stale pointer. | +| Every mutating object-only operation (maintenance workers, catalog recovery, export, diagnostics, external catalog bridge writes) is inventoried and confirmed supported in durable-strong mode. | Unsupported operations fail closed after cutover rather than continuing against object-backed state. | + +## Cutover Procedure + +1. Take the object-backed backup and record pointer and version token for representative tables. +2. Run the preflight for each warehouse and treat every `blockers` entry as fail-closed. Repair commit recovery state and backfill the warehouse prefix index before continuing. Requests are SigV4-signed with the catalog's REST signing name; the `/_iceberg/v1` alias accepts the same paths. + + ```text + GET /iceberg/v1/{warehouse}/catalog/migration + ``` +3. Drain every catalog writer that predates the migration fence and restart it on a fence-aware release. Keep all writers on that release until cutover completes. +4. Quiesce mutating object-only operations (step 4 of Preconditions). +5. Run the migration `POST` with `admin:MigrateTableCatalog`. It acquires the exclusive migration fence to drain in-flight fence-aware mutations, persists the source fence while exclusivity is held, then copies catalog state and reports `ready_to_enable_durable_strong`. + + ```text + POST /iceberg/v1/{warehouse}/catalog/migration + ``` + +6. Repeat preflight and materialization for every table bucket. Do not proceed until the preflight reports `SNAPSHOT_MATERIALIZED`, no blockers, and `ready_to_enable_durable_strong: true` for all of them. +7. Restart with `RUSTFS_TABLE_CATALOG_BACKING=durable-strong`, then verify catalog config, table and view loads, commit idempotency, and table data-plane policy resolution before admitting writers. +8. Preserve the object-backed backup until durable strong backing has passed the operator's retention window. + +## Cancelling Before Cutover + +Before the restart in step 7, `DELETE` on the migration endpoint removes a migration-created target bucket snapshot and releases the source fence. It releases the bucket fence only while the target state has not advanced, and releases the registry fence after the last bucket is cancelled. Retries and `DELETE` may restore a known-absent initial target after an ambiguous first write, but fail closed if a previously existing or materialized global snapshot disappears. + +```text +DELETE /iceberg/v1/{warehouse}/catalog/migration +``` + +After the durable-strong state advances, cancellation fails closed; recovery requires an operator-selected restore or reverse migration. + +## Strong Snapshot Version 1 to Version 2 + +1. Keep snapshot writes on version 1 during a rolling binary upgrade. Current binaries read both versions. +2. After every catalog writer can read version 2, set both `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2=true` and `RUSTFS_TABLE_CATALOG_STRONG_SNAPSHOT_V2_FLEET_CONFIRMED=true` and restart the catalog writers. Setting only one gate does not change the write format. +3. Perform a controlled catalog write or migration materialization and confirm the persisted snapshot is version 2 before serving table data-plane traffic. Once v2 is fleet-confirmed, data-plane resolution fails closed until the persisted snapshot is v2. +4. After any version 2 snapshot is persisted, do not roll writers back to a binary that only reads version 1. Current binaries preserve version 2 even when the gates are later disabled. + +## Rollback And Collision Repair Rules + +- A running process rejects restored version 1 content after observing version 2, but cannot distinguish an older snapshot with the same format version from a deliberate restore. The format high-water mark is process-local: restoring any older snapshot and restarting every writer is a privileged disaster-recovery rollback that must restore a compatible binary and an operator-selected snapshot together. +- Migration preflight rejects an active table/view identifier collision before writing a migration fence. A pre-existing version 1 strong snapshot with such a collision loads in cleanup-only quarantine: ambiguous reads fail closed, each cleanup mutation must reduce the collision set, and unrelated writes stay blocked until all collisions are removed. Drain writers that predate cleanup quarantine before starting the repair, and finish cleanup before the first version 2 write. + +## Related + +- [S3 Tables support matrix](../architecture/s3-tables-support-matrix.md) +- [Table catalog conformance scripts](../../scripts/table-catalog/README.md) (`failure_coverage.py --print-disaster-recovery-rehearsal` generates the rehearsal for this procedure) diff --git a/docs/operations/scanner-benchmark-runbook.md b/docs/operations/scanner-benchmark-runbook.md index 3bc9e85a9..83e9315a2 100644 --- a/docs/operations/scanner-benchmark-runbook.md +++ b/docs/operations/scanner-benchmark-runbook.md @@ -1,42 +1,18 @@ # Scanner Benchmark Runbook -This runbook describes how to collect reproducible evidence for scanner -pressure, scanner progress, and scanner runtime tuning. It is intended for -maintainers and operators validating scanner changes in an isolated test -deployment. +**Use this when:** you need reproducible before/after evidence that a scanner pacing or cycle change reduces background pressure without stalling lifecycle, replication, heal, or bitrot progress, or you are assembling evidence for a scanner-behavior PR. -Use this runbook together with -[Scanner Runtime Controls](scanner-runtime-controls.md). The runtime controls -document explains each field and configuration key; this document explains how -to run a comparable before/after observation and decide whether the scanner is -healthier after tuning. +**Source of truth:** `scripts/run_scanner_validation_harness.sh` (collection, `scanner-summary.csv` columns), `scripts/run_object_batch_bench.sh` (workload), and [Scanner Runtime Controls](scanner-runtime-controls.md) for the meaning of every status field and configuration key. ## Scope -This runbook verifies that scanner pacing and cycle controls reduce background -pressure while preserving maintenance progress. It is useful for: +This runbook verifies that scanner pacing and cycle controls reduce background pressure while preserving maintenance progress. It covers mostly idle single-node deployments with many small objects, multi-disk or erasure-set nodes, distributed clusters where scanner pressure mixes with lifecycle, replication, heal, or bitrot queues, and backlog investigations for any of those subsystems. -- mostly idle single-node, single-disk deployments with many small objects; -- single-node multi-disk or erasure-set deployments where scanner work is - spread across disks and sets; -- distributed clusters where scanner pressure is mixed with lifecycle, - replication, heal, or bitrot queues; -- lifecycle expiry or transition backlog investigations; -- bucket replication repair backlog investigations; -- scanner-originated heal or bitrot admission investigations; -- pull request evidence when scanner behavior, scanner status, or scanner - controls change. - -This runbook does not prove full MinIO parity, site-replication correctness, -or replaced-disk heal correctness. Those flows need dedicated distributed -tests because their failure modes are not limited to scanner pacing. +It does not prove full MinIO parity, site-replication correctness, or replaced-disk heal correctness. Those flows need dedicated distributed tests because their failure modes are not limited to scanner pacing. ## Safety -Run the workload only in a disposable test environment. The commands below can -create many buckets and objects and can overwrite runtime scanner settings. - -Record the current scanner and heal configuration before changing anything: +Run the workload only in a disposable test environment. The commands below can create many buckets and objects and overwrite runtime scanner settings. Record the current scanner and heal configuration before changing anything: ```bash mkdir -p artifacts @@ -44,75 +20,41 @@ mc admin config get ALIAS scanner > artifacts/scanner-config.before.txt mc admin config get ALIAS heal > artifacts/heal-config.before.txt ``` -Replace `ALIAS`, endpoint, and credentials with values for the test -deployment. Do not paste production credentials into saved artifacts. +The `scanner` and `heal` subsystems are served by `GetConfigKVHandler` (`rustfs/src/admin/handlers/config_admin.rs`, route `/v3/get-config-kv`); this was confirmed by code inspection, not by running `mc` against a live deployment. Replace `ALIAS`, endpoint, and credentials with values for the test deployment. Do not paste production credentials into saved artifacts. ## Required Tools -- `mc` or a compatible admin client for config changes. -- `awscurl` or another SigV4-capable HTTP client for `/v3/scanner/status`. -- `jq` for status extraction. -- `pidstat`, `mpstat`, `iostat`, `top`, or equivalent host telemetry. -- `warp`, `s3bench`, or the repository object benchmark scripts for workload - generation. +| Tool | Purpose | +|---|---| +| `mc` or a compatible admin client | Config snapshots and changes. | +| `awscurl` or another SigV4-capable HTTP client | `/v3/scanner/status` and admin metrics. | +| `jq` | Status extraction. | +| `pidstat`, `mpstat`, `iostat`, `top`, or equivalent | Host telemetry. | +| `warp`, `s3bench`, or `scripts/run_object_batch_bench.sh` | Workload generation. | ## Test Matrix -At minimum, collect two runs on the same RustFS commit and the same workload: +Collect at least two runs on the same RustFS commit and the same workload. Keep hardware, commit, object count, object size, bucket count, scanner-enabled state, and foreground workload constant between runs. | Run | Purpose | Example scanner settings | |---|---|---| | Baseline | Observe current behavior without additional pacing changes. | Existing config. | | Pacing override | Measure whether cooperative scanner sleeps reduce pressure. | `scanner.delay="30"` and `scanner.max_wait="15"`. | - -Add bounded-cycle runs when a single scan cycle is too long: - -| Run | Purpose | Example scanner settings | -|---|---|---| -| Duration budget | Bound wall-clock time per cycle. | `scanner.cycle_max_duration="1800"`. | +| Duration budget (when one cycle is too long) | Bound wall-clock time per cycle. | `scanner.cycle_max_duration="1800"`. | | Object budget | Bound objects processed per cycle. | `scanner.cycle_max_objects="1000000"`. | | Directory budget | Bound directories entered per cycle. | `scanner.cycle_max_directories="100000"`. | -When comparing runs, keep hardware, RustFS commit, object count, object size, -bucket count, scanner-enabled state, and foreground workload constant. - ## Deployment Matrix -Use the smallest deployment that reproduces the symptom, but do not treat -single-node validation as the whole scanner test surface. Scanner changes that -touch queues, admission, or cross-node maintenance should include a distributed -run when practical. +Use the smallest deployment that reproduces the symptom. The single-node, single-disk run is the cheap, repeatable baseline; it is not sufficient for PRs that claim to improve distributed queue behavior, replication repair, or heal/bitrot admission. -| Deployment | What it validates | Minimum evidence | -|---|---|---| -| Single-node, single-disk | Small-object scanner pressure, pacing, cycle interval, and basic progress. | Scanner status time series plus host CPU and disk telemetry. | -| Single-node, multi-disk or erasure set | Set and disk scan concurrency, cycle budgets, checkpoint movement, usage cache persistence, and active path age. | Scanner status time series, per-disk host telemetry, and before/after data usage freshness. | -| Distributed cluster | Lifecycle transition queues, bucket replication repair admission, scanner-originated heal and bitrot admission, and queue/backlog pressure under cross-node work. | Scanner status time series from the cluster, host telemetry from each node, and subsystem-specific queued/skipped/missed counters. | +| Deployment | What it validates | Minimum evidence | Workload shape | +|---|---|---|---| +| Single-node, single-disk | Small-object scanner pressure, pacing, cycle interval, basic progress. | Scanner status time series plus host CPU and disk telemetry. | One node, one data disk, several buckets, at least 100,000 small objects, scanner enabled, no sustained foreground workload during observation. | +| Single-node, multi-disk or erasure set | Set and disk scan concurrency, cycle budgets, checkpoint movement, usage cache persistence, active path age. | Scanner status time series, per-disk host telemetry, before/after data usage freshness. | Same as above across all disks. | +| Distributed cluster | Lifecycle transition queues, bucket replication repair admission, scanner-originated heal and bitrot admission, queue/backlog pressure under cross-node work. | Scanner status time series from the cluster, host telemetry from each node, subsystem-specific queued/skipped/missed counters. | Same structure plus the relevant subsystem condition (lifecycle rules, a replication target, a heal/bitrot scenario); keep status and telemetry cadence identical to the baseline. | -The single-node, single-disk run is the baseline because it is cheap and -repeatable. It is not sufficient for PRs that claim to improve distributed -queue behavior, replication repair, or heal/bitrot admission. - -## Workload Shape - -For the baseline small-object scanner-pressure run, use a workload that -creates many small objects and then leaves the service mostly idle while the -scanner walks the namespace. A useful minimum shape is: - -- one RustFS node; -- one data disk; -- several buckets; -- at least 100,000 small objects total; -- scanner enabled; -- no sustained foreground workload during the observation window. - -For a distributed backlog run, use the same structure but add the relevant -subsystem condition, such as lifecycle rules, a bucket replication target, or -a configured heal/bitrot scenario. Keep the status and host telemetry cadence -the same so the result remains comparable with the baseline run. - -The repository object benchmark script can generate object traffic if `warp` -or `s3bench` is installed: +Generate object traffic with the repository script if `warp` or `s3bench` is installed; repeat with new buckets or prefixes if one run cannot create enough objects, and record the final object count: ```bash scripts/run_object_batch_bench.sh \ @@ -129,15 +71,9 @@ scripts/run_object_batch_bench.sh \ --out-dir artifacts/object-load ``` -If the object generator cannot create enough objects in one run, repeat the -same command with new buckets or prefixes and record the final object count. - ## Status Collection -Capture scanner status before the workload, after the workload finishes, and -throughout the idle observation window. - -The repository includes a scanner validation harness for repeatable collection: +Capture scanner status before the workload, after the workload finishes, and throughout the idle observation window. The validation harness does this repeatably and writes scanner/heal config snapshots, scanner status samples, background heal status samples, host telemetry when available, run metadata, `scanner-summary.csv`, and `scanner-validation-report.md`: ```bash export RUSTFS_ACCESS_KEY="" @@ -153,12 +89,7 @@ scripts/run_scanner_validation_harness.sh \ --out-dir artifacts/scanner-validation ``` -The harness writes scanner/heal config snapshots, scanner status samples, -background heal status samples, host telemetry when available, run metadata, -`scanner-summary.csv`, and `scanner-validation-report.md`. - -Use `--metrics-endpoints` when the validation needs per-node distributed -evidence. The value is a comma-separated list of RustFS endpoints: +For per-node distributed evidence pass `--metrics-endpoints` (comma-separated). Each sample then stores `/v3/scanner/status`, one `/v3/background-heal/status` response per listed endpoint, and one by-host admin metrics response per listed endpoint; without it, background-heal status is captured only from `--endpoint`: ```bash scripts/run_scanner_validation_harness.sh \ @@ -172,55 +103,22 @@ scripts/run_scanner_validation_harness.sh \ --out-dir artifacts/scanner-validation-distributed ``` -Each sample stores `/v3/scanner/status`, one `/v3/background-heal/status` -response per endpoint listed in `--metrics-endpoints`, and one by-host admin -metrics response per listed endpoint. When `--metrics-endpoints` is omitted, -the harness captures background-heal status only from `--endpoint`. +For ad hoc per-node snapshots outside the harness window, use the by-host `awscurl` loop in [Reading Distributed Metrics](scanner-runtime-controls.md#reading-distributed-metrics); the metrics endpoint reports only the node that handles the request. -For bucket metrics freshness validation, use the same harness around a -post-start bucket creation workload: +### Bucket metrics freshness validation + +Use the harness around a post-start bucket creation workload to cover the timing where scanner startup sees no buckets, a bucket is created afterwards, and the first metrics collection must not confuse a cold usage cache with real zero usage: 1. Start RustFS from an empty data path. 2. Start the harness before creating buckets. -3. Create a bucket, upload objects, and keep the harness running until at - least one usage save is observed. -4. Compare `scanner-summary.csv` with - `/rustfs/admin/v3/metrics?types=1&n=1` bucket metrics. +3. Create a bucket, upload objects, and keep the harness running until at least one usage save is observed. +4. Compare `scanner-summary.csv` with `/rustfs/admin/v3/metrics?types=1&n=1` bucket metrics. -This covers the issue 3496 timing where scanner startup sees no buckets, the -bucket is created after startup, and the first metrics collection must not -confuse cold usage cache with real zero usage. The expected evidence is that -dirty usage is marked, `life_time_scan_cycle` or -`life_time_scan_bucket_drive` advances, `life_time_scan_object` advances for -object workloads, and `life_time_save_usage` plus -`usage_last_save_result=success` appear before accepting non-zero bucket usage -metrics as fresh. +Expected evidence: dirty usage is marked, `life_time_scan_cycle` or `life_time_scan_bucket_drive` advances, `life_time_scan_object` advances for object workloads, and `life_time_save_usage` plus `usage_last_save_result=success` appear before non-zero bucket usage metrics are accepted as fresh. -For distributed runs, capture scanner admin metrics from every node with -`by-host=true`. The metrics endpoint reports the node that handles the request; -`by-host=true` preserves that node's host view but does not collect peer nodes. -These per-node artifacts include active path age, checkpoint state, pacing -pressure, source work, and queued/skipped/missed downstream admission counters. -The validation harness can collect these artifacts automatically with -`--metrics-endpoints`; the manual loop below is useful when adding extra nodes -or collecting ad hoc snapshots outside the harness window. +### Manual status sampling -```bash -for endpoint in http://node-a:9000 http://node-b:9000 http://node-c:9000; do - node="${endpoint#http://}" - node="${node%%:*}" - awscurl \ - --service s3 \ - --region us-east-1 \ - --access_key "$RUSTFS_ACCESS_KEY" \ - --secret_key "$RUSTFS_SECRET_KEY" \ - --request GET \ - "${endpoint}/rustfs/admin/v3/metrics?types=1&by-host=true&n=1" \ - > "artifacts/scanner-metrics.${node}.$(date -u +%Y%m%dT%H%M%SZ).ndjson" -done -``` - -Example status request: +Single snapshot: ```bash awscurl \ @@ -233,7 +131,7 @@ awscurl \ | jq . > "artifacts/scanner-status.$(date -u +%Y%m%dT%H%M%SZ).json" ``` -For a time series, sample once per minute: +Time series (stop after the planned observation window): ```bash mkdir -p artifacts/status @@ -250,11 +148,9 @@ while sleep 60; do done ``` -Stop the loop after the planned observation window. - ## Host Telemetry -Collect host metrics over the same window as scanner status. +Collect host metrics over the same window as scanner status. If `pidstat` is unavailable, use `top`, `ps`, or the platform monitoring system, but record the sampling interval and window in the report. ```bash pidstat -p "$(pidof rustfs)" 60 > artifacts/pidstat.txt @@ -262,13 +158,9 @@ iostat -xz 60 > artifacts/iostat.txt mpstat 60 > artifacts/mpstat.txt ``` -If `pidstat` is not available, use `top`, `ps`, or the platform monitoring -system, but keep the sampling interval and observation window in the report. - ## Runtime Tuning Examples -Persistent scanner config values use seconds for time fields. Use numeric -strings instead of duration suffixes: +Persistent scanner config values use seconds for time fields; use numeric strings, not duration suffixes. The canonical persistent bitrot cadence belongs to the `heal` subsystem. ```bash mc admin config set ALIAS scanner delay="30" max_wait="15" @@ -276,16 +168,10 @@ mc admin config set ALIAS scanner cycle="3600" mc admin config set ALIAS scanner cycle_max_duration="1800" mc admin config set ALIAS scanner cycle_max_objects="1000000" mc admin config set ALIAS scanner cycle_max_directories="100000" -``` - -The canonical persistent bitrot cadence belongs to the `heal` subsystem: - -```bash mc admin config set ALIAS heal bitrot_cycle="2592000" ``` -Environment variables take precedence over persisted config and should be -recorded separately: +Environment variables take precedence over persisted config and should be recorded separately: ```bash RUSTFS_SCANNER_DELAY=30 @@ -297,8 +183,7 @@ RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES=100000 RUSTFS_SCANNER_BITROT_CYCLE_SECS=2592000 ``` -After each config change, read scanner status and confirm the effective value -and `source` under `runtime_config`. +After each config change, read scanner status and confirm the effective value and `source` under `runtime_config`. ## Observation Window @@ -308,85 +193,45 @@ Use the same window for each run: 2. Wait until foreground workload is idle. 3. Save scanner and heal config. 4. Save one scanner status snapshot. -5. Collect scanner status and host telemetry for at least 30 minutes, or for - one complete scanner cycle when that is practical. +5. Collect scanner status and host telemetry for at least 30 minutes, or for one complete scanner cycle when practical. 6. Save one final scanner status snapshot. -Longer windows are better for cycle interval comparisons. Short windows are -acceptable for quick pressure checks only if the conclusion avoids changing -defaults. +Longer windows are better for cycle interval comparisons. Short windows are acceptable for quick pressure checks only if the conclusion avoids changing defaults. ## Fields To Compare -Compare these fields between baseline and tuned runs: +Field semantics are defined in [Scanner Runtime Controls](scanner-runtime-controls.md); the decision fields for a before/after comparison are: -| Field | Why it matters | +| Field | Decision it supports | |---|---| -| `runtime_config.*.value` and `runtime_config.*.source` | Confirms the tested settings actually took effect. | -| `metrics.pacing_pressure.primary_pressure` | Shows whether pressure is from queues, budgets, pause activity, active scans, or no scanner pressure. | -| `metrics.pacing_pressure.last_cycle_total_pause_ratio` | Shows how much of the last cycle was cooperative scanner pause time. | -| `metrics.maintenance_control.primary_control` | Shows whether source-level maintenance is blocked, deferred, active, only pacing-limited, or idle. | -| `metrics.maintenance_control.sources` | Shows the source, state, reason, backlog, current or last-cycle missed work, and partial-cycle count for each scanner maintenance source. | -| `metrics.current_cycle_objects_scanned` | Confirms object scan progress during the current cycle. | -| `metrics.current_cycle_directories_scanned` | Confirms directory walk progress during the current cycle. | -| `metrics.last_cycle_result` | Confirms whether the previous cycle completed, stopped partially, or failed. | -| `metrics.last_cycle_partial_reason` | Shows which budget stopped a partial cycle. | -| `metrics.last_cycle_partial_source` | Shows which scanner work source consumed the stopping budget. | -| `metrics.source_work` | Shows cumulative work found, queued, skipped, missed, executed, and failed by source. | -| `metrics.current_cycle_source_work` | Shows which source is consuming the current scan cycle. | -| `metrics.last_cycle_source_work` | Shows which source consumed the previous scan cycle. | -| `metrics.replication_repair` | Splits scanner-discovered replication repair by source, kind, scanner role, and execution owner, including bucket object, delete-marker, version-purge, existing-object repair, and site replication boundary states. | -| `metrics.current_cycle_replication_repair` | Shows which replication repair kind is being discovered or admitted in the current cycle. | -| `metrics.last_cycle_replication_repair` | Shows which replication repair kind consumed the previous cycle. | -| `metrics.lifecycle_expiry.current_queued` | Shows scanner-driven expiry/delete work waiting in the expiry worker queue. | -| `metrics.lifecycle_expiry.current_active` | Shows scanner-driven expiry/delete work currently running in expiry workers. | -| `metrics.lifecycle_expiry.queue_missed` | Shows expiry/delete queue admission failures outside the scanner walk itself. | -| `metrics.lifecycle_expiry.scanner_missed` | Shows scanner-discovered expiry/delete work that could not be queued. | -| `metrics.lifecycle_transition.scanner_missed` | Shows scanner-discovered transition work that could not be queued. | -| `metrics.lifecycle_transition.queue_full` | Shows transition queue pressure outside the scanner walk itself. | -| `metrics.lifecycle_transition.compensation_pending` | Shows transition compensation still pending or running after queue pressure. | -| `metrics.lifecycle_transition.failed` | Shows transition worker failures, which should also surface as lifecycle source failure. | -| `metrics.current_cycle_usage_saves` | Shows usage cache saves produced by the active scan cycle. | -| `metrics.last_cycle_usage_saves` | Shows usage cache saves produced by the previous completed or partial scan cycle. | -| `metrics.usage_freshness.dirty_pending_buckets` | Shows whether bucket/object mutations are still waiting for usage refresh. | -| `metrics.usage_freshness.last_cycle_dirty_buckets` | Shows how many dirty buckets were picked up by the last cycle. | -| `metrics.usage_freshness.last_cycle_cleared_dirty_buckets` | Shows how many dirty bucket marks were cleared by a successful cycle. | -| `metrics.usage_freshness.last_usage_save_result` | Confirms whether the last usage save succeeded, failed, or was skipped. | -| `metrics.life_time_ops.scan_cycle` | Confirms scanner cycles actually started after the workload. | -| `metrics.life_time_ops.scan_bucket_drive` | Confirms bucket-drive scan work reached the storage layer. | -| `metrics.life_time_ops.scan_object` | Confirms object metadata scanning advanced for object workloads. | -| `metrics.life_time_ops.save_usage` | Confirms `DataUsageInfo` save work happened; this is the key freshness signal for bucket metrics. | -| `metrics.scan_checkpoint` | Confirms partial cycles preserve resume context. | -| `metrics.oldest_active_path_age_seconds` | Helps identify scanner paths that may be stuck. | +| `runtime_config.*.value` and `runtime_config.*.source` | The tested settings actually took effect. | +| `metrics.pacing_pressure.primary_pressure`, `last_cycle_total_pause_ratio` | Where pressure comes from and how much of the cycle was cooperative pause. | +| `metrics.maintenance_control.primary_control`, `metrics.maintenance_control.sources` | Whether a maintenance source is blocked, deferred, active, or only pacing-limited. | +| `metrics.current_cycle_objects_scanned`, `metrics.current_cycle_directories_scanned` | Scan progress continues. | +| `metrics.last_cycle_result`, `last_cycle_partial_reason`, `last_cycle_partial_source` | Whether the previous cycle completed, which budget stopped it, and which source consumed it. | +| `metrics.source_work`, `metrics.current_cycle_source_work`, `metrics.last_cycle_source_work` | `missed` growth per source is a downstream admission problem, not pacing. | +| `metrics.replication_repair` (and current/last-cycle variants) | Repair kind, `scanner_role`, and `execution_owner` for replication backlog runs. | +| `metrics.lifecycle_expiry.{current_queued,current_active,queue_missed,scanner_missed}` | Expiry backlog and admission failures. | +| `metrics.lifecycle_transition.{scanner_missed,queue_full,compensation_pending,failed}` | Transition backlog, queue pressure, and worker failures. | +| `metrics.usage_freshness.*`, `metrics.current_cycle_usage_saves`, `metrics.last_cycle_usage_saves` | Bucket metrics freshness; `last_usage_save_result` must be `success`. | +| `metrics.life_time_ops.{scan_cycle,scan_bucket_drive,scan_object,save_usage}` | Cycles, bucket-drive scans, object scans, and `DataUsageInfo` saves actually happened after the workload. | +| `metrics.scan_checkpoint`, `metrics.oldest_active_path_age_seconds` | Partial cycles preserve resume context; stuck paths. | -Do not use a single CPU spike as the conclusion. Compare average and p95 CPU -over the same observation window. +Do not use a single CPU spike as the conclusion; compare average and p95 CPU over the same observation window. -For heal or bitrot pressure investigations, also capture -`/v3/background-heal/status` from every distributed endpoint and compare -`healOperations.queueLength`, -`healOperations.activeTasks`, `healOperations.queuedBySource`, -`healOperations.activeBySource`, `healOperations.queuedByPriority`, and -`healOperations.activeByPriority`. These fields distinguish scanner-submitted -low-priority work from manual admin heal and auto-heal work. +For heal or bitrot pressure investigations, also capture `/v3/background-heal/status` from every distributed endpoint and compare `healOperations.queueLength`, `activeTasks`, `queuedBySource`, `activeBySource`, `queuedByPriority`, and `activeByPriority` (see [Reading Heal Operations](scanner-runtime-controls.md#reading-heal-operations)). -`scanner-summary.csv` includes the heal operation totals needed for quick -before/after comparison. In distributed runs, these fields are aggregated from -the background-heal status snapshots captured across `--metrics-endpoints`. +### `scanner-summary.csv` columns -| Field | Why it matters | +In distributed runs the heal columns are aggregated from the background-heal snapshots captured across `--metrics-endpoints`. + +| Column | Meaning | |---|---| | `heal_queue_length` | Total queued heal requests at the same timestamp as the scanner status sample. | | `heal_active_tasks` | Total running heal tasks. | | `heal_scanner_queued` | Scanner-submitted heal or bitrot work waiting in the queue. | | `heal_admin_queued` | Manual/admin heal work waiting in the queue. | | `heal_auto_heal_queued` | Auto-heal work waiting in the queue, typically from disk/set recovery paths. | - -`scanner-summary.csv` also includes usage freshness columns for quick -post-start bucket metrics validation: - -| Field | Why it matters | -|---|---| | `current_cycle_usage_saves` | Usage saves during the current cycle. | | `last_cycle_usage_saves` | Usage saves from the last finished or partial cycle. | | `usage_dirty_pending_buckets` | Dirty buckets still waiting for scanner refresh. | @@ -404,74 +249,36 @@ post-start bucket metrics validation: A useful tuning result has all of these properties: - average or p95 scanner-related CPU and disk pressure decreases; -- `current_cycle_objects_scanned` or `current_cycle_directories_scanned` - continues to advance; -- `source_work.missed` does not grow unexpectedly for lifecycle, replication, - heal, or bitrot; -- `last_cycle_result` is either `success` or a partial result with a clear - budget reason and checkpoint; +- `current_cycle_objects_scanned` or `current_cycle_directories_scanned` continues to advance; +- `source_work.missed` does not grow unexpectedly for lifecycle, replication, heal, or bitrot; +- `last_cycle_result` is either `success` or a partial result with a clear budget reason and checkpoint; - data usage freshness remains acceptable for the tested deployment. Treat these as failure signals: -- CPU drops only because the scanner stops making progress; -- `primary_pressure` stays at `queued_scans` while queues grow; -- `last_cycle_partial_reason` repeats forever with no checkpoint movement; -- lifecycle expiry `queue_missed`, `scanner_missed`, `current_queued`, or - `current_active` grows during a run that was expected to reduce expiry - backlog; -- lifecycle transition `scanner_missed`, `queue_full`, - `compensation_pending`, or `failed` grows during a run that was expected to - reduce backlog; -- bucket metrics show zero usage after post-start uploads while dirty usage - remains pending and `life_time_save_usage` does not advance; -- `bucket_replication` missed work with `scanner_role=repair_admission` grows - while replication worker queues or target failures are also growing; treat - this as downstream replication pressure, not only scanner pacing pressure; -- `site_replication` `active_resync` grows and is interpreted as scanner-owned - repair execution; `scanner_role=boundary_signal` and - `execution_owner=site_replication_runtime` mean active site resync remains - owned by the site replication runtime and admin resync path; -- heal or bitrot work moves from `queued` to `missed` after a scanner pacing - change. +| Signal | Reading | +|---|---| +| CPU drops only because the scanner stops making progress | Not a tuning win. | +| `primary_pressure` stays at `queued_scans` while queues grow | Concurrency, not pacing, is the constraint. | +| `last_cycle_partial_reason` repeats forever with no checkpoint movement | Budget too small or checkpoint not advancing. | +| Lifecycle expiry `queue_missed`, `scanner_missed`, `current_queued`, or `current_active` grows during a run meant to reduce expiry backlog | Downstream expiry pressure. | +| Lifecycle transition `scanner_missed`, `queue_full`, `compensation_pending`, or `failed` grows during a run meant to reduce backlog | Downstream transition pressure. | +| Bucket metrics show zero usage after post-start uploads while dirty usage remains pending and `life_time_save_usage` does not advance | Usage freshness regression. | +| `bucket_replication` missed work with `scanner_role=repair_admission` grows while replication worker queues or target failures also grow | Downstream replication pressure, not only scanner pacing. | +| `site_replication` `active_resync` grows and is read as scanner-owned repair execution | Misreading: `scanner_role=boundary_signal` and `execution_owner=site_replication_runtime` mean active resync remains owned by the site replication runtime. | +| Heal or bitrot work moves from `queued` to `missed` after a scanner pacing change | Heal admission regression. | ## PR Evidence Checklist -For scanner behavior PRs, include this evidence when available: +For scanner behavior PRs, include when available: - RustFS commit SHA and branch. -- Deployment shape: node count, disk count, disk type, CPU count, memory, and - object count. +- Deployment shape: node count, disk count, disk type, CPU count, memory, object count. - Workload command or script and benchmark artifact path. - Scanner and heal config before and after tuning. - Observation window and sample interval. - Scanner status snapshots or time series. - Host CPU and disk telemetry. -- Usage freshness fields from `scanner-summary.csv` when validating bucket - metrics or issue 3496-style timing. -- Short conclusion that separates pressure reduction from scanner progress. -- `scanner-validation-report.md` from the harness when using the scripted - collection path. - -## Final Parity Validation Closure - -Use the final validation run to prove the scanner control plane is coherent, -not to introduce new runtime behavior. A complete closure package should have -at least these runs: - -| Run | Required evidence | -|---|---| -| Single-node, single-disk small-object idle | Scanner status series, host telemetry, `scanner-summary.csv`, and a conclusion that CPU or disk pressure is lower without scan progress stopping. | -| Single-node post-start bucket metrics freshness | Empty data path startup, post-start bucket creation/upload, bucket metrics snapshots, `scanner-summary.csv` usage freshness columns, and evidence that `DataUsageInfo` save work occurred before accepting bucket usage metrics. | -| Single-node erasure or multi-disk | Checkpoint movement, active path age, set/disk scan pressure, data usage freshness, and before/after scanner config. | -| Distributed lifecycle backlog | `maintenance_control`, lifecycle expiry/transition queue fields, source work missed/failed counts, and by-host admin metrics. | -| Distributed replication backlog | Bucket replication repair kind counters, `scanner_role`, `execution_owner`, site replication passive/active boundary counters, source work queued/skipped/missed counts, and by-host admin metrics. | -| Heal or bitrot pressure | Background heal `healOperations` queued/active source and priority counts, scanner source work for heal/bitrot, and by-host admin metrics. | - -The expected conclusion is MinIO-style scanner behavior at the operational -contract level: scanner remains enabled, pacing is observable and adjustable, -partial progress is explainable, maintenance work is attributed by source, and -downstream lifecycle, replication, heal, and bitrot backlog can be diagnosed -without guessing from CPU usage alone. - -For documentation-only PRs, it is enough to verify links and formatting. +- Usage freshness fields from `scanner-summary.csv` when validating bucket metrics timing. +- A short conclusion that separates pressure reduction from scanner progress. +- `scanner-validation-report.md` from the harness when using the scripted collection path. diff --git a/docs/operations/scanner-excess-alerts.md b/docs/operations/scanner-excess-alerts.md index 8079daba6..650816c64 100644 --- a/docs/operations/scanner-excess-alerts.md +++ b/docs/operations/scanner-excess-alerts.md @@ -1,37 +1,39 @@ # Scanner Excess Alerts: Metrics, S3 Events, and Thresholds -> 中文版：[scanner-excess-alerts_zh.md](scanner-excess-alerts_zh.md) +**Use this when:** debugging an excess-versions / excess-version-size / excess-folders alert, wiring a notification subscriber for `s3:Scanner:*` events, or explaining why a threshold differs from MinIO. -Date: 2026-08-18 (rustfs/backlog#1868 / HS-04; includes the HS-15 threshold-delta notes) +**Source of truth:** `crates/scanner/src/scanner_folder.rs` (`EVENT_SCANNER_*`, `DEFAULT_SCANNER_ALERT_COOLDOWN_SECS`, `MAX_SCANNER_ALERT_COOLDOWN_KEYS`, `METRIC_SCANNER_EXCESS_*`), `crates/config/src/constants/scanner.rs` (`DEFAULT_SCANNER_ALERT_EXCESS_*`), `crates/s3-types/src/event_name.rs` (`EventName::Scanner*`). -The background scanner detects three classes of "excess" conditions while it walks buckets and surfaces them as alerts. This page documents each alert's trigger condition, the subscribable S3 event, the cooldown semantics, and the threshold differences versus MinIO — for operators debugging alerts and for event consumers wiring up subscriptions. +The background scanner detects three "excess" conditions while it walks buckets and surfaces them as metrics, structured logs, and S3 events. Threshold keys are also listed in the runtime-controls table in [Scanner Runtime Controls](scanner-runtime-controls.md). ## The three alerts | Alert | Trigger (per scan cycle) | Metric | S3 event (RustFS wire name) | MinIO event name | |---|---|---|---|---| -| Excess versions | Retained versions of one object ≥ `scanner:alert_excess_versions` | `rustfs_scanner_excess_object_versions_total{bucket}` | `s3:Scanner:ManyVersions` | `s3:ObjectManyVersions` | -| Excess version size | Cumulative bytes of all versions of one object ≥ `scanner:alert_excess_version_size` | `rustfs_scanner_excess_object_version_size_total{bucket}` | `s3:Scanner:LargeVersions` | `s3:ObjectLargeVersions` | -| Excess folders | Direct subfolders of one directory > `scanner:alert_excess_folders` | `rustfs_scanner_excess_folders_total{root}` | `s3:Scanner:BigPrefix` | `s3:PrefixManyFolders` | +| Excess versions | Retained versions of one object >= `scanner.alert_excess_versions` | `rustfs_scanner_excess_object_versions_total{bucket}` | `s3:Scanner:ManyVersions` | `s3:ObjectManyVersions` | +| Excess version size | Cumulative bytes of all versions of one object >= `scanner.alert_excess_version_size` | `rustfs_scanner_excess_object_version_size_total{bucket}` | `s3:Scanner:LargeVersions` | `s3:ObjectLargeVersions` | +| Excess folders | Direct subfolders of one directory > `scanner.alert_excess_folders` | `rustfs_scanner_excess_folders_total{root}` | `s3:Scanner:BigPrefix` | `s3:PrefixManyFolders` | -Subscribe like any bucket notification: configure a notification on the target bucket with the RustFS wire name above (or the `s3:Scanner:*` wildcard). Events carry `UserAgent: Scanner` as their origin marker, and `req_params` holds the observed value and the threshold (`versions` / `cumulativeSize` / `folders` / `threshold`), so consumers can judge severity directly. +Subscribe like any bucket notification: configure a notification on the target bucket with the RustFS wire name above (or the `s3:Scanner:*` wildcard, `EventName::ObjectScannerAll`). Events carry `UserAgent: Scanner` as their origin marker, and `req_params` holds the observed value and the threshold (`versions` / `cumulativeSize` / `folders` / `threshold`), so consumers can judge severity directly. ## Metrics and events fire on different cadences -- **Metrics and structured logs are level-triggered**: as long as the object stays over the threshold, every scan cycle counts and logs it (default cycle ≈ 60s; see `scanner:speed`). -- **S3 events are edge-triggered with a cooldown**: the same (alert kind, bucket, object) emits at most once per cooldown window — 24 hours by default (`RUSTFS_SCANNER_ALERT_COOLDOWN_SECS`; set it to 0 to emit every cycle). When the window lapses and the object is still over the threshold, the event fires again. The cooldown table lives in process memory with a 4096-entry hard cap; on overflow it is cleared and rebuilt (worst case: one extra emission per still-hot key). -- A process restart resets the cooldown (every still-over-threshold object emits once more after a restart) — deliberately: restarts usually accompany incident response, and the re-emission buys visibility. +| Surface | Cadence | +|---|---| +| Metrics and structured logs | Level-triggered: as long as the object stays over the threshold, every scan cycle counts and logs it (default cycle about 60s; see `scanner.speed`). | +| S3 events | Edge-triggered with a cooldown: the same (alert kind, bucket, object) emits at most once per cooldown window, `RUSTFS_SCANNER_ALERT_COOLDOWN_SECS` (`DEFAULT_SCANNER_ALERT_COOLDOWN_SECS`, 86400; `0` emits every cycle). When the window lapses and the object is still over the threshold, the event fires again. | +| Cooldown table | Process memory, hard cap `MAX_SCANNER_ALERT_COOLDOWN_KEYS` (4096) distinct keys; on overflow it is cleared and rebuilt (worst case one extra emission per still-hot key). A process restart resets it, so every still-over-threshold object emits once more after a restart. | -## Threshold defaults and the MinIO deltas (HS-15) +## Threshold defaults and MinIO deltas | Config key | ENV | RustFS default | MinIO default | Notes | |---|---|---|---|---| -| `scanner:alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | 100 | 100 | Identical | -| `scanner:alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | 1 TiB | 1 TB | Same order of magnitude; different unit basis (TiB vs TB) | -| `scanner:alert_excess_folders` | `RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS` | 65538 | 50000 | **Deliberate divergence**: 65538 tolerates the Proxmox Backup Server chunk layout (65536 chunks per directory plus the directory's own entries); MinIO's 50000 would fire continuously for PBS users. Set it to 50000 explicitly to match MinIO behavior | +| `scanner.alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | 100 (`DEFAULT_SCANNER_ALERT_EXCESS_VERSIONS`) | 100 | Identical. | +| `scanner.alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | 1 TiB, 1099511627776 (`DEFAULT_SCANNER_ALERT_EXCESS_VERSION_SIZE`) | 1 TB | Same order of magnitude; different unit basis (TiB vs TB). | +| `scanner.alert_excess_folders` | `RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS` | 65538 (`DEFAULT_SCANNER_ALERT_EXCESS_FOLDERS`) | 50000 | Deliberate divergence: 65538 tolerates the Proxmox Backup Server chunk layout (65536 chunks per directory plus the directory's own entries); MinIO's 50000 would fire continuously for PBS users. Set 50000 explicitly to match MinIO. | -All three keys accept both env and admin config (`PUT /rustfs/admin/v3/config`, `scanner` subsystem); hot updates take effect immediately. +All three keys accept both the environment variable and the `scanner` admin config subsystem (`SCANNER_SUB_SYS`, applied through `apply_scanner_runtime_config`); config updates take effect without a restart. ## Why the event names are mapped -RustFS's event enum (`rustfs_s3_types::EventName::ScannerManyVersions/LargeVersions/BigPrefix`) keeps the repo's established `s3:Scanner:*` wire names (literally different from MinIO's `s3:ObjectManyVersions`; the enum comments preserve the mapping). Subscribers should use the RustFS wire names in this page. If you need MinIO-literal compatibility, map the names on the console/consumer side — do not change the published wire names. +`EventName::ScannerManyVersions` / `ScannerLargeVersions` / `ScannerBigPrefix` keep the repository's established `s3:Scanner:*` wire names, which differ literally from MinIO's `s3:ObjectManyVersions` family; the enum comments preserve the mapping. Subscribers should use the RustFS wire names. If MinIO-literal compatibility is needed, map the names on the console or consumer side rather than changing the published wire names. diff --git a/docs/operations/scanner-excess-alerts_zh.md b/docs/operations/scanner-excess-alerts_zh.md deleted file mode 100644 index d4f4995ac..000000000 --- a/docs/operations/scanner-excess-alerts_zh.md +++ /dev/null @@ -1,37 +0,0 @@ -# Scanner 超限告警：指标、S3 事件与阈值 - -> English version: [scanner-excess-alerts.md](scanner-excess-alerts.md) - -日期：2026-08-18（rustfs/backlog#1868 / HS-04，含 HS-15 阈值差异说明） - -后台 scanner 在扫描过程中检测三类"超限"状态并对外告警。本文说明每类告警的触发条件、可订阅的 S3 事件、冷却语义，以及与 MinIO 的阈值差异，供运维排障与事件消费方对接。 - -## 三类告警 - -| 告警 | 触发条件（任一扫描周期） | 指标 | S3 事件（RustFS wire 名） | MinIO 对应事件名 | -|---|---|---|---|---| -| 版本数超限 | 单对象保留版本数 ≥ `scanner:alert_excess_versions` | `rustfs_scanner_excess_object_versions_total{bucket}` | `s3:Scanner:ManyVersions` | `s3:ObjectManyVersions` | -| 版本总大小超限 | 单对象全部版本累计字节 ≥ `scanner:alert_excess_version_size` | `rustfs_scanner_excess_object_version_size_total{bucket}` | `s3:Scanner:LargeVersions` | `s3:ObjectLargeVersions` | -| 子目录数超限 | 单目录直接子目录数 > `scanner:alert_excess_folders` | `rustfs_scanner_excess_folders_total{root}` | `s3:Scanner:BigPrefix` | `s3:PrefixManyFolders` | - -订阅方式与普通桶通知一致：对目标桶配置 notification，事件名填上表 RustFS wire 名（或通配 `s3:Scanner:*`）。事件以 `UserAgent: Scanner` 标记来源，`req_params` 携带实际值与阈值（`versions` / `cumulativeSize` / `folders` / `threshold`），便于消费方直接判断严重程度。 - -## 指标与事件的触发节奏不同 - -- **指标与结构化日志是电平触发**：只要对象仍在阈值之上，每个扫描周期都会计数/打日志（默认周期约 60s，见 `scanner:speed`）。 -- **S3 事件是边沿触发 + 冷却**：同一 (告警类型， 桶, 对象) 在冷却窗口内只发一次，默认 24 小时（`RUSTFS_SCANNER_ALERT_COOLDOWN_SECS`，设 0 表示每周期都发）。窗口过后对象仍超限会再次发出。冷却表在进程内有 4096 条硬顶，超限清空重建（最坏情况是每个仍超限的 key 多发一次）。 -- 进程重启会重置冷却（重启后每个仍超限的对象会再发一次）——这是有意为之：重启常伴随排障，重发提供可见性。 - -## 阈值默认值与 MinIO 差异（HS-15） - -| 配置键 | ENV | RustFS 默认 | MinIO 默认 | 差异说明 | -|---|---|---|---|---| -| `scanner:alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | 100 | 100 | 一致 | -| `scanner:alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | 1 TiB | 1 TB | 语义同量级，单位口径不同（TiB vs TB） | -| `scanner:alert_excess_folders` | `RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS` | 65538 | 50000 | **有意差异**：65538 兼容 Proxmox Backup Server 的 chunk 布局（每目录 65536 个 chunk + 目录自身条目），按 MinIO 的 50000 会对 PBS 用户持续误报。如需与 MinIO 行为一致可显式配置为 50000 | - -三个键均支持 env 与 admin config（`PUT /rustfs/admin/v3/config` 的 `scanner` 子系统）双通道，热更新即时生效。 - -## 事件名映射的由来 - -RustFS 的事件枚举（`rustfs_s3_types::EventName::ScannerManyVersions/LargeVersions/BigPrefix`）沿用仓库既有 wire 名 `s3:Scanner:*`（与 MinIO 的 `s3:ObjectManyVersions` 字面不同，枚举注释中保留了映射关系）。订阅方应以本文的 RustFS wire 名为准；如需 MinIO 字面兼容，请在 console/消费侧做名称映射，不要修改已发布的 wire 名。 diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index aade3999b..31fe6124c 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -1,150 +1,114 @@ # Scanner Runtime Controls -This document describes the runtime controls and status fields for the RustFS -data scanner. It is written for operators who need to reduce scanner pressure, -diagnose slow scan progress, or confirm that background lifecycle, replication, -heal, bitrot, and usage work is still moving. +**Use this when:** tuning scanner pacing or heal runtime knobs, reading `/v3/scanner/status`, deciding whether slow lifecycle/replication/heal progress is a scanner problem or a downstream queue problem, or migrating scanner settings from MinIO. -For reproducible scanner-pressure validation and before/after evidence, see -[Scanner Benchmark Runbook](scanner-benchmark-runbook.md). +**Source of truth:** `crates/config/src/constants/scanner.rs` and `crates/config/src/constants/heal.rs` (keys, env names, `DEFAULT_*` constants), `crates/scanner/src/runtime_config.rs` (resolution order, parsing, hot update), `crates/scanner/src/scanner_folder.rs` (env-only scanner knobs, alerts), `crates/scanner/src/sleeper.rs` (throttling), `crates/heal/src/heal/manager.rs` (`HealConfig::default`), `crates/utils/src/envs.rs` (`EXTERNAL_COMPATIBLE_SUFFIXES`, MinIO env aliases). + +For reproducible scanner-pressure validation and before/after evidence, see [Scanner Benchmark Runbook](scanner-benchmark-runbook.md). Alert thresholds and S3 event names are detailed in [Scanner Excess Alerts](scanner-excess-alerts.md). ## What the scanner does -The scanner is the background maintenance loop that walks stored objects and -feeds several subsystems: - -- usage accounting and data usage cache updates; -- lifecycle expiry and transition admission; -- bucket replication repair admission; -- scanner-originated heal and bitrot checks; -- namespace alerts for excessive versions, retained version size, and folder - fan-out. - -Slowing the scanner can reduce idle CPU and disk pressure, but it also delays -the maintenance work above. Prefer using the status fields below before changing -cycle or pacing values. +The scanner is the background maintenance loop that walks stored objects and feeds usage accounting, lifecycle expiry and transition admission, bucket replication repair admission, scanner-originated heal and bitrot checks, and the namespace excess alerts. Slowing it reduces idle CPU and disk pressure but delays all of that work. Read the status fields below before changing cycle or pacing values. ## Configuration Sources Scanner runtime config is resolved in this order: 1. Environment variables. -2. Persisted admin config for the `scanner` subsystem. +2. Persisted admin config for the `scanner` subsystem (`SCANNER_SUB_SYS`). 3. Built-in defaults or speed preset-derived values. -Bitrot cycle resolution is slightly different because the canonical persistent -key belongs to the `heal` subsystem: +Bitrot cycle resolution differs because the canonical persistent key belongs to the `heal` subsystem: 1. `RUSTFS_SCANNER_BITROT_CYCLE_SECS`. 2. `heal.bitrot_cycle`. 3. Legacy compatibility key `scanner.bitrot_cycle`. 4. Built-in default. -The `/v3/scanner/status` response reports each effective runtime value with a -`source` of `env`, `config`, `scanner_compat_config`, or `default`. +`/v3/scanner/status` reports each effective runtime value with a `source` of `env`, `config`, `scanner_compat_config`, or `default`. Every key in the table below accepts admin config updates that take effect without a restart (`apply_scanner_runtime_config`). ## Runtime Controls -| Persistent key | Environment variable | Unit | Default | Effect | -|---|---|---:|---:|---| -| `scanner.speed` | `RUSTFS_SCANNER_SPEED` | preset | `default` | Selects the base pacing preset: `fastest`, `fast`, `default`, `slow`, or `slowest`. | -| `scanner.delay` | `RUSTFS_SCANNER_DELAY` | factor | preset-derived | Overrides the sleep multiplier. Valid range is `0` through `10000`. | +| Persistent key | Environment variable | Unit | Default (constant) | Effect | +|---|---|---:|---|---| +| `scanner.speed` | `RUSTFS_SCANNER_SPEED` | preset | `default` (`DEFAULT_SCANNER_SPEED`) | Selects the base pacing preset: `fastest`, `fast`, `default`, `slow`, or `slowest`. | +| `scanner.delay` | `RUSTFS_SCANNER_DELAY` | factor | preset-derived | Overrides the sleep multiplier. Valid range is `0` through `10000` (`MAX_SCANNER_DELAY_FACTOR`). | | `scanner.max_wait` | `RUSTFS_SCANNER_MAX_WAIT_SECS` | seconds | preset-derived | Caps one scanner sleep. | | `scanner.cycle` | `RUSTFS_SCANNER_CYCLE` | seconds | preset-derived | Sets the interval between scanner cycles. | -| `scanner.start_delay` | `RUSTFS_SCANNER_START_DELAY_SECS` | seconds | unset | Sets startup delay and, for compatibility, the cycle interval when `scanner.cycle` is unset. | -| `scanner.cycle_max_duration` | `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` | seconds | `1800` | Caps one cycle's runtime. An explicit `0` disables this budget. | -| `scanner.cycle_max_objects` | `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` | objects | `0` | Caps objects processed by one cycle. `0` disables this budget. | -| `scanner.cycle_max_directories` | `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` | directories | `0` | Caps directories entered by one cycle. `0` disables this budget. | -| `heal.bitrot_cycle` | `RUSTFS_SCANNER_BITROT_CYCLE_SECS` | seconds | `2592000` | Controls periodic deep bitrot scans. `false`, `off`, `no`, or `disabled` disables periodic deep scans; `0`, `true`, `on`, or `yes` runs deep mode every scanner cycle. | -| `scanner.idle_mode` | `RUSTFS_SCANNER_IDLE_MODE` | boolean | `true` | Enables scanner sleeps and cooperative throttling. | -| `scanner.cache_save_timeout` | `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` | seconds | `14` | Timeout for saving scanner cache; runtime enforces a minimum of `1` and keeps the default persistence budget within the distributed publication lease. | -| `scanner.max_concurrent_set_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` | count | `4` | Caps concurrent set-level scanner tasks. `0` keeps topology-derived concurrency. | -| `scanner.max_concurrent_disk_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` | count | `4` | Caps concurrent disk bucket walks per set. `0` keeps disk-count-derived concurrency. | -| `scanner.yield_every_n_objects` | `RUSTFS_SCANNER_YIELD_EVERY_N_OBJECTS` | objects | `128` | Controls how often object loops yield to the async runtime. `0` disables this extra yield. | -| `scanner.alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | versions | `100` | Version count threshold for scanner alerts. | -| `scanner.alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | bytes | `1099511627776` | Retained version byte threshold for scanner alerts. | -| `scanner.alert_excess_folders` | `RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS` | folders | `65538` | Direct subfolder threshold for scanner alerts. | +| `scanner.start_delay` | `RUSTFS_SCANNER_START_DELAY_SECS` (deprecated alias `RUSTFS_DATA_SCANNER_START_DELAY_SECS`) | seconds | unset | Sets startup delay and, for compatibility, the cycle interval when `scanner.cycle` is unset. | +| `scanner.cycle_max_duration` | `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` | seconds | `1800` (`DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS`) | Caps one cycle's runtime. An explicit `0` disables this budget. | +| `scanner.cycle_max_objects` | `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` | objects | `0` (`DEFAULT_SCANNER_CYCLE_MAX_OBJECTS`) | Caps objects processed by one cycle. `0` disables this budget. | +| `scanner.cycle_max_directories` | `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` | directories | `0` (`DEFAULT_SCANNER_CYCLE_MAX_DIRECTORIES`) | Caps directories entered by one cycle. `0` disables this budget. | +| `heal.bitrot_cycle` | `RUSTFS_SCANNER_BITROT_CYCLE_SECS` | seconds | `2592000` (`DEFAULT_HEAL_BITROT_CYCLE_SECS`, 30 days) | Controls periodic deep bitrot scans. `false`, `off`, `no`, or `disabled` disables periodic deep scans; `0`, `true`, `on`, or `yes` runs deep mode every scanner cycle. | +| `scanner.idle_mode` | `RUSTFS_SCANNER_IDLE_MODE` | boolean | `true` (`DEFAULT_SCANNER_IDLE_MODE`) | Master switch for scanner throttling: preset sleeps plus the foreground-read backoff floor. `false` disables both and the scanner runs at full speed. | +| `scanner.cache_save_timeout` | `RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS` | seconds | `14` (`DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS`) | Timeout for saving scanner cache; runtime enforces a minimum of `1` and keeps the default persistence budget within the distributed publication lease. | +| `scanner.max_concurrent_set_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` | count | `4` (`DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS`) | Caps concurrent set-level scanner tasks. `0` keeps topology-derived concurrency. | +| `scanner.max_concurrent_disk_scans` | `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` | count | `4` (`DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS`) | Caps concurrent disk bucket walks per set. `0` keeps disk-count-derived concurrency. | +| `scanner.yield_every_n_objects` | `RUSTFS_SCANNER_YIELD_EVERY_N_OBJECTS` | objects | `128` (`DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS`) | Controls how often object loops yield to the async runtime. `0` disables this extra yield. | +| `scanner.alert_excess_versions` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSIONS` | versions | `100` (`DEFAULT_SCANNER_ALERT_EXCESS_VERSIONS`) | Version count threshold for scanner alerts. | +| `scanner.alert_excess_version_size` | `RUSTFS_SCANNER_ALERT_EXCESS_VERSION_SIZE` | bytes | `1099511627776` (`DEFAULT_SCANNER_ALERT_EXCESS_VERSION_SIZE`) | Retained version byte threshold for scanner alerts. | +| `scanner.alert_excess_folders` | `RUSTFS_SCANNER_ALERT_EXCESS_FOLDERS` | folders | `65538` (`DEFAULT_SCANNER_ALERT_EXCESS_FOLDERS`) | Direct subfolder threshold for scanner alerts. | -The `fastest`, `fast`, `default`, `slow`, and `slowest` presets set the base -sleep multiplier, maximum wait, and cycle interval. Use `scanner.delay`, -`scanner.max_wait`, and `scanner.cycle` when the preset is close but one axis -needs a precise override. +Speed presets (`crates/config/src/constants/scanner.rs`) set the base sleep multiplier, maximum wait, and cycle interval: -When the cycle duration control is unset, RustFS uses a finite 1800-second -(30-minute) default, matching the scanner benchmark guidance. An explicit `0` -preserves the compatibility behavior of an unbounded cycle; object and -directory budgets likewise remain unbounded when explicitly set to `0`. Invalid -or overflowing duration environment values are configuration errors rather than -silent fallback values. +| Preset | Sleep factor | Max sleep | Cycle interval | +|---|---:|---:|---:| +| `fastest` | 0 | 0 | 1s | +| `fast` | 1x | 100ms | 60s | +| `default` | 2x | 1s | 60s | +| `slow` | 10x | 15s | 60s | +| `slowest` | 100x | 15s | 30m | -When a finite deadline expires, RustFS cancels cooperative scanner work and -waits only for the existing bounded shutdown window. A non-yielding I/O future -is dropped after that window. RustFS then attempts a higher leadership epoch so -late cycle, usage, cache, and remote writes from the old generation fail closed. -If the worker cannot stop cooperatively, the cycle state was not confirmed -durable, or that epoch fence cannot be durably persisted, the scanner reports -`recovery-required`; it does not claim an uncooperative cursor was saved. +Use `scanner.delay`, `scanner.max_wait`, and `scanner.cycle` when the preset is close but one axis needs a precise override. With `idle_mode=true`, directory-level sleep is `1ms x factor` and object-level sleep is `time spent on the object x factor`, both capped at `max_wait`; a foreground-read floor of `FOREGROUND_READ_BACKOFF_PER_REQUEST_MS` (10ms) per concurrent GetObject/streaming read, capped at `FOREGROUND_READ_BACKOFF_MAX_MS` (250ms), is applied on top and can exceed the preset's `max_wait` (`crates/scanner/src/sleeper.rs`). -An explicit `scanner.cycle` or `RUSTFS_SCANNER_CYCLE` is a minimum inter-cycle -cadence: dirty-usage notifications do not bypass that configured interval. -The default adaptive policy continues to use dirty-usage notifications to wake -the scanner between timer-driven cycles. +### Environment-only scanner knobs + +These have no persistent key and are read from the environment only. + +| Environment variable | Default (constant) | Effect | +|---|---|---| +| `RUSTFS_SCANNER_ENABLED` (deprecated alias `RUSTFS_ENABLE_SCANNER`) | `true` (`scanner_enabled_from_env`, `rustfs/src/module_switches.rs`) | Starts the data scanner at all. The heal manager is initialized whenever heal or scanner is enabled, because scanner-produced heal candidates need a consumer. | +| `RUSTFS_SCANNER_ALERT_COOLDOWN_SECS` | `86400` (`DEFAULT_SCANNER_ALERT_COOLDOWN_SECS`, `scanner_folder.rs`) | Per-(kind, bucket, object) cooldown between S3 excess-alert events; `0` emits every cycle. See [Scanner Excess Alerts](scanner-excess-alerts.md). | +| `RUSTFS_SCANNER_DEEP_VERIFY_COOLDOWN_SECS` | `60` (`DEFAULT_SCANNER_DEEP_VERIFY_COOLDOWN_SECS`, `scanner_folder.rs`) | Objects modified within this window are skipped by deep (bitrot) verification in the current cycle. | +| `RUSTFS_HEAL_OBJECT_SELECT_PROB` | `1024` (`DEFAULT_HEAL_OBJECT_SELECT_PROB`, `scanner_folder.rs`) | Sampling divisor for scanner-originated heal checks: roughly one object in N per cycle is selected for a low-priority heal check. | +| `RUSTFS_DATA_USAGE_UPDATE_DIR_CYCLES` | `16` (`DATA_USAGE_UPDATE_DIR_CYCLES`, `scanner_folder.rs`) | Every N cycles a compacted directory is re-descended instead of reusing its cached usage. `1` forces re-descent every cycle (used by lifecycle e2e lanes). | +| `RUSTFS_DATA_USAGE_FAILED_OBJECT_TTL_SECS` | `86400` (`DEFAULT_FAILED_OBJECT_TTL_SECS`, `scanner_folder.rs`) | Retention of per-bucket failed-object entries in the usage cache. | +| `RUSTFS_DATA_USAGE_FAILED_OBJECTS_MAX` | `10000` (`DEFAULT_FAILED_OBJECTS_MAX`, `scanner_folder.rs`) | Cap on retained failed-object entries per bucket. | + +### Cycle budgets and cadence + +When the cycle duration control is unset, RustFS uses the finite 1800-second default. An explicit `0` preserves the compatibility behavior of an unbounded cycle; object and directory budgets likewise remain unbounded when explicitly set to `0`. Invalid or overflowing duration environment values are configuration errors rather than silent fallback values. + +When a finite deadline expires, RustFS cancels cooperative scanner work and waits only for the existing bounded shutdown window. A non-yielding I/O future is dropped after that window. RustFS then attempts a higher leadership epoch so late cycle, usage, cache, and remote writes from the old generation fail closed. If the worker cannot stop cooperatively, the cycle state was not confirmed durable, or that epoch fence cannot be durably persisted, the scanner reports `recovery-required`; it does not claim an uncooperative cursor was saved. + +An explicit `scanner.cycle` or `RUSTFS_SCANNER_CYCLE` is a minimum inter-cycle cadence: dirty-usage notifications do not bypass that configured interval. The default adaptive policy continues to use dirty-usage notifications to wake the scanner between timer-driven cycles. ## Single-disk clean-idle scheduling -An erasure single-disk deployment using the built-in cycle and bitrot defaults -automatically backs off repeated clean idle scans instead of walking the same -unchanged namespace every minute. Each successful timer-driven cycle that -finds no dirty usage or unresolved maintenance work doubles the next interval. -The status endpoint reports the effective interval and multiplier. +An erasure single-disk deployment using the built-in cycle and bitrot defaults automatically backs off repeated clean idle scans instead of walking the same unchanged namespace every minute. Each successful timer-driven cycle that finds no dirty usage or unresolved maintenance work doubles the next interval. The status endpoint reports the effective interval and multiplier. -The backoff is reset to the base interval by object or bucket mutations, -lifecycle or replication configuration changes, partial or failed cycles, -usage persistence failures, and unresolved scanner-originated heal or bitrot -work. Active lifecycle or replication rules keep the base cadence. An explicit -cycle, a non-default persisted speed, any environment speed or start-delay -override, an environment bitrot override, or a non-default persisted active -bitrot cycle also keeps the configured cadence rather than applying the -automatic policy. Persisting `scanner.speed=default` or the default bitrot cycle -is normalized to the built-in default and therefore keeps automatic scheduling -enabled. +The backoff is reset to the base interval by object or bucket mutations, lifecycle or replication configuration changes, partial or failed cycles, usage persistence failures, and unresolved scanner-originated heal or bitrot work. Active lifecycle or replication rules keep the base cadence. An explicit cycle, a non-default persisted speed, any environment speed or start-delay override, an environment bitrot override, or a non-default persisted active bitrot cycle also keeps the configured cadence rather than applying the automatic policy. Persisting `scanner.speed=default` or the default bitrot cycle is normalized to the built-in default and therefore keeps automatic scheduling enabled. -Lifecycle and replication configuration inspection is bounded so a slow -metadata read cannot stall scanner startup or scheduling. A failed or timed-out -inspection keeps the base cadence and is retried after 5 minutes, doubling up -to a maximum of 60 minutes while failures continue. A lifecycle or replication -configuration change wakes the scanner and retries inspection immediately. +Lifecycle and replication configuration inspection is bounded so a slow metadata read cannot stall scanner startup or scheduling. A failed or timed-out inspection keeps the base cadence and is retried after 5 minutes, doubling up to a maximum of 60 minutes while failures continue. A lifecycle or replication configuration change wakes the scanner and retries inspection immediately. -With the default 30-day bitrot cycle, the clean-idle interval is capped at the -bitrot cycle divided by the object selection window. With the default selection -window this is about 42 minutes, which preserves the intended wall-clock bitrot -coverage. If periodic bitrot is disabled, the clean-idle policy cap is 24 hours. -The effective interval is jittered by up to 10 percent to avoid synchronized -scanner starts. +With the default 30-day bitrot cycle, the clean-idle interval is capped at the bitrot cycle divided by the object selection window (about 42 minutes with the default `RUSTFS_HEAL_OBJECT_SELECT_PROB`), which preserves the intended wall-clock bitrot coverage. If periodic bitrot is disabled, the clean-idle cap is 24 hours. The effective interval is jittered by up to 10 percent to avoid synchronized scanner starts. ## Status Endpoint -The scanner status route is: - ```text GET /v3/scanner/status ``` -The request must be authenticated with an admin identity that has -`ServerInfoAdminAction`. The JSON response has three scanner-specific top-level -objects: +The request must be authenticated with an admin identity that has `ServerInfoAdminAction`. The JSON response has these scanner-specific top-level objects: -- `runtime_config`: the effective runtime controls and their value sources. -- `cycle_schedule`: the current effective cycle interval and clean-idle - backoff state. -- `metrics`: scanner work, pressure, checkpoint, lifecycle, replication, heal, - bitrot, and alert counters. -- `data_movement_pause`: the global-pause policy, current movement reason, - operation epoch, start time, duration, and estimated movement work items. -- `pause_backlog`: the replicated durable pause ledger, post-pause catch-up - phase, rate window, retry state, thresholds, and active alert reasons. -- `catch_up_estimate`: movement work plus current dirty-usage and already - discovered lifecycle queues. +| Object | Content | +|---|---| +| `runtime_config` | Effective runtime controls and their value sources. | +| `cycle_schedule` | Current effective cycle interval and clean-idle backoff state. | +| `metrics` | Scanner work, pressure, checkpoint, lifecycle, replication, heal, bitrot, and alert counters. | +| `data_movement_pause` | Global-pause policy, current movement reason, operation epoch, start time, duration, and estimated movement work items. | +| `pause_backlog` | Replicated durable pause ledger, post-pause catch-up phase, rate window, retry state, thresholds, and active alert reasons. | +| `catch_up_estimate` | Movement work plus current dirty-usage and already discovered lifecycle queues. | Example fields to inspect: @@ -189,28 +153,9 @@ catch_up_estimate.discovered_transition_items ## Usage State Reset -The supported break-glass route for rebuilding scanner usage state is: +The supported break-glass route for rebuilding scanner usage state is `POST /v3/scanner/usage-state/reset` with body `{"mode":"full-rebuild"}`, authenticated as an admin identity holding `ConfigUpdateAdminAction` (route registered in `rustfs/src/admin/route_registration_test.rs`). Use it only after the scanner status shows a usage-floor load failure, a conflicting persisted usage floor, or an operator decision to discard the durable usage baseline and rebuild it from a full scanner pass. -```text -POST /v3/scanner/usage-state/reset -{"mode":"full-rebuild"} -``` - -The request must be authenticated with an admin identity that has -`ConfigUpdateAdminAction`. Use it only after confirming the scanner status shows -a usage-floor load failure, a conflicting persisted usage floor, or an operator -decision to discard the durable usage baseline and rebuild it from a full -scanner pass. - -The reset does not delete metadata files from disk by hand and does not publish -an authoritative zero-usage snapshot. It holds the scanner leader lock, fences -the operation with the storage-owned publication epoch, CAS-publishes a v2 -`bootstrap-pending` marker in the primary usage slot, then clears stale backup, -legacy, and observed usage slots by object revision. The next scanner -leadership claim binds that marker to a fresh epoch and the next complete -scanner cycle replaces it with authoritative usage. - -The JSON response is machine-readable: +The reset does not delete metadata files by hand and does not publish an authoritative zero-usage snapshot. It holds the scanner leader lock, fences the operation with the storage-owned publication epoch, CAS-publishes a v2 `bootstrap-pending` marker in the primary usage slot, then clears stale backup, legacy, and observed usage slots by object revision. The next scanner leadership claim binds that marker to a fresh epoch, and the next complete scanner cycle replaces it with authoritative usage. ```json { @@ -229,92 +174,57 @@ The JSON response is machine-readable: } ``` -If the response is an error mentioning data movement, wait for decommission or -rebalance to leave the scanner metadata path and retry. If it reports that the -scanner cycle state is invalid, run the cycle-state recovery reset first: - -```text -POST /v3/scanner/cycle-state/reset -{"mode":"full-rescan"} -``` +| Error mentions | Do | +|---|---| +| data movement | wait for decommission or rebalance to leave the scanner metadata path, then retry | +| invalid scanner cycle state | run `POST /v3/scanner/cycle-state/reset` with `{"mode":"full-rescan"}` first | ## Data Movement Pauses -RustFS currently uses a `global_pause` policy while pool decommission or -rebalance can hide scanner metadata. Usage publication, lifecycle discovery, -tier cleanup discovery, scanner-originated heal and bitrot checks, and -replication discovery are deferred together. A failed or canceled -decommission remains a publication barrier until an operator retries or clears -it. +RustFS uses a `global_pause` policy while pool decommission or rebalance can hide scanner metadata: usage publication, lifecycle discovery, tier cleanup discovery, scanner-originated heal and bitrot checks, and replication discovery are deferred together. A failed or canceled decommission remains a publication barrier until an operator retries or clears it. The same pause and estimate objects are included in `GET /v3/ilm/expiry/status`. -`data_movement_pause.reasons` combines the in-process decommission worker state -with the durable pool and rebalance operation metadata. Exhausted operation -epochs or movement generations also fail closed and appear as explicit pause -reasons. Its start time, duration, and movement backlog come from the durable -metadata; a worker-only or exhausted-counter snapshot can therefore report -`paused=true` with zero start time and backlog. -`movement_backlog_work_items` counts remaining movement bucket work units, not -expired objects. `catch_up_estimate` combines that estimate with dirty-usage -buckets and lifecycle items that were already discovered before or during the -pause. The API sets `undiscovered_ilm_items_known=false` because a global pause -cannot count newly expired objects without scanning the namespace. Use -`usage_baseline_unix_secs` to judge the age of that estimate. +| Field | Meaning | +|---|---| +| `data_movement_pause.reasons` | In-process decommission worker state combined with durable pool and rebalance operation metadata. Exhausted operation epochs or movement generations fail closed and appear as explicit reasons. | +| `data_movement_pause.duration_seconds`, start time, `movement_backlog_work_items` | From durable metadata; a worker-only or exhausted-counter snapshot can report `paused=true` with zero start time and backlog. `movement_backlog_work_items` counts remaining movement bucket work units, not expired objects. | +| `catch_up_estimate` | Movement estimate plus dirty-usage buckets and lifecycle items discovered before or during the pause. `undiscovered_ilm_items_known=false` because a global pause cannot count newly expired objects without scanning; use `usage_baseline_unix_secs` to judge the estimate's age. | +| `pause_backlog.persistence_state` | `persistence_unavailable` when the `.scanner-pause-backlog.json` ledger cannot be read or updated; scanner cycles stay gated and persistence is retried every five minutes. | +| `pause_backlog.phase` | `idle`, `paused`, `catching_up`, or `retry_exhausted`. The ledger returns to `idle` only after one successful full namespace scan and zero known dirty-usage, expiry, and transition queues. | +| `membership_repair_pending` | A rejoining decommission source is being re-seeded from the last committed surviving-set ledger before a new full-membership commit is allowed. | +| `pause_backlog.thresholds` | Exact pause-duration, deferred-cycle, backlog-size, rate, and failure limits used by the running binary. | +| `pause_backlog.alert_reasons` | Exceeded thresholds, exhausted counters or retries, replica degradation, and persistence failures. | -The same pause and estimate objects are included in -`GET /v3/ilm/expiry/status`. The gauges -`rustfs_scanner_data_movement_paused`, -`rustfs_scanner_data_movement_pause_duration_seconds`, and -`rustfs_scanner_data_movement_backlog_work_items` expose the local snapshot -without bucket-name labels. +Built-in limits reported under `pause_backlog.thresholds`: -The scanner persists `.scanner-pause-backlog.json` independently on erasure -sets in every surviving pool. A generation becomes authoritative only after -the identical commit record reaches every set named by its membership marker. -When a failed, canceled, or cleared decommission source rejoins, the last -committed surviving-set ledger seeds it before a new full-membership commit is -allowed; a smaller stale source membership cannot override the largest valid -surviving-set proof, and a membership claim is valid only when every declared -member stores the same proof. This repair appears as -`membership_repair_pending`. A partial commit is -rolled back to the previous stable generation after a crash or leader switch. -The ledger never rewrites pool or rebalance movement state. A new scanner -leader recovers the committed writer epoch and generation, counts an -interrupted attempt as a failure, and requires one successful full namespace -scan after movement clears. Known dirty-usage, expiry, and transition queues -must also reach zero before the ledger returns to `idle`. If the ledger cannot -be read or updated, scanner cycles remain gated and persistence is retried -every five minutes; the management status reports `persistence_unavailable` -until recovery. +| Limit | Value | +|---|---| +| Pause-duration alert | 24 hours | +| Movement-deferral alert | 3 deferrals in one unconverged pause episode | +| Backlog alert | 10,000 known pending work items | +| Catch-up rate window | At most 4 attempts per hour, no more than 1 per 5 minutes | +| Retry exhaustion | 5 consecutive failed or interrupted attempts, then a sparse hourly probe | -Catch-up attempts remain subject to the normal cycle duration, object, -directory, sleeper, and foreground-read budgets. The additional durable rate -window admits at most four attempts per hour and no more than one attempt per -five minutes. Five consecutive failed or interrupted attempts move the ledger -to `retry_exhausted`; accelerated retries stop and a sparse hourly probe is -used instead. A successful probe can return to bounded catch-up. +Catch-up attempts remain subject to the normal cycle duration, object, directory, sleeper, and foreground-read budgets. -`pause_backlog.thresholds` reports the exact pause-duration, deferred-cycle, -backlog-size, rate, and failure limits used by the running binary. -`pause_backlog.alert_reasons` identifies exceeded thresholds, exhausted -counters or retries, replica degradation, and persistence failures. The -threshold alerts fire after a 24-hour pause, three movement deferrals in one -unconverged pause episode, or 10,000 known pending work items. The -corresponding unlabeled gauges are: +Unlabeled Prometheus gauges: -- `rustfs_scanner_pause_backlog_phase` (`0` idle, `1` paused, `2` catching up, - `3` retry exhausted); -- `rustfs_scanner_pause_backlog_pause_duration_seconds`; -- `rustfs_scanner_pause_backlog_pending_work_items`; -- `rustfs_scanner_pause_backlog_consecutive_failures`; -- `rustfs_scanner_pause_backlog_rate_limited`; -- `rustfs_scanner_pause_backlog_retry_exhausted`; -- `rustfs_scanner_pause_backlog_alerting`; -- `rustfs_scanner_pause_backlog_replica_degraded`. +| Gauge | Meaning | +|---|---| +| `rustfs_scanner_data_movement_paused` | Local pause snapshot. | +| `rustfs_scanner_data_movement_pause_duration_seconds` | Local pause duration. | +| `rustfs_scanner_data_movement_backlog_work_items` | Local movement backlog. | +| `rustfs_scanner_pause_backlog_phase` | `0` idle, `1` paused, `2` catching up, `3` retry exhausted. | +| `rustfs_scanner_pause_backlog_pause_duration_seconds` | Durable pause duration. | +| `rustfs_scanner_pause_backlog_pending_work_items` | Known pending work items. | +| `rustfs_scanner_pause_backlog_consecutive_failures` | Consecutive failed catch-up attempts. | +| `rustfs_scanner_pause_backlog_rate_limited` | Catch-up currently rate limited. | +| `rustfs_scanner_pause_backlog_retry_exhausted` | Ledger in `retry_exhausted`. | +| `rustfs_scanner_pause_backlog_alerting` | Any alert reason active. | +| `rustfs_scanner_pause_backlog_replica_degraded` | Ledger replica set degraded. | ## Reading Pacing Pressure -`metrics.pacing_pressure.primary_pressure` summarizes the highest-priority -scanner pressure signal: +`metrics.pacing_pressure.primary_pressure` summarizes the highest-priority scanner pressure signal: | Value | Meaning | Usual response | |---|---|---| @@ -324,49 +234,21 @@ scanner pressure signal: | `active_scans` | Scanner work is active but not currently queued or budget-limited. | Usually healthy; correlate with CPU/disk metrics. | | `none` | No current scanner pressure was observed. | No scanner pacing action needed. | -The ratio fields are fractions of the last cycle duration: - -- `last_cycle_throttle_sleep_ratio` -- `last_cycle_yield_ratio` -- `last_cycle_total_pause_ratio` - -If CPU is high but pause ratios are already high, increasing `scanner.delay` or -`scanner.max_wait` may have limited value. Check active paths, source work, and -disk activity before changing the cycle interval. +The ratio fields `last_cycle_throttle_sleep_ratio`, `last_cycle_yield_ratio`, and `last_cycle_total_pause_ratio` are fractions of the last cycle duration. If CPU is high but pause ratios are already high, increasing `scanner.delay` or `scanner.max_wait` may have limited value; check active paths, source work, and disk activity before changing the cycle interval. ## Reading Source Work -`metrics.source_work`, `metrics.current_cycle_source_work`, and -`metrics.last_cycle_source_work` group scanner work by source: +`metrics.source_work`, `metrics.current_cycle_source_work`, and `metrics.last_cycle_source_work` group scanner work by source: `usage`, `lifecycle`, `bucket_replication`, `site_replication`, `heal`, `bitrot`, `alerts`. -- `usage` -- `lifecycle` -- `bucket_replication` -- `site_replication` -- `heal` -- `bitrot` -- `alerts` - -Each source has `checked`, `queued`, `executed`, `failed`, `skipped`, and -`missed` counters. `missed` means the scanner found work but could not admit it -to the downstream queue. `skipped` means the work was intentionally merged or -deduplicated. - -Use these counters to decide whether scan progress is limited by scanner pacing -or by a downstream subsystem such as lifecycle transition, replication repair, -or heal admission. +Each source has `checked`, `queued`, `executed`, `failed`, `skipped`, and `missed` counters. `missed` means the scanner found work but could not admit it to the downstream queue. `skipped` means the work was intentionally merged or deduplicated. Use these counters to decide whether scan progress is limited by scanner pacing or by a downstream subsystem such as lifecycle transition, replication repair, or heal admission. ## Reading Heal Operations -The background heal status route is: - ```text POST /v3/background-heal/status ``` -It reports scanner-driven bitrot state together with heal queue execution -state. `healQueueLength` and `healActiveTasks` keep the legacy totals. -`healOperations` adds the same totals split by request source and priority: +Reports scanner-driven bitrot state together with heal queue execution state. `healQueueLength` and `healActiveTasks` keep the legacy totals; `healOperations` adds the same totals split by request source and priority: | Field | Meaning | |---|---| @@ -377,32 +259,83 @@ state. `healQueueLength` and `healActiveTasks` keep the legacy totals. | `queuedByPriority` | Queued requests split into `low`, `normal`, `high`, and `urgent`. | | `activeByPriority` | Running tasks split into `low`, `normal`, `high`, and `urgent`. | -Use this route when `metrics.source_work` shows `heal` or `bitrot` queued or -missed work. Scanner-originated object checks should appear under -`scanner/low` for opportunistic work, while manual admin heal should appear -under `admin/high`. If scanner work grows but admin work remains blocked, treat -that as heal queue pressure rather than scanner pacing pressure. +Use this route when `metrics.source_work` shows `heal` or `bitrot` queued or missed work. Scanner-originated object checks should appear under `scanner/low`, manual admin heal under `admin/high`. If scanner work grows but admin work remains blocked, treat that as heal queue pressure rather than scanner pacing pressure. + +## Heal runtime controls + +Heal knobs are environment-only and read by `HealConfig::default` (`crates/heal/src/heal/manager.rs`), the MRF queue (`crates/heal/src/heal/mrf_queue.rs`), or the erasure-set healer (`crates/heal/src/heal/erasure_healer.rs`). The admin `heal` config subsystem accepts only `bitrot_cycle` (`HEAL_KEYS`), which is documented in the scanner table above. Constants live in `crates/config/src/constants/heal.rs` unless another file is named. + +| Environment variable | Default (constant) | Effect | +|---|---|---| +| `RUSTFS_HEAL_ENABLED` (deprecated alias `RUSTFS_ENABLE_HEAL`) | `true` (`heal_enabled_from_env`, `rustfs/src/module_switches.rs`) | Master switch for the background heal manager. | +| `RUSTFS_HEAL_AUTO_HEAL_ENABLE` | `true` (`DEFAULT_HEAL_AUTO_HEAL_ENABLE`) | Enables automatic healing of detected issues; `false` leaves healing to manual admin requests. | +| `RUSTFS_HEAL_QUEUE_SIZE` | `10000` (`DEFAULT_HEAL_QUEUE_SIZE`) | Heal request queue capacity. | +| `RUSTFS_HEAL_INTERVAL_SECS` | `10` (`DEFAULT_HEAL_INTERVAL_SECS`) | Heal manager polling interval. | +| `RUSTFS_HEAL_TASK_TIMEOUT_SECS` | `300` (`DEFAULT_HEAL_TASK_TIMEOUT_SECS`) | Per-task timeout. | +| `RUSTFS_HEAL_MAX_CONCURRENT_HEALS` | `4` (`DEFAULT_HEAL_MAX_CONCURRENT_HEALS`) | Global concurrent heal task limit. | +| `RUSTFS_HEAL_MAX_CONCURRENT_PER_SET` | `1` (`DEFAULT_HEAL_MAX_CONCURRENT_PER_SET`) | Per-erasure-set limit; effective value is `min(global, per_set)`, each floored at `1`. | +| `RUSTFS_HEAL_LOW_PRIORITY_MERGE_ENABLE` | `true` (`DEFAULT_HEAL_LOW_PRIORITY_MERGE_ENABLE`) | Merge duplicate low-priority requests with the same dedup key. | +| `RUSTFS_HEAL_LOW_PRIORITY_DROP_WHEN_FULL` | `true` (`DEFAULT_HEAL_LOW_PRIORITY_DROP_WHEN_FULL`) | Drop, rather than block on, low-priority requests when the queue is full. | +| `RUSTFS_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE` | `true` (`DEFAULT_HEAL_EVENT_DRIVEN_SCHEDULER_ENABLE`) | Notify-driven scheduler wakeups. | +| `RUSTFS_HEAL_SET_BULKHEAD_ENABLE` | `true` (`DEFAULT_HEAL_SET_BULKHEAD_ENABLE`) | Per-set bulkhead scheduling. | +| `RUSTFS_HEAL_PAGE_PARALLEL_ENABLE` | `true` (`DEFAULT_HEAL_PAGE_PARALLEL_ENABLE`) | Page-level parallel object healing during erasure-set repair. | +| `RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY` | `8` (`DEFAULT_HEAL_PAGE_OBJECT_CONCURRENCY`) | Concurrent object heals within one erasure-set page. Forced to `1` when page parallelism is off, for `Deep` scan mode, and for `AutoHeal`-sourced requests (`ErasureSetHealer::effective_heal_page_object_concurrency_for_source`). | +| `RUSTFS_HEAL_MAINLINE_THROTTLE_ENABLE` | `true` (`DEFAULT_HEAL_MAINLINE_THROTTLE_ENABLE`) | Pause best-effort heal task starts while foreground I/O is saturated. | +| `RUSTFS_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT` | `80` (`DEFAULT_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT`, capped at 100) | Foreground read-permit utilization at which heal starts pause. | +| `RUSTFS_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT` | `80` (`DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT`, capped at 100) | Foreground write utilization at which heal starts pause. | +| `RUSTFS_HEAL_MAINLINE_MAX_SLEEP_MS` | `250` (`DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS`) | Recheck delay after deferring heal starts for foreground pressure. | +| `RUSTFS_HEAL_OVERLAP_POLICY` | `merge` (`DEFAULT_HEAL_OVERLAP_POLICY`) | `merge` dedups an admin heal start that overlaps a running or queued heal; `minio_error` returns a typed already-running / overlapping-paths rejection like madmin. | +| `RUSTFS_HEAL_MRF_ENABLE` | `true` (`DEFAULT_HEAL_MRF_ENABLE`) | MRF intent pipeline: error paths deliver repair intents to the heal runtime and unconsumed intents replay from the durable journal after restart. | +| `RUSTFS_HEAL_MRF_QUEUE_SIZE` | `100000` (`DEFAULT_HEAL_MRF_QUEUE_SIZE`) | MRF in-memory queue capacity. | +| `RUSTFS_HEAL_MRF_JOURNAL_MAX_BYTES` | `8388608` (`DEFAULT_HEAL_MRF_JOURNAL_MAX_BYTES`, 8 MiB) | MRF journal size at which compaction runs. | +| `RUSTFS_HEAL_MRF_REPLAY_BATCH` | `256` (`DEFAULT_HEAL_MRF_REPLAY_BATCH`) | Intents per replay push round. | +| `RUSTFS_HEAL_DANGLING_DELETE_GRACE_SECS` | `3600` (`DEFAULT_HEAL_DANGLING_DELETE_GRACE_SECS`, `crates/ecstore/src/set_disk/core/io_primitives.rs`) | A recently modified object is never deleted as dangling inside this window; `0` disables the grace window. | + +## Deliberate non-parity with MinIO + +These differences from MinIO are design decisions, recorded so they are not re-filed as gaps. + +| Area | RustFS behavior | Why it is not a gap | +|---|---|---| +| Bloom filter | `.bloomcycle.bin` (`DATA_USAGE_BLOOM_NAME`, `crates/scanner/src/data_usage_define.rs`) is reused only as the cycle/epoch fence. | MinIO master removed the bloom filter too. | +| Scanner leadership | Single cluster-wide scanner leader plus an epoch fence. | Same model as MinIO; the fence is additive. | +| Heal notifications | Heal emits no S3 bucket notification; results are exposed through admin status. | Same as MinIO. | +| Incomplete multipart cleanup | Runs as an independent background routine, not inside the scanner or ILM. | Same as MinIO. | +| Inline heal | The scanner only enqueues heal candidates; nothing heals inline on the scan path. | MinIO's inline `applyHealing` path is intentionally not a parity target. | +| Heal-sequence keep-alive | Admin heal status is a snapshot query with incremental `sinceSeq`/`nextSeq` semantics (`crates/heal-contracts/src/heal_channel.rs`). | MinIO's 10-second blank keep-alive write-back belongs to its streaming model and is not copied. | +| `.trash` / `tmp-old` paths | Layout constants are RustFS's own (`crates/ecstore/src/disk/local.rs`). | No literal alignment with MinIO path names is intended. | + +## Migrating from MinIO scanner settings + +Only two MinIO scanner variables are recognized. `apply_external_env_compat` (`crates/utils/src/envs.rs`, called from `rustfs/src/startup_preflight.rs`) copies `MINIO_` into `RUSTFS_` at startup for suffixes on `EXTERNAL_COMPATIBLE_SUFFIXES`, and only when the `RUSTFS_` key is absent; when both are set with different values the `RUSTFS_` value wins and a `Detected external-prefix compatibility conflicts` warning is logged. The scanner suffixes on that list are `SCANNER_SPEED` and `SCANNER_CYCLE` (tests `scanner_aliases_are_mapped_when_rustfs_missing` in `crates/utils/src/envs.rs`, `test_cycle_interval_supports_minio_speed_alias` and `test_cycle_interval_supports_minio_cycle_alias` in `crates/scanner/src/scanner/tests.rs`). Every other `MINIO_SCANNER_*` or `MINIO_HEAL_*` variable is silently ignored. + +| MinIO setting | RustFS setting | Migration | +|---|---|---| +| `MINIO_SCANNER_SPEED` / `scanner speed` | `RUSTFS_SCANNER_SPEED` / `scanner.speed` | Env alias mapped at startup; preset names and the preset table are identical. | +| `MINIO_SCANNER_CYCLE` / `scanner cycle` | `RUSTFS_SCANNER_CYCLE` / `scanner.cycle` | Env alias mapped at startup. | +| `MINIO_SCANNER_IDLE_SPEED` / `scanner idle_speed` (`on` default, `off`) | `RUSTFS_SCANNER_IDLE_MODE` / `scanner.idle_mode` (`true` default, `false`) | Not mapped; must be rewritten. Direction matches (`on` and `true` both mean throttled). Both RustFS channels also accept `on`/`off` as booleans (`parse_config_bool`, `parse_bool_str`). RustFS `false` additionally disables the foreground-read backoff floor that MinIO does not have, so the scanner competes with foreground reads at full speed; use it only for benchmarks or exclusive-I/O windows. | +| `MINIO_HEAL_BITROTSCAN` / `heal bitrotscan` (default `off`) | `RUSTFS_SCANNER_BITROT_CYCLE_SECS` / `heal.bitrot_cycle` (default 30 days) | Not mapped. RustFS deep-scans periodically by default; set `off` or `disabled` to reproduce MinIO's default. | +| `MINIO_API_STALE_UPLOADS_EXPIRY` (24h) | `RUSTFS_API_STALE_UPLOADS_EXPIRY` (`DEFAULT_STALE_UPLOADS_EXPIRY`, 24h, `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs`) | Not mapped; same default. | +| `MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL` (6h) | `RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL` (`DEFAULT_STALE_UPLOADS_CLEANUP_INTERVAL`, 6h) | Not mapped; same default. | +| `MINIO_API_DELETE_CLEANUP_INTERVAL` (5m) | None; `DELETED_OBJECTS_CLEANUP_INTERVAL` is a 5-minute constant in `crates/ecstore/src/disk/local.rs`. | No knob. Trash draining is not per-entry throttled. | +| `scanner alert_excess_folders` (50000) | `scanner.alert_excess_folders` (65538) | Not mapped; see [Scanner Excess Alerts](scanner-excess-alerts.md). | +| Any other `MINIO_SCANNER_*` | Corresponding `RUSTFS_SCANNER_*` from the tables above | Not mapped; rename explicitly. | + +Stale-upload cleanup differs in one crash-recovery detail. RustFS's stale multipart cleanup (`cleanup_stale_multipart_uploads_in_set`) takes a namespace write lock, re-checks the upload under write quorum (`check_multipart_upload_path_exists`), and fans the delete out to every disk, where the local recursive delete renames the directory into `.rustfs.sys/tmp/.trash/` (`move_to_trash`). If the process dies mid fan-out after more disks than the parity count have already moved the upload directory, the next cleanup pass's quorum re-check fails (`FileNotFound` is not in `OBJECT_OP_IGNORED_ERRS`) and the candidate is skipped, so the remaining per-disk residue is not reclaimed by that job. The residue is invisible to the S3 API and only consumes disk space; the window is milliseconds wide. MinIO processes each disk independently and converges in the same scenario. ## Replacement Recovery Completion -`POST /v3/background-heal/status` is an execution-queue view. `state=idle`, zero queue and active counts, an online disk, a readable object, or acceptance of an Admin deep-heal request do not independently prove that a replacement disk contains every erasure shard. +`POST /v3/background-heal/status` is an execution-queue view. `state=idle`, zero queue and active counts, an online disk, a readable object, or acceptance of an admin deep-heal request do not independently prove that a replacement disk contains every erasure shard. -Treat replacement recovery as verified only after the repair task has completed for the exact replacement instance and an operator has confirmed the target disk contains the expected `xl.meta` and data parts for every relevant object version. A replacement that is not mounted, is unsafe to format, loses its marker, or returns a partial target outcome must be treated as deferred or incomplete rather than complete. +Treat replacement recovery as verified only after the repair task has completed for the exact replacement instance and an operator has confirmed the target disk contains the expected `xl.meta` and data parts for every relevant object version. A replacement that is not mounted, is unsafe to format, loses its marker, or returns a partial target outcome must be treated as deferred or incomplete. Do not automate destructive replacement actions from an `idle` observation alone. A new node must not infer replacement completion from an old or unavailable peer; regard that information as unknown or degraded until every required peer can report the same replacement instance and verified completion. -The v3 route and its peer status protocol preserve their existing fields for mixed-version clusters. A new node must not infer replacement completion from an old or unavailable peer; regard that information as unknown or degraded until every required peer can report the same replacement instance and verified completion. Do not automate destructive replacement actions from an `idle` observation alone. +`GET /rustfs/admin/v4/heal/replacement-recovery` reports durable automatic replacement records from survivor disks. `local.records[]` entries distinguish `waiting_for_replacement`, `running`, `incomplete`, `unrecoverable`, `cleanup_pending`, `completed`, and `unknown`; `local.definitive=false` or any `unknown` record means the node could not prove a local replacement state. The `cluster` section queries the replacement-recovery peer RPC and sets `cluster.definitive=true` only when the expected peer topology is complete, every peer supports the RPC, every peer snapshot is locally definitive, and all peers report the same replacement records. Old peers, unavailable peers, malformed peer payloads, topology gaps, and generation disagreements are reported as degraded or unknown rather than complete. -`GET /rustfs/admin/v4/heal/replacement-recovery` reports durable automatic replacement records from survivor disks. Its `local.records[]` entries distinguish `waiting_for_replacement`, `running`, `incomplete`, `unrecoverable`, `cleanup_pending`, `completed`, and `unknown`; `local.definitive=false` or any `unknown` record means the node could not prove a local replacement state. Its `cluster` section queries the replacement-recovery peer RPC and sets `cluster.definitive=true` only when the expected peer topology is complete, every peer supports the RPC, every peer snapshot is locally definitive, and all peers report the same replacement records. Old peers, unavailable peers, malformed peer payloads, topology gaps, and generation disagreements are reported as degraded or unknown rather than complete. - -Replacement resume and checkpoint files use an independent on-disk schema. A newer reader rejects a future schema rather than continuing with data it cannot interpret, while an older binary cannot safely enforce the new generation fence because it may ignore fields it does not know. Do not roll a cluster back after a replacement generation has started. Complete that recovery with the current-or-newer release; if it cannot complete, keep that version for diagnosis rather than deleting its durable records or continuing with an older binary. +Replacement resume and checkpoint files use an independent on-disk schema. A newer reader rejects a future schema rather than continuing with data it cannot interpret, while an older binary cannot safely enforce the new generation fence. Do not roll a cluster back after a replacement generation has started; complete that recovery with the current-or-newer release, and if it cannot complete, keep that version for diagnosis rather than deleting its durable records or continuing with an older binary. ## Reading Replication Repair -`metrics.replication_repair`, `metrics.current_cycle_replication_repair`, and -`metrics.last_cycle_replication_repair` split scanner-discovered replication -repair work by source and repair kind. - -Each entry has the same `checked`, `queued`, `executed`, `failed`, `skipped`, -and `missed` counters used by `source_work`, plus: +`metrics.replication_repair`, `metrics.current_cycle_replication_repair`, and `metrics.last_cycle_replication_repair` split scanner-discovered replication repair work by source and repair kind. Each entry has the same `checked`, `queued`, `executed`, `failed`, `skipped`, and `missed` counters used by `source_work`, plus: | Field | Meaning | |---|---| @@ -411,39 +344,21 @@ and `missed` counters used by `source_work`, plus: | `scanner_role` | `repair_admission` means scanner found work and attempted to admit it to a worker queue. `boundary_signal` means scanner is reporting state owned by another runtime. | | `execution_owner` | `bucket_replication_queue` for bucket replication repair execution, or `site_replication_runtime` for site replication resync execution. | -For bucket replication, `queued` means scanner-discovered repair was admitted -to the replication queue, `missed` means the queue or worker path could not -accept it, and `skipped` means the object did not require a new repair task. - -The site replication kinds keep passive scanner discovery separate from active -resync. Scanner status may report site replication boundary counters, but the -scanner should not be treated as the active site replication resync controller. - -Use this boundary when interpreting replication pressure: +For bucket replication, `queued` means scanner-discovered repair was admitted to the replication queue, `missed` means the queue or worker path could not accept it, and `skipped` means the object did not require a new repair task. The site replication kinds keep passive scanner discovery separate from active resync; the scanner is never the active site replication resync controller. | Scenario | Scanner source | Repair kind | Scanner role | Execution owner | Operational meaning | |---|---|---|---|---|---| | Bucket object, delete-marker, version-purge, or existing-object repair found during a scan | `bucket_replication` | `object`, `delete_marker`, `version_purge`, `existing_object` | `repair_admission` | `bucket_replication_queue` | Scanner found bucket replication repair work and attempted to admit it to the replication queue. | | Peer-originated or passive site replication work is observed while scanning | `site_replication` | `passive_requeue` | `boundary_signal` | `site_replication_runtime` | Scanner is reporting a passive site-replication boundary signal; it is not taking ownership of active site resync. | -| Admin-triggered or runtime-owned site resync activity is visible in scanner metrics | `site_replication` | `active_resync` | `boundary_signal` | `site_replication_runtime` | Treat this as a boundary/status signal owned by the site replication runtime, not as scanner-controlled repair execution. | +| Admin-triggered or runtime-owned site resync activity is visible in scanner metrics | `site_replication` | `active_resync` | `boundary_signal` | `site_replication_runtime` | A boundary/status signal owned by the site replication runtime, not scanner-controlled repair execution. | -If `site_replication` counters grow while bucket replication counters stay -flat, investigate site replication status and resync state before tuning -scanner pacing. If `bucket_replication` `missed` grows, investigate the bucket -replication worker queue or target health before changing scanner cycle -settings. +If `site_replication` counters grow while bucket replication counters stay flat, investigate site replication status and resync state before tuning scanner pacing. If `bucket_replication` `missed` grows, investigate the bucket replication worker queue or target health before changing scanner cycle settings. ## Reading Maintenance Control -`metrics.maintenance_control` derives a source-level control snapshot from -scanner pacing, partial-cycle state, source work, and lifecycle transition -queue state. It does not change scanner scheduling by itself; it explains why a -source is moving, deferred, or blocked. When no scan cycle is currently active, -source-work controls use the last completed cycle so recently missed work stays -visible between scanner passes. +`metrics.maintenance_control` derives a source-level control snapshot from scanner pacing, partial-cycle state, source work, and lifecycle transition queue state. It does not change scanner scheduling; it explains why a source is moving, deferred, or blocked. When no scan cycle is active, source-work controls use the last completed cycle so recently missed work stays visible between passes. -`metrics.maintenance_control.primary_control` summarizes the highest-priority -source state: +`metrics.maintenance_control.primary_control`: | Value | Meaning | |---|---| @@ -453,49 +368,35 @@ source state: | `pacing_pressure` | No source-specific state dominated, but scanner pacing pressure is still visible. | | `none` | No source-level maintenance control pressure was observed. | -Each `metrics.maintenance_control.sources[]` entry has: +Each `metrics.maintenance_control.sources[]` entry: | Field | Meaning | |---|---| -| `source` | Scanner source such as `usage`, `lifecycle`, `bucket_replication`, `site_replication`, `heal`, `bitrot`, or `alerts`. | +| `source` | `usage`, `lifecycle`, `bucket_replication`, `site_replication`, `heal`, `bitrot`, or `alerts`. | | `state` | `idle`, `active`, `deferred`, or `blocked`. | -| `reason` | Derived reason such as `active_work`, `queued_work`, `partial_cycle`, `missed_work`, `expiry_queue_backlog`, `transition_failed`, `transition_compensation_backlog`, `transition_queue_backlog`, or `transition_queue_full`. | +| `reason` | `active_work`, `queued_work`, `partial_cycle`, `missed_work`, `expiry_queue_backlog`, `transition_failed`, `transition_compensation_backlog`, `transition_queue_backlog`, or `transition_queue_full`. | | `backlog` | Current source-level backlog estimate from queued or missed work. | -| `current_checked` | Current-cycle checked work for this source, or the last completed cycle when no scan cycle is active. | -| `current_queued` | Current-cycle queued work for this source, or the last completed cycle when no scan cycle is active. | -| `current_missed` | Current-cycle work that could not be admitted, or the last completed cycle when no scan cycle is active. | -| `lifetime_missed` | Lifetime missed work counter for context. | +| `current_checked` / `current_queued` / `current_missed` | Current-cycle counters for this source, or the last completed cycle when no scan cycle is active. | +| `lifetime_missed` | Lifetime missed work counter. | | `partial_cycles` | Partial cycles attributed to this source. | -Use this snapshot before changing scanner controls. For example, -`blocked_source` with `lifecycle/missed_work` points at downstream lifecycle -admission, while `deferred_source` with `usage/partial_cycle` points at scanner -cycle budgets. `lifecycle/expiry_queue_backlog` means scanner-driven expiry or -delete work is still queued or active in the expiry worker pool. -`lifecycle/transition_failed` means transition worker execution failed during -the current or last completed scan cycle, while -`lifecycle/transition_compensation_backlog` means transition compensation is -still pending or running after queue backpressure. +Read this snapshot before changing scanner controls: `blocked_source` with `lifecycle/missed_work` points at downstream lifecycle admission, `deferred_source` with `usage/partial_cycle` points at scanner cycle budgets, `lifecycle/expiry_queue_backlog` means expiry or delete work is still queued or active in the expiry worker pool, `lifecycle/transition_failed` means transition worker execution failed during the current or last completed cycle, and `lifecycle/transition_compensation_backlog` means transition compensation is still pending or running after queue backpressure. -`metrics.lifecycle_expiry` exposes the expiry/delete worker queue observed by -scanner-driven lifecycle work: +`metrics.lifecycle_expiry` exposes the expiry/delete worker queue: | Field | Meaning | |---|---| | `current_queue_capacity` | Effective expiry worker queue capacity for this node. | -| `current_queued` | Expiry/delete tasks currently waiting in the worker queue. | -| `current_active` | Expiry/delete tasks currently running in a worker. | +| `current_queued` | Expiry/delete tasks waiting in the worker queue. | +| `current_active` | Expiry/delete tasks currently running. | | `current_workers` | Configured expiry worker count. | -| `queue_missed` | Expiry/delete tasks that could not be queued because no worker channel was available or the queue was closed. | +| `queue_missed` | Tasks that could not be queued because no worker channel was available or the queue was closed. | | `scanner_queued` | Scanner-discovered expiry/delete object versions admitted to the expiry queue. | | `scanner_missed` | Scanner-discovered expiry/delete object versions that could not be admitted. | ## Reading Distributed Metrics -`/rustfs/admin/v3/scanner/status` and `/rustfs/admin/v3/metrics` report the -node that handles the HTTP request. The metrics endpoint does not fan out to -peer nodes. In distributed deployments, query every node explicitly and keep -`by-host=true` enabled so each response includes that node's host view: +`/rustfs/admin/v3/scanner/status` and `/rustfs/admin/v3/metrics` report the node that handles the HTTP request; the metrics endpoint does not fan out to peers. In distributed deployments, query every node explicitly and keep `by-host=true` so each response includes that node's host view: ```bash for endpoint in http://node-a:9000 http://node-b:9000 http://node-c:9000; do @@ -512,19 +413,11 @@ for endpoint in http://node-a:9000 http://node-b:9000 http://node-c:9000; do done ``` -The `aggregated.scanner` payload preserves the same scanner progress, -checkpoint, pacing, source work, maintenance control, lifecycle expiry, and -lifecycle transition fields used by the local scanner status, but only for the -node that returned the response. The `by_host.*.scanner` payload keeps that -node's host view. -Compare the per-node artifacts externally to find old active paths, partial -checkpoints, pacing pressure, source-level control pressure, or downstream -queue admission problems across the deployment. +The `aggregated.scanner` payload preserves the same scanner progress, checkpoint, pacing, source work, maintenance control, lifecycle expiry, and lifecycle transition fields used by the local scanner status, but only for the responding node; `by_host.*.scanner` keeps that node's host view. Compare the per-node artifacts externally to find old active paths, partial checkpoints, pacing pressure, source-level control pressure, or downstream queue admission problems across the deployment. ## Reading Lifecycle Transition Status -`metrics.lifecycle_transition` focuses on scanner-driven lifecycle transition -work: +`metrics.lifecycle_transition`: | Field | Meaning | |---|---| @@ -542,43 +435,28 @@ work: | `completed` | Transition worker completions. | | `failed` | Transition worker failures. | -When `scanner_missed` or `queue_full` rises, scanner lifecycle work is finding -transition candidates faster than the transition queue can accept them. That is -a downstream transition pressure signal, not just a scanner walk pressure signal. +When `scanner_missed` or `queue_full` rises, scanner lifecycle work is finding transition candidates faster than the transition queue accepts them: a downstream transition pressure signal, not just a scanner walk pressure signal. ## Tuning Workflow -For symptoms where a mostly idle single-node, single-disk deployment has -sustained CPU usage while the scanner is enabled: +For a mostly idle single-node, single-disk deployment with sustained CPU usage while the scanner is enabled: 1. Read `/v3/scanner/status`. 2. Check `metrics.pacing_pressure.primary_pressure`. -3. Check `metrics.maintenance_control.primary_control` and source entries - before changing runtime controls. -4. Check `runtime_config.delay`, `runtime_config.max_wait_seconds`, and - `runtime_config.cycle_interval_seconds` to confirm the active values and - their sources. -5. Check `metrics.current_cycle_objects_scanned`, - `metrics.current_cycle_directories_scanned`, and active paths to confirm the - scanner is the active work. -6. If `primary_pressure` is `throttle_pause` and pause ratios are low, raise - `scanner.delay` first. +3. Check `metrics.maintenance_control.primary_control` and source entries before changing runtime controls. +4. Check `runtime_config.delay`, `runtime_config.max_wait_seconds`, and `runtime_config.cycle_interval_seconds` to confirm the active values and their sources. +5. Check `metrics.current_cycle_objects_scanned`, `metrics.current_cycle_directories_scanned`, and active paths to confirm the scanner is the active work. +6. If `primary_pressure` is `throttle_pause` and pause ratios are low, raise `scanner.delay` first. 7. If individual sleeps are too short, raise `scanner.max_wait`. 8. If each scan cycle finishes but starts too often, raise `scanner.cycle`. -9. If scans must be broken into bounded chunks, set one of the cycle budgets: - `scanner.cycle_max_duration`, `scanner.cycle_max_objects`, or - `scanner.cycle_max_directories`. -10. Recheck `pacing_pressure`, `maintenance_control`, source work, and - lifecycle transition status after one or more scanner cycles. +9. If scans must be broken into bounded chunks, set one of `scanner.cycle_max_duration`, `scanner.cycle_max_objects`, or `scanner.cycle_max_directories`. +10. Recheck `pacing_pressure`, `maintenance_control`, source work, and lifecycle transition status after one or more scanner cycles. -Do not rely only on a longer cycle interval if lifecycle, replication, heal, or -bitrot work must keep moving. Use source work and transition status to confirm -that background maintenance is still making progress. +Do not rely only on a longer cycle interval if lifecycle, replication, heal, or bitrot work must keep moving; use source work and transition status to confirm that background maintenance still progresses. ## Helm -The Helm chart exposes the scanner environment variables under -`config.rustfs.scanner`. Example: +The Helm chart exposes the scanner environment variables under `config.rustfs.scanner` (`helm/rustfs/values.yaml`): ```yaml config: @@ -596,5 +474,4 @@ config: bitrot_cycle_secs: "2592000" ``` -Use `extraEnv` for experimental or unrelated environment variables that are not -represented by chart values. +Use `extraEnv` for environment variables that are not represented by chart values, including every heal knob above. diff --git a/docs/operations/sftp.md b/docs/operations/sftp.md index 97821f19e..ac82bdf36 100644 --- a/docs/operations/sftp.md +++ b/docs/operations/sftp.md @@ -1,15 +1,12 @@ # SFTP Operations Guide -The guide covers enabling and operating the SFTP server in RustFS. -It is written for operators who need to expose buckets over SFTP, manage host -keys across platforms, size the server for large transfers, and diagnose -session cleanup behaviour. +**Use this when:** enabling the SFTP front end, managing host keys across platforms, sizing it for large transfers, granting IAM permissions for SFTP users, or diagnosing session cleanup and orphaned multipart uploads. -## Enabling SFTP: Recommended Configuration +**Source of truth:** `crates/config/src/constants/protocols.rs` (`ENV_SFTP_*`, `DEFAULT_SFTP_*`), `crates/protocols/src/sftp/constants.rs` (session, keepalive, handle, and listing limits), `crates/protocols/src/sftp/driver.rs` (abort permit pool), `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs` (`DEFAULT_STALE_UPLOADS_EXPIRY`, `DEFAULT_STALE_UPLOADS_CLEANUP_INTERVAL`). -Two settings are required: the enable flag and the host-key directory. The -listen address has a default of `0.0.0.0:2222` and setting it explicitly is -recommended practice. +## Enabling SFTP + +Two settings are required: the enable flag and the host-key directory. Setting the listen address explicitly is recommended even though it has a default. ```bash RUSTFS_SFTP_ENABLE=true @@ -17,38 +14,24 @@ RUSTFS_SFTP_ADDRESS=0.0.0.0:2222 RUSTFS_SFTP_HOST_KEY_DIR=/etc/rustfs/sftp-keys ``` -- `RUSTFS_SFTP_ENABLE` starts the SFTP listener at server startup. Off by - default. -- `RUSTFS_SFTP_ADDRESS` is the listen address and port. `0.0.0.0` accepts - connections on every interface. Port `2222` avoids the privileged port 22. -- `RUSTFS_SFTP_HOST_KEY_DIR` is the directory the SSH host keys are loaded - from. No default, and startup fails when SFTP is enabled without it. +| Variable | Role | +| --- | --- | +| `RUSTFS_SFTP_ENABLE` | Starts the SFTP listener at server startup. Off by default. | +| `RUSTFS_SFTP_ADDRESS` | Listen address and port. `0.0.0.0` accepts connections on every interface; port `2222` avoids the privileged port 22. | +| `RUSTFS_SFTP_HOST_KEY_DIR` | Directory the SSH host keys are loaded from. No default; startup fails when SFTP is enabled without it. | -Generate a host key before first start. On Unix, `ssh-keygen` also writes a -world-readable `.pub` file into the directory, and the server requires every -file it considers in the host-key directory (regular, non-empty, at most -1 MiB) to be owner-only, so restrict or remove it: +Generate a host key before first start. On Unix, `ssh-keygen` also writes a world-readable `.pub` file into the directory, and the server requires every file it considers in the host-key directory (regular, non-empty, at most 1 MiB) to be owner-only, so restrict or remove it. The server does not read the `.pub` file. Keys must be unencrypted (no passphrase). ```bash ssh-keygen -t ed25519 -f /etc/rustfs/sftp-keys/ssh_host_ed25519_key -N "" chmod 600 /etc/rustfs/sftp-keys/ssh_host_ed25519_key.pub ``` -The private key itself is already written with owner-only permissions. The -server does not read the `.pub` file, so it can be removed instead. Keys must -be unencrypted (no passphrase). - -On success the server logs `SFTP server listening` with the bound address. A -port of `0` in `RUSTFS_SFTP_ADDRESS` is resolved to a free port at startup -and the resolved port appears in that log line. - -Every other setting has a tested default and should be left alone unless a -section below gives a concrete reason to change it. +On success the server logs `SFTP server listening` with the bound address. A port of `0` in `RUSTFS_SFTP_ADDRESS` is resolved to a free port at startup and the resolved port appears in that log line. Every other setting has a tested default and should be left alone unless a section below gives a concrete reason to change it. ## Path Model -The SFTP root directory is the account's bucket list. The first path -component names the bucket and the remainder is the object key: +The SFTP root directory is the account's bucket list. The first path component names the bucket and the remainder is the object key: ```text /reports/2026/q1.pdf @@ -57,153 +40,78 @@ bucket: reports object key: 2026/q1.pdf ``` -Files cannot be created at the root level. Creating or removing a top-level -directory creates or removes a bucket. The root listing shows at most 10000 -buckets and logs a warning when truncated. +Files cannot be created at the root level. Creating or removing a top-level directory creates or removes a bucket. The root listing shows at most 10000 buckets (`ROOT_LISTING_MAX_ENTRIES`) and logs `root READDIR truncated` when cut off. ## Host Keys -`RUSTFS_SFTP_HOST_KEY_DIR` must name an existing directory containing at -least one decodable private key. Startup fails otherwise. There is no -generated fallback key. +`RUSTFS_SFTP_HOST_KEY_DIR` must name an existing directory containing at least one decodable private key. Startup fails otherwise; there is no generated fallback key. -- Any private key that russh can decode is accepted. Ed25519, ECDSA, and RSA - are the expected formats. Passphrase-protected keys cannot be decoded and - do not count. Keys are offered to clients in the order Ed25519, ECDSA, - RSA, then anything else. Multiple keys of one algorithm all load, so during - key rotation clients can be offered the new key as soon as it is added, - not when the old one is removed. -- Empty files and files larger than 1 MiB are skipped entirely. -- On Unix, every other regular file in the directory must have no group or - other permission bits set (owner-only, for example mode `0600`, `0400`, or - `0700`). The check covers non-key files too: a world-readable README or - `.pub` file fails startup with `host key file has insecure permissions`. - Keep only owner-only files in the directory. -- On Windows there is no permission-bit check. The server logs a one-time - warning at startup whose alertable first sentence reads exactly `SFTP host - key file permission enforcement is not active on Windows`. Restrict the - NTFS ACL on the host-key directory to the running rustfs service account, - `NT AUTHORITY\SYSTEM`, and `BUILTIN\Administrators`. Default ProgramData - inheritance grants `BUILTIN\Users` read access. Remove that grant on the - host-key directory. -- Targets that are neither Unix nor Windows do not support SFTP and fail - startup. - -Host keys can be hot-reloaded without a restart by setting -`RUSTFS_SFTP_HOST_KEY_RELOAD_ENABLE=true`. The directory is rescanned every -`RUSTFS_SFTP_HOST_KEY_RELOAD_INTERVAL` seconds (default 30, silently raised -to 5 if set lower). A failed rescan, including a permission violation that -would be fatal at startup, keeps the previous keys and logs `SFTP host key -reload failed; keeping previous keys`. Reloaded keys affect new connections -only. +| Rule | Detail | +| --- | --- | +| Accepted keys | Any private key russh can decode: Ed25519, ECDSA, RSA. Passphrase-protected keys cannot be decoded and do not count. | +| Offer order | Ed25519, ECDSA, RSA, then anything else. Multiple keys of one algorithm all load, so during rotation clients see the new key as soon as it is added, not when the old one is removed. | +| Skipped files | Empty files and files larger than 1 MiB. | +| Unix permissions | Every other regular file in the directory must have no group or other permission bits (`0600`, `0400`, `0700`). Non-key files count too: a world-readable README or `.pub` file fails startup with `host key file has insecure permissions`. | +| Windows | No permission-bit check. A one-time startup warning whose first sentence reads exactly `SFTP host key file permission enforcement is not active on Windows` is logged. Restrict the NTFS ACL on the directory to the rustfs service account, `NT AUTHORITY\SYSTEM`, and `BUILTIN\Administrators`; default ProgramData inheritance grants `BUILTIN\Users` read access, so remove that grant. | +| Other targets | Neither Unix nor Windows: SFTP is unsupported and startup fails. | +| Hot reload | `RUSTFS_SFTP_HOST_KEY_RELOAD_ENABLE=true` rescans every `RUSTFS_SFTP_HOST_KEY_RELOAD_INTERVAL` seconds (`DEFAULT_SFTP_HOST_KEY_RELOAD_INTERVAL`, 30; silently raised to 5 if lower). A failed rescan, including a permission violation that would be fatal at startup, keeps the previous keys and logs `SFTP host key reload failed; keeping previous keys`. Reloaded keys affect new connections only. | ## Configuration Reference -Changing values beyond the recommended section is rarely necessary. The -defaults are the configuration the test suites and stress runs exercise. -Tuning values interact (for example part size multiplies against the handle -cap in worst-case memory), so change them deliberately and one at a time. +Changing values beyond the recommended section is rarely necessary. The defaults are the configuration the test suites and stress runs exercise. Tuning values interact (part size multiplies against the handle cap in worst-case memory), so change them deliberately and one at a time. Invalid values fall into four classes: -1. Fatal at startup. The server refuses to start and names the problem. -2. Warn and use the default. The server starts and logs a warning naming - the variable, the rejected value, and the bounds. -3. Silently clamped. The reload interval is the one variable raised to its - floor without a warning. -4. Silent fallback. Non-numeric text in any numeric variable behaves as if - the variable were unset. Invalid boolean text logs a one-time warning - and uses the default. +| Class | Behavior | +| --- | --- | +| Fatal at startup | The server refuses to start and names the problem. | +| Warn and use the default | The server starts and logs a warning naming the variable, the rejected value, and the bounds. | +| Silently clamped | Only the reload interval is raised to its floor without a warning. | +| Silent fallback | Non-numeric text in any numeric variable behaves as if the variable were unset. Invalid boolean text logs a one-time warning and uses the default. | -| Variable | Default | Valid values | On invalid | +| Variable | Default (constant) | Valid values | On invalid | | -------- | ------- | ------------ | ---------- | | `RUSTFS_SFTP_ENABLE` | `false` | boolean | warn, default | -| `RUSTFS_SFTP_ADDRESS` | `0.0.0.0:2222` | see binding note below | fatal | -| `RUSTFS_SFTP_HOST_KEY_DIR` | none | existing directory, required | fatal | -| `RUSTFS_SFTP_IDLE_TIMEOUT` | `600` | seconds, greater than zero | fatal | -| `RUSTFS_SFTP_PART_SIZE` | `16777216` | bytes, 5 MiB to 5 GiB | fatal | -| `RUSTFS_SFTP_READ_ONLY` | `false` | boolean | warn, default | +| `RUSTFS_SFTP_ADDRESS` | `0.0.0.0:2222` (`DEFAULT_SFTP_ADDRESS`) | see binding note below | fatal | +| `RUSTFS_SFTP_HOST_KEY_DIR` | none (`DEFAULT_SFTP_HOST_KEY_DIR`) | existing directory, required | fatal | +| `RUSTFS_SFTP_IDLE_TIMEOUT` | `600` (`DEFAULT_SFTP_IDLE_TIMEOUT`) | seconds, greater than zero | fatal | +| `RUSTFS_SFTP_PART_SIZE` | `16777216` (`DEFAULT_SFTP_PART_SIZE`) | bytes, 5 MiB to 5 GiB | fatal | +| `RUSTFS_SFTP_READ_ONLY` | `false` (`DEFAULT_SFTP_READ_ONLY`) | boolean | warn, default | | `RUSTFS_SFTP_BANNER` | `SSH-2.0-RustFS` | must start with `SSH-2.0-` | fatal | -| `RUSTFS_SFTP_HANDLES_PER_SESSION` | `64` | `8` to `1024` | warn, default | +| `RUSTFS_SFTP_HANDLES_PER_SESSION` | `64` (`DEFAULT_HANDLES_PER_SESSION`) | `8` to `1024` (`HANDLES_PER_SESSION_MIN`/`_MAX`) | warn, default | | `RUSTFS_SFTP_BACKEND_OP_TIMEOUT_SECS` | `60` | `5` to `600` | warn, default | | `RUSTFS_SFTP_READ_CACHE_WINDOW_BYTES` | `4194304` | `0` disables, else 256 KiB to 64 MiB | warn, default | | `RUSTFS_SFTP_READ_CACHE_TOTAL_MEM_BYTES` | `268435456` | at least 16 MiB | warn, default | -| `RUSTFS_SFTP_HOST_KEY_RELOAD_ENABLE` | `false` | boolean | warn, default | -| `RUSTFS_SFTP_HOST_KEY_RELOAD_INTERVAL` | `30` | seconds, minimum 5 | clamped to 5 | +| `RUSTFS_SFTP_HOST_KEY_RELOAD_ENABLE` | `false` (`DEFAULT_SFTP_HOST_KEY_RELOAD_ENABLE`) | boolean | warn, default | +| `RUSTFS_SFTP_HOST_KEY_RELOAD_INTERVAL` | `30` (`DEFAULT_SFTP_HOST_KEY_RELOAD_INTERVAL`) | seconds, minimum 5 | clamped to 5 | Notes on individual variables: -- `RUSTFS_SFTP_ADDRESS`: the host part must be a wildcard (`0.0.0.0` or - `[::]`) or an address or hostname assigned to the host, otherwise startup - fails. Use `[::]:2222` to listen on IPv6, which on Linux usually accepts - IPv4 as well via dual-stack. Port `0` auto-assigns a free port. -- `RUSTFS_SFTP_BANNER` is the SSH protocol identification string sent on - connect, not a free-text login banner. Values that do not start with - `SSH-2.0-` fail startup. -- `RUSTFS_SFTP_IDLE_TIMEOUT` cannot be set to `0` to disable idle - disconnects. See the sessions section for what closes idle sessions. -- `RUSTFS_SFTP_PART_SIZE` bounds follow S3 multipart limits. See the large - files section before changing it. -- `RUSTFS_SFTP_READ_CACHE_WINDOW_BYTES` set to exactly `0` disables read - caching, which turns every client read request into one backend call. -- Worst-case buffered write memory per session is the handle cap times the - part size: 64 handles at 16 MiB is 1 GiB per session. The server imposes - no limit on concurrent sessions, so total worst case is that figure - times however many clients connect. Only the read cache has a global - cap. Enforce connection limits externally if that matters. +- `RUSTFS_SFTP_ADDRESS`: the host part must be a wildcard (`0.0.0.0` or `[::]`) or an address or hostname assigned to the host, otherwise startup fails. `[::]:2222` listens on IPv6, which on Linux usually accepts IPv4 as well via dual-stack. Port `0` auto-assigns a free port. +- `RUSTFS_SFTP_BANNER` is the SSH protocol identification string sent on connect, not a free-text login banner. +- `RUSTFS_SFTP_IDLE_TIMEOUT` cannot be `0`; see the sessions section for what closes idle sessions. +- `RUSTFS_SFTP_PART_SIZE` bounds follow S3 multipart limits; see the large files section before changing it. +- `RUSTFS_SFTP_READ_CACHE_WINDOW_BYTES=0` disables read caching, turning every client read request into one backend call. +- Worst-case buffered write memory per session is the handle cap times the part size: 64 handles at 16 MiB is 1 GiB per session. The server imposes no limit on concurrent sessions, so the total worst case is that figure times the number of connected clients. Only the read cache has a global cap. Enforce connection limits externally if that matters. ## Sessions and Cleanup -Several mechanisms close sessions: +| Mechanism | Behavior | +| --- | --- | +| SSH keepalive | Sent every 15 seconds (`KEEPALIVE_INTERVAL_SECS`); the connection is closed after 3 consecutive unanswered keepalives (`KEEPALIVE_MAX`), about 60 seconds after a client stops responding. Not configurable. Cleans up clients that vanish without closing TCP. | +| `RUSTFS_SFTP_IDLE_TIMEOUT` | SSH inactivity timeout (default 600 seconds). Keepalive replies count as SSH traffic and reset it, so a client that answers keepalives is never disconnected by it. | +| Session watchdog (all platforms) | Closes sessions silent at the SFTP request layer for 30 minutes; keepalives do not count. This is what ends a healthy but idle session. Linux logs `wedge watchdog cancelling session` with reason `fallback_silence`; other platforms log `fallback watchdog cancelling session`. | +| Linux TCP-state probe | A socket in `CLOSE_WAIT` across two consecutive 15-second checks while the SFTP layer has been silent for 30 seconds is cancelled, typically 45 to 60 seconds after the client vanished. Two consecutive failed TCP-state probes are treated the same way, so a container that blocks `/proc/net/tcp` can see sessions cancelled on that schedule. The `reason` field names the trigger. | +| Socket dup failure | If duplicating the connection socket fails at accept time (rare, usually file-descriptor exhaustion) the session runs with no watchdog; only keepalive and idle mechanisms apply. Log line: `wedge watchdog: dup_socket failed`. | +| Server shutdown | Cancels every live session immediately; clients are disconnected mid-transfer. The server then waits up to 30 seconds (`SHUTDOWN_DRAIN_TIMEOUT_SECS`) for session cleanup, including aborting in-flight multipart uploads, before remaining tasks are dropped. | -- The server sends an SSH keepalive every 15 seconds and closes the - connection after 3 consecutive unanswered keepalives, about 60 seconds - after a client stops responding. Not configurable. The keepalive check - cleans up clients that vanish without closing TCP. -- `RUSTFS_SFTP_IDLE_TIMEOUT` (default 600 seconds) sets the SSH inactivity - timeout, but keepalive replies count as SSH traffic and reset it, so a - client that answers keepalives is never disconnected by it. -- The session watchdog closes sessions that are silent at the SFTP request - layer for 30 minutes, on every platform. Keepalives do not count as - SFTP-layer activity. The watchdog is the mechanism that ends a healthy - but idle session. On Linux the close logs `wedge watchdog cancelling - session` with reason `fallback_silence`. On other platforms it logs - `fallback watchdog cancelling session`. -- On Linux the watchdog additionally reads kernel TCP state for fast - cleanup of wedged sessions: a socket sitting in `CLOSE_WAIT` across two - consecutive 15-second checks while the SFTP layer has been silent for 30 - seconds is cancelled, typically 45 to 60 seconds after the client - vanished. Two consecutive failed TCP-state probes are treated the same - way, so a container that blocks `/proc/net/tcp` can see sessions - cancelled on that schedule. The `reason` field of the log line names the - trigger. -- If duplicating the connection socket fails at accept time (rare, usually - file-descriptor exhaustion) the session runs with no watchdog at all and - only the keepalive and idle mechanisms apply. Log line: `wedge watchdog: - dup_socket failed`. -- Server shutdown cancels every live session immediately. Clients are - disconnected mid-transfer. The server then waits up to 30 seconds for - session cleanup, including aborting in-flight multipart uploads, before - remaining tasks are dropped. - -A session ended by any cancellation path logs `SFTP session cancelled -(watchdog or server shutdown)`. +A session ended by any cancellation path logs `SFTP session cancelled (watchdog or server shutdown)`. ## Authentication and Authorization -Authentication is by password only, verified against RustFS IAM users. -Public-key authentication is rejected. Anonymous access is not available. -Failed logins are rejected without delay and there is no lockout, so apply -rate limiting externally when the listener is exposed to untrusted -networks. Accepted logins log `SFTP auth accepted` at info level, rejections -log `SFTP auth rejected` at warn level. +Authentication is by password only, verified against RustFS IAM users. Public-key authentication is rejected. Anonymous access is not available. Failed logins are rejected without delay and there is no lockout, so apply rate limiting externally when the listener is exposed to untrusted networks. Accepted logins log `SFTP auth accepted` at info level, rejections log `SFTP auth rejected` at warn level. -Every SFTP operation is authorized against IAM policy before it reaches -storage. Policy condition keys (for example `aws:SourceIp`) are not -evaluated on the SFTP path. Only unconditional Allow and Deny statements -take effect. - -The S3 actions a user needs: +Every SFTP operation is authorized against IAM policy before it reaches storage. Policy condition keys (for example `aws:SourceIp`) are not evaluated on the SFTP path; only unconditional Allow and Deny statements take effect. | SFTP activity | Required S3 actions | | ------------- | ------------------- | @@ -215,66 +123,38 @@ The S3 actions a user needs: | Stat a bucket | `s3:ListBucket` | | Delete a file | `s3:DeleteObject` | | Rename (implemented as copy then delete) | `s3:GetObject` on the source, `s3:PutObject` on the destination, `s3:DeleteObject` on the source | -| Create or remove a top-level directory | `s3:CreateBucket` or `s3:DeleteBucket`, removal also needs `s3:ListBucket` for the emptiness check | +| Create or remove a top-level directory | `s3:CreateBucket` or `s3:DeleteBucket`; removal also needs `s3:ListBucket` for the emptiness check | | Upload cleanup on disconnect | `s3:AbortMultipartUpload` (see the large files section) | -Upload-only users also need `s3:GetObject`, because SFTP clients stat files -as part of normal transfers. +Upload-only users also need `s3:GetObject`, because SFTP clients stat files as part of normal transfers. -With `RUSTFS_SFTP_READ_ONLY=true` the server rejects all mutating packets -at the protocol layer: opening a file for write, `WRITE`, `REMOVE`, -`MKDIR`, `RMDIR`, `RENAME`, `SETSTAT`, and `FSETSTAT`. +With `RUSTFS_SFTP_READ_ONLY=true` the server rejects all mutating packets at the protocol layer: opening a file for write, `WRITE`, `REMOVE`, `MKDIR`, `RMDIR`, `RENAME`, `SETSTAT`, and `FSETSTAT`. ## Client Compatibility -The server speaks SFTP version 3 and maps onto object storage. Differences -from a filesystem-backed SFTP server: +The server speaks SFTP version 3 (`SFTP_VERSION`) and maps onto object storage. Differences from a filesystem-backed SFTP server: -- Uploads must be a single sequential stream from offset zero. Transfer - resume, append mode, in-place edits (read-write opens), and segmented or - multi-connection uploads of one file are rejected. Configure clients for - whole-file, single-connection transfers. -- In normal (read-write) mode, `SETSTAT` and `FSETSTAT` are accepted and - ignored: chmod, timestamp preservation, and ownership changes silently - have no effect, and listed permissions are fixed server-generated values. -- Symlink operations (`SYMLINK`, `READLINK`) are not supported and return - an unsupported-operation error. Object storage has no symlink equivalent. -- Renaming copies the object server-side and then deletes the source, so - large-file renames are slow and not atomic: a failure after the copy can - leave the file at both paths. Renaming a top-level directory (a bucket) - is not supported. +| Behavior | Detail | +| --- | --- | +| Uploads | Must be a single sequential stream from offset zero. Transfer resume, append mode, in-place edits (read-write opens), and segmented or multi-connection uploads of one file are rejected. Configure clients for whole-file, single-connection transfers. | +| `SETSTAT` / `FSETSTAT` | Accepted and ignored in read-write mode: chmod, timestamp preservation, and ownership changes silently have no effect; listed permissions are fixed server-generated values. | +| Symlinks | `SYMLINK` and `READLINK` return an unsupported-operation error. | +| Rename | Server-side copy then delete: slow for large files and not atomic (a failure after the copy can leave the file at both paths). Renaming a top-level directory (a bucket) is not supported. | ## Large Files and Multipart Uploads -An upload smaller than `RUSTFS_SFTP_PART_SIZE` bytes is buffered in memory -and written with a single `PutObject` when the file is closed. At part size -or larger the handle switches to S3 multipart, flushing a part each time a -full part accumulates. +An upload smaller than `RUSTFS_SFTP_PART_SIZE` bytes is buffered in memory and written with a single `PutObject` when the file is closed. At part size or larger the handle switches to S3 multipart, flushing a part each time a full part accumulates. -The server enforces the S3 limit of 10000 parts per upload, so the largest -single upload is part size times 10000: 156.25 GiB at the default 16 MiB -part size. Raise `RUSTFS_SFTP_PART_SIZE` until that product covers the -largest expected file. A larger part size raises per-session memory. A -write that would exceed the cap is rejected with log line `SFTP write would -exceed the S3 multipart parts limit`. The same cap can reject at file -close, logging `SFTP close rejected: trailing part would exceed S3 -multipart parts limit`. +The server enforces the S3 limit of 10000 parts per upload (`S3_MAX_MULTIPART_PARTS`), so the largest single upload is part size times 10000: 156.25 GiB at the default 16 MiB part size. Raise `RUSTFS_SFTP_PART_SIZE` until that product covers the largest expected file; a larger part size raises per-session memory. A write that would exceed the cap logs `SFTP write would exceed the S3 multipart parts limit`; the same cap at file close logs `SFTP close rejected: trailing part would exceed S3 multipart parts limit`. -When a session ends mid-upload the server aborts the in-flight multipart -upload. Four situations leave an orphaned upload behind that the session -itself cannot clean up: +When a session ends mid-upload the server aborts the in-flight multipart upload. Four situations leave an orphaned upload behind that the session itself cannot clean up: -- the user lacks `s3:AbortMultipartUpload`, -- the abort itself fails or times out, -- a burst of simultaneous disconnects exhausts the global abort task pool - (sized at twice the CPU parallelism, between 8 and 128), +- the user lacks `s3:AbortMultipartUpload`; +- the abort itself fails or times out; +- a burst of simultaneous disconnects exhausts the global abort task pool (sized at twice the CPU parallelism, between `ABORT_PERMITS_FLOOR` 8 and `ABORT_PERMITS_CEILING` 128); - the server process is killed outright. -Orphaned uploads are not permanent. RustFS runs a background cleanup that -aborts stale incomplete multipart uploads (by default, uploads older than -24 hours, checked every 6 hours). An `AbortIncompleteMultipartUpload` -bucket lifecycle rule reclaims them as well and gives per-bucket control of -the window. The log lines when a session-drop abort is skipped or fails: +Orphaned uploads are not permanent. The background stale-upload cleanup aborts incomplete multipart uploads older than `DEFAULT_STALE_UPLOADS_EXPIRY` (24 hours), checked every `DEFAULT_STALE_UPLOADS_CLEANUP_INTERVAL` (6 hours). An `AbortIncompleteMultipartUpload` bucket lifecycle rule reclaims them as well and gives per-bucket control of the window. The log lines when a session-drop abort is skipped or fails: ```text skipped abort of orphaned multipart upload on session drop, principal lacks s3:AbortMultipartUpload, bucket lifecycle rules must reclaim parts @@ -283,31 +163,26 @@ failed to abort orphaned multipart upload Drop abort of orphaned multipart upload timed out; bucket lifecycle rule must reclaim parts ``` -Abort failures at file close log with different wording and are retried at -session drop, where the lines above appear. +Abort failures at file close log with different wording and are retried at session drop, where the lines above appear. ## Log Lines Worth Alerting On -Every line below logs at warn level except `SFTP server listening` and -`SFTP auth accepted`, which log at info. The server's default log level is -error, so none of them are visible until the log level is raised to info -(for example `RUSTFS_OBS_LOGGER_LEVEL=info`, or warn to capture alerts -only). +Every line below logs at warn level except `SFTP server listening` and `SFTP auth accepted`, which log at info. The server's default log level is error, so none of them are visible until the log level is raised (for example `RUSTFS_OBS_LOGGER_LEVEL=info`, or `warn` to capture alerts only). | Message | Meaning | | ------- | ------- | -| `SFTP server listening` | Startup complete, address bound | -| `SFTP auth rejected` | Failed login, no built-in lockout exists | -| `SFTP host key file permission enforcement is not active on Windows` | Expected once per start on Windows, verify the ACL guidance once | -| `host key file has insecure permissions` | Fatal at startup on Unix, fix file modes. During hot reload the same text appears inside the reload-failed warning and previous keys stay active | -| `SFTP host key reload failed; keeping previous keys` | Hot reload rescan failed, service unaffected, investigate the directory | -| `wedge watchdog cancelling session` | Linux only. `reason` field: `tcp_state_close_wait_confirmed` or `probe_failed_confirmed` mean a wedged or unprobeable socket, `fallback_silence` means routine 30-minute idle cleanup | -| `fallback watchdog cancelling session` | Non-Linux platforms: session silent at the SFTP layer for 30 minutes | -| `wedge watchdog: dup_socket failed` | Rare, that session runs without any watchdog | -| `SFTP session cancelled (watchdog or server shutdown)` | Session ended by cancellation rather than client close | -| `SFTP write would exceed the S3 multipart parts limit` | Client hit the per-upload size cap, raise the part size if legitimate. Also check the close-time variant below | -| `SFTP close rejected: trailing part would exceed S3 multipart parts limit` | Same cap hit at file close | -| `root READDIR truncated` | A principal can see more than 10000 buckets, listing was cut off | -| `skipped abort of orphaned multipart upload` | Orphaned parts on storage until background cleanup or lifecycle rule reclaims them | -| `abort permit pool exhausted on session drop` | Mass-disconnect burst, orphaned parts on storage until reclaimed | -| `RUSTFS_SFTP_` prefix in a warning | A tuning variable was rejected and its default applied | +| `SFTP server listening` | Startup complete, address bound. | +| `SFTP auth rejected` | Failed login; no built-in lockout exists. | +| `SFTP host key file permission enforcement is not active on Windows` | Expected once per start on Windows; verify the ACL guidance once. | +| `host key file has insecure permissions` | Fatal at startup on Unix; fix file modes. During hot reload the same text appears inside the reload-failed warning and previous keys stay active. | +| `SFTP host key reload failed; keeping previous keys` | Hot reload rescan failed; service unaffected; investigate the directory. | +| `wedge watchdog cancelling session` | Linux only. `reason` `tcp_state_close_wait_confirmed` or `probe_failed_confirmed` mean a wedged or unprobeable socket; `fallback_silence` means routine 30-minute idle cleanup. | +| `fallback watchdog cancelling session` | Non-Linux platforms: session silent at the SFTP layer for 30 minutes. | +| `wedge watchdog: dup_socket failed` | Rare; that session runs without any watchdog. | +| `SFTP session cancelled (watchdog or server shutdown)` | Session ended by cancellation rather than client close. | +| `SFTP write would exceed the S3 multipart parts limit` | Client hit the per-upload size cap; raise the part size if legitimate. | +| `SFTP close rejected: trailing part would exceed S3 multipart parts limit` | Same cap hit at file close. | +| `root READDIR truncated` | A principal can see more than 10000 buckets; listing was cut off. | +| `skipped abort of orphaned multipart upload` | Orphaned parts on storage until background cleanup or a lifecycle rule reclaims them. | +| `abort permit pool exhausted on session drop` | Mass-disconnect burst; orphaned parts on storage until reclaimed. | +| `RUSTFS_SFTP_` prefix in a warning | A tuning variable was rejected and its default applied. | diff --git a/docs/operations/tier-ilm-debugging.md b/docs/operations/tier-ilm-debugging.md index 2dcf1373f..7abe51b08 100644 --- a/docs/operations/tier-ilm-debugging.md +++ b/docs/operations/tier-ilm-debugging.md @@ -1,37 +1,36 @@ # Tier / ILM Transition Debugging Guide -How to debug lifecycle tiering (hot → cold transition) issues: inspecting -`xl.meta`, tracing the versionId sent to the remote tier, and known pitfalls. +**Use this when:** a tiered (transitioned) object returns `NoSuchVersion` or fails to restore, a transition run looks stuck, or you need to inspect `xl.meta` and trace the versionId sent to the remote tier. -> Code map (post `#3929` layout): -> -> | Concern | Location | -> |---------|----------| -> | ILM actions (`transition_object`, `expire_transitioned_object`, `get_transitioned_object_reader`, `gen_transition_objname`) | `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs` | -> | Erasure-set transition/restore entry points | `crates/ecstore/src/set_disk/` and `crates/ecstore/src/store/` | -> | `WarmBackend` trait (put/get/remove/in_use) | `crates/ecstore/src/services/tier/warm_backend.rs` | -> | Per-provider tier backends (S3, MinIO, GCS, Azure, …) | `crates/ecstore/src/services/tier/warm_backend_*.rs` | -> | Remote-tier sweep (`delete_object_from_remote_tier`) | `crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs` | -> | `ObjectInfo` / `TransitionedObject` types | `crates/ecstore/src/object_api/types.rs` | -> | `FileMeta` / `FileInfo` / version metadata | `crates/filemeta/src/` | -> | Dual-key internal metadata helpers (`insert_bytes` / `get_bytes`) | `crates/utils/src/http/metadata_compat.rs` | +**Source of truth:** `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs` (transition/expiry actions), `crates/utils/src/http/metadata_compat.rs` (dual-key helpers), `crates/filemeta/src/filemeta/version.rs` (`SUFFIX_TRANSITIONED_VERSION_ID` read pattern), `crates/filemeta/examples/dump_fileinfo.rs`. + +## Code map + +| Concern | Location | +|---------|----------| +| ILM actions (`transition_object`, `expire_transitioned_object`, `get_transitioned_object_reader`, `gen_transition_objname`) | `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs` | +| Erasure-set transition/restore entry points | `crates/ecstore/src/set_disk/` and `crates/ecstore/src/store/` | +| `WarmBackend` trait (put/get/remove/in_use) | `crates/ecstore/src/services/tier/warm_backend.rs` | +| Per-provider tier backends (S3, MinIO, GCS, Azure, ...) | `crates/ecstore/src/services/tier/warm_backend_*.rs` | +| Remote-tier sweep (`delete_object_from_remote_tier`) | `crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs` | +| Persisted free-version recovery (remote cleanup after local-first expiry) | `crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs` | +| `ObjectInfo` / `TransitionedObject` types | `crates/ecstore/src/object_api/types.rs` | +| `FileMeta` / `FileInfo` / version metadata | `crates/filemeta/src/` | +| Dual-key internal metadata helpers (`insert_bytes` / `get_bytes`) | `crates/utils/src/http/metadata_compat.rs` | ## Metadata key conventions -Internal metadata is stored under **both** `x-rustfs-internal-` and -`x-minio-internal-` for MinIO interoperability. `get_bytes` prefers -the RustFS key and falls back to the MinIO key. +Internal metadata is stored under both `x-rustfs-internal-` and `x-minio-internal-` for MinIO interoperability. `get_bytes` prefers the RustFS key and falls back to the MinIO key. | Suffix | Meaning | |--------|---------| | `transition-status` | `"complete"` when tiered | -| `transitioned-object` | tier key path (stored **without** the tier prefix; `get_dest` adds it) | +| `transitioned-object` | tier key path (stored without the tier prefix; `get_dest` adds it) | | `transitioned-versionID` | S3 version_id returned by tier PUT (16 raw UUID bytes, or absent) | | `transition-tier` | tier name | | `tier-free-versionID` | delete-marker version for free-version sweep | -Reading binary values must reject empty/malformed/nil values (regression -covered in `crates/filemeta/src/filemeta/version.rs` tests): +Reading binary values must reject empty, malformed, and nil values (regression covered in `crates/filemeta/src/filemeta/version.rs` tests): ```rust get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) @@ -40,9 +39,7 @@ get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) // None for: absent key, wrong-length bytes, nil UUID ``` -`transition_version_id == None` means the tier bucket is unversioned; the -GET/DELETE against the tier must then send **no** `versionId` parameter. -A nil UUID (`00000000-…`) sent as `?versionId=` causes `NoSuchVersion`. +`transition_version_id == None` means the tier bucket is unversioned; the GET/DELETE against the tier must then send no `versionId` parameter. A nil UUID (`00000000-...`) sent as `?versionId=` causes `NoSuchVersion`. Do not use `Uuid::from_slice(..).unwrap_or_default()` here: it converts an empty metadata value into `Uuid::nil()`, which is exactly that failure. ## Inspect xl.meta directly @@ -52,106 +49,89 @@ cargo build -p rustfs-filemeta --example dump_fileinfo # Shows: transition_status, transition_tier, transitioned_obj, transition_ver_id ``` -- `transition_ver_id: ` → no versionId will be sent to the tier - (correct for a non-versioned tier bucket). -- `transition_ver_id: ` → that UUID will be sent as `?versionId=`. +| Output | Meaning | +|---|---| +| `transition_ver_id: ` | No versionId will be sent to the tier (correct for a non-versioned tier bucket). | +| `transition_ver_id: ` | That UUID will be sent as `?versionId=`. | -There is one `xl.meta` per erasure shard disk -(`{disk}/{bucket}/{object}/xl.meta`); all shards of a healthy object should -be identical. `dump_versions` (same crate) lists every version in a file. +There is one `xl.meta` per erasure shard disk (`{disk}/{bucket}/{object}/xl.meta`); all shards of a healthy object should be identical. `dump_versions` (same crate) lists every version in a file. ## Trace the versionId at runtime ```bash -RUST_LOG=rustfs_ecstore::bucket::lifecycle=debug rustfs … +RUST_LOG=rustfs_ecstore::bucket::lifecycle=debug rustfs ... ``` -- `fetching transitioned object from tier` — DEBUG, before the tier request. -- `tier GET failed` — ERROR, includes `tier_version_id`. +| Log line | Level | Meaning | +|---|---|---| +| `fetching transitioned object from tier` | DEBUG | Emitted before the tier request. | +| `tier GET failed` | ERROR | Includes `tier_version_id`. | -If both `x-rustfs-internal-transitioned-versionID` and -`x-minio-internal-transitioned-versionID` are the **empty string**, the object -was transitioned to a non-versioned tier bucket and no versionId must be sent. +If both `x-rustfs-internal-transitioned-versionID` and `x-minio-internal-transitioned-versionID` are the empty string, the object was transitioned to a non-versioned tier bucket and no versionId must be sent. ## Manual transition run Manual transition run is an operator trigger for the existing lifecycle transition evaluator. It does not force objects that are not due under the bucket lifecycle rule, and it does not bypass versioning, replication, delete-marker, directory-marker, tier, or in-flight transition checks. -The admin endpoint is: - ```text POST /rustfs/admin/v3/ilm/transition/run?bucket=&prefix=&tier=&dryRun=true&maxObjects=10000&maxDurationSeconds=30 ``` -Only `bucket` is required. `prefix`, `tier`, `dryRun`, `maxObjects`, and `maxDurationSeconds` narrow or bound the run. `maxObjects` defaults to `10000` and is capped at `100000`. `maxDurationSeconds` is optional, capped at `3600`, and enforced as a best-effort budget checked between listed object versions and pages; an in-flight listing call is not cancelled. +| Parameter | Contract | +|---|---| +| `bucket` | Required. | +| `prefix`, `tier`, `dryRun` | Narrow the run. | +| `maxObjects` | Defaults to `10000`, capped at `100000`. | +| `maxDurationSeconds` | Optional, capped at `3600`; a best-effort budget checked between listed object versions and pages. An in-flight listing call is not cancelled. | -The current contract is `enqueue_only`: the response reports what this bounded scan evaluated and enqueued into the in-memory transition queue. `state=completed` means the bounded scan reached the end of its current scope without queue pressure or configured budget truncation. It does not mean every remote tier PUT has completed. `state=partial` means the run stopped early because it hit `maxObjects`, `maxDurationSeconds`, or queue pressure (`skipped_queue_full`, `skipped_queue_closed`, or `skipped_queue_timeout`). +The current contract is `enqueue_only`: the response reports what this bounded scan evaluated and enqueued into the in-memory transition queue. `state=completed` means the bounded scan reached the end of its current scope without queue pressure or budget truncation; it does not mean every remote tier PUT has completed. `state=partial` means the run stopped early on `maxObjects`, `maxDurationSeconds`, or queue pressure (`skipped_queue_full`, `skipped_queue_closed`, or `skipped_queue_timeout`). -`job_id` and `status_endpoint` are currently `null`. There is no durable background job, no restart recovery cursor, no cluster-wide single-flight admission, no status polling endpoint, and no cancel endpoint for this trigger yet. Re-running the command is safe only in the normal lifecycle sense: already in-flight object versions are deduplicated in the local process, but separate nodes do not yet share a durable manual-run admission record. +`job_id` and `status_endpoint` are `null`. There is no durable background job, restart recovery cursor, cluster-wide single-flight admission, status polling endpoint, or cancel endpoint for this trigger. Re-running is safe only in the normal lifecycle sense: in-flight object versions are deduplicated in the local process, but separate nodes do not share a durable manual-run admission record. -Recommended operator flow: +Recommended operator flow (external `rc` CLI): ```bash rc admin ilm transition run local/mybucket --prefix logs/ --tier cold --dry-run --max-objects 1000 --max-duration-seconds 30 rc admin ilm transition run local/mybucket --prefix logs/ --tier cold --max-objects 1000 --max-duration-seconds 30 ``` -Inspect the aggregate counters before widening scope. Full object-key lists are intentionally not returned by the admin response. If `RUSTFS_RPC_SECRET` or other credentials were pasted into an issue, chat, log, or ticket while debugging tiering, rotate them on every node, restart the cluster with the new value, and redact the exposed copy before sharing more diagnostics. +Inspect the aggregate counters before widening scope. Full object-key lists are intentionally not returned. If `RUSTFS_RPC_SECRET` or other credentials were pasted into an issue, chat, log, or ticket while debugging tiering, rotate them on every node, restart the cluster with the new value, and redact the exposed copy before sharing more diagnostics. ## Reconcile an unknown transition upload Historical transition transactions in `upload_outcome_unknown` state can use an explicit two-stage operator workflow when the tier probe is ambiguous and the provider supports exact version deletion. The endpoint refuses transactions that are still inside their ownership window or are in any other state. -First inspect the transaction without changing it: +1. Inspect the transaction without changing it: -```text -GET /rustfs/admin/v3/ilm/transition/reconcile/ -``` + ```text + GET /rustfs/admin/v3/ilm/transition/reconcile/ + ``` -If independent provider evidence identifies the exact remote version to remove, submit that opaque version identifier with explicit confirmation: +2. If independent provider evidence identifies the exact remote version to remove, submit that opaque version identifier with explicit confirmation. This performs only an exact version delete; the response reports whether the transaction journal was still observed afterwards, since background recovery may have finalized the same transaction concurrently: -```json -POST /rustfs/admin/v3/ilm/transition/reconcile/ -{ - "action": "delete_candidate", - "confirm": true, - "remote_version_id": "" -} -``` + ```json + POST /rustfs/admin/v3/ilm/transition/reconcile/ + { + "action": "delete_candidate", + "confirm": true, + "remote_version_id": "" + } + ``` -This operation performs only an exact version delete. Its response reports whether the transaction journal was still observed after the delete; background recovery may have finalized the same transaction concurrently. If the journal remains, inspect the transaction again and finalize it only after the live provider probe proves that the candidate is missing: +3. If the journal remains, inspect again and finalize only after the live provider probe proves the candidate is missing: -```json -POST /rustfs/admin/v3/ilm/transition/reconcile/ -{ - "action": "finalize_missing", - "confirm": true -} -``` + ```json + POST /rustfs/admin/v3/ilm/transition/reconcile/ + { + "action": "finalize_missing", + "confirm": true + } + ``` -`finalize_missing` re-runs the provider probe and fails closed for `unversioned_present`, `versioned_present`, `ambiguous`, `unsupported`, or probe errors. It never accepts an operator assertion in place of a live `missing` result. Providers without an authoritative probe or exact version deletion remain pending; this endpoint does not infer provider capabilities, accept external absence assertions, or select a candidate automatically. +`finalize_missing` re-runs the provider probe and fails closed for `unversioned_present`, `versioned_present`, `ambiguous`, `unsupported`, or probe errors. It never accepts an operator assertion in place of a live `missing` result. Providers without an authoritative probe or exact version deletion remain pending; the endpoint does not infer provider capabilities, accept external absence assertions, or select a candidate automatically. -## Historical fixes (for context, already merged) +## Invariant: local-first expiry ordering -- Expire/GET race (`NoSuchVersion` during expiry of a tiered object): - `expire_transitioned_object` used to delete the remote tier version - **before** local metadata, so a concurrent GET between the two steps read a - stored version_id whose remote version was already gone. Fixed in `#3491`: - local metadata is deleted **first** (making the object unreachable), and - remote-tier cleanup is driven by persisted free-version recovery - (`crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs`). - **Invariant — keep local-first ordering**: never remove a remote tier - version while live local metadata still points at it. Regression test: - `serial_tests::test_expire_transitioned_object_never_races_concurrent_get` - in `crates/scanner/tests/lifecycle_integration_test.rs` (runs in the CI - ILM Integration serial lane) pins both the local-first ordering and the - "concurrent GET never sees `NoSuchVersion`" contract. -- Nil-UUID versionId sent to tier (`NoSuchVersion`): reading code used - `Uuid::from_slice(..).unwrap_or_default()`, converting an empty metadata - value into `Uuid::nil()`. Fixed by the `and_then`/`filter` pattern above. -- `warm_backend_s3sdk` ignored the remote version and range options on - GET/DELETE. -- `copy_object` returned 501 (`NotImplemented`) for tiered objects on - self-copy with `--storage-class`, blocking de-tiering via - `mc cp --storage-class STANDARD obj obj`. Fixed by writing the - tier-fetched `put_object_reader` back through `put_object`. +`expire_transitioned_object` deletes local metadata first (making the object unreachable) and leaves remote-tier cleanup to persisted free-version recovery (`crates/ecstore/src/bucket/lifecycle/tier_free_version_recovery.rs`). Never remove a remote tier version while live local metadata still points at it: doing so lets a concurrent GET read a stored version_id whose remote version is already gone and fail with `NoSuchVersion`. + +Regression test: `serial_tests::test_expire_transitioned_object_never_races_concurrent_get` in `crates/scanner/tests/lifecycle_integration_test.rs` (CI ILM Integration serial lane) pins both the local-first ordering and the "concurrent GET never sees `NoSuchVersion`" contract. diff --git a/docs/operations/two-factor-auth.md b/docs/operations/two-factor-auth.md index 07f12b34b..decd6115d 100644 --- a/docs/operations/two-factor-auth.md +++ b/docs/operations/two-factor-auth.md @@ -1,23 +1,16 @@ # Two-Factor Authentication -> Scope: the self-service account surface (`/rustfs/admin/v3/account/*`), the -> login gate on `AssumeRole`, and the administrative reset -> (`/rustfs/admin/v3/user/mfa`). +**Use this when:** changing anything under `/rustfs/admin/v3/account/*`, `/v3/mfa/challenge`, `/v3/user/mfa`, or the `AssumeRole` login gate, or before "fixing" a 2FA boundary that looks like a gap. This is the design record of what the second factor protects and why. -This document records what the second factor does and does not protect, and why. -The boundaries are deliberate; several of them look like gaps until the -alternative is spelled out. +**Source of truth:** `crates/iam/src/mfa/` (`record.rs` `MAX_FAILED_ATTEMPTS`, `totp.rs` `TOTP_SKEW_STEPS`, `challenge.rs` `CHALLENGE_TTL_SECONDS`, `recovery.rs` `RECOVERY_CODE_COUNT` / `RECOVERY_CODE_ENTROPY_BITS`), `crates/credentials/src/credentials.rs` (root credential `OnceLock`), `crates/iam/src/root_credentials.rs` (`token_signing_key`), `rustfs/src/admin/route_policy.rs` (`CredentialOnly` routes). ## What is protected -TOTP gates **session minting**: the `AssumeRole` call that turns a long-term -credential into a short-lived STS session. That is the only interactive login -RustFS has — the Console holds nothing but an STS session, and obtains it by -signing an `AssumeRole` request with the access key the user typed. +TOTP gates session minting: the `AssumeRole` call that turns a long-term credential into a short-lived STS session. That is the only interactive login RustFS has; the Console holds nothing but an STS session, obtained by signing an `AssumeRole` request with the access key the user typed. When an identity has an active enrollment: -``` +```text access key + secret key │ ▼ @@ -32,268 +25,131 @@ POST / Action=AssumeRole STS credentials, with the claim x-rustfs-mfa-verified: true ``` -Without a `TokenCode`, `AssumeRole` fails with `AccessDenied` and a message -carrying the `MultiFactorAuthRequired` marker. Clients match on that marker to -prompt for a code rather than reporting a failed login — the password *was* -accepted. - -`SerialNumber` and `TokenCode` are `AssumeRole`'s own parameters, so an SDK or a -script authenticates the same way the Console does, with no RustFS-specific -protocol. +Without a `TokenCode`, `AssumeRole` fails with `AccessDenied` and a message carrying the `MultiFactorAuthRequired` marker. Clients match on that marker to prompt for a code rather than reporting a failed login; the password was accepted. `SerialNumber` and `TokenCode` are `AssumeRole`'s own parameters, so an SDK or script authenticates the same way the Console does, with no RustFS-specific protocol. ## What is deliberately not protected -**A request signed directly with a long-term access key is not gated.** This is -the most important boundary in the design, and it is intentional: +| Boundary | Rationale | +| --- | --- | +| A request signed directly with a long-term access key is not gated. | Gating it would break every script, SDK client, and `rc` invocation the moment a human enabled 2FA on their own account, and it would add no protection: whoever holds the secret key already has full access and never needs to mint a session. AWS draws the same line: MFA gates `AssumeRole` and is enforced for API calls through the `aws:MultiFactorAuthPresent` policy condition, not by refusing signed requests. | +| OIDC and Keystone sessions are not gated. | Those identities are authenticated by their provider; a RustFS-side enrollment would not be consulted at login and would give a false impression of protection. `CallerIdentity` reports such sessions as `FederatedIdentity` and refuses enrollment. | +| Service-account credentials cannot manage their parent's factor. | A machine credential must not be able to take over the human identity it was minted from. | -- Gating it would break every script, SDK client and `rc` invocation the moment - a human enabled 2FA on their own account. An operator who turned on a security - feature would discover it by way of a production outage. -- It would add no protection. Whoever holds the secret key already has full - access to everything that identity can reach; they never need to present a - code, because they never need to mint a session. - -This is the same division AWS draws: MFA gates `AssumeRole` and is enforced for -API calls through the `aws:MultiFactorAuthPresent` policy condition, not by -refusing signed requests. - -**Consequence to state plainly:** 2FA raises the cost of a stolen *password*. It -does not contain a stolen *secret key*. Because in RustFS the password **is** the -S3 secret key (see below), those are the same string — so 2FA protects the -console login path against credential reuse and phishing, and nothing more, -until the policy-condition work lands. - -The tracked follow-up is an `aws:MultiFactorAuthPresent` condition key populated -from the `x-rustfs-mfa-verified` session claim, which would let an operator write -a policy that denies administrative actions to a session that presented no -second factor. That is the mechanism that makes 2FA meaningful for API access. - -**OIDC and Keystone sessions are not gated either.** Those identities are -authenticated by their provider; a RustFS-side TOTP enrollment would not be -consulted at login and would give a false impression of protection. MFA for a -federated identity belongs to its IdP. `CallerIdentity` reports such sessions as -`FederatedIdentity` and refuses enrollment. - -**Service-account credentials cannot manage their parent's factor.** A machine -credential must not be able to take over the human identity it was minted from. +Consequence to state plainly: 2FA raises the cost of a stolen password; it does not contain a stolen secret key. Because in RustFS the password is the S3 secret key (below), those are the same string, so 2FA protects the console login path against credential reuse and phishing, and nothing more, until the policy-condition work lands. The tracked follow-up is an `aws:MultiFactorAuthPresent` condition key populated from the `x-rustfs-mfa-verified` session claim, which would let an operator deny administrative actions to a session that presented no second factor. That is the mechanism that makes 2FA meaningful for API access; it is not implemented yet. ## Password reality: there is no password hash -RustFS is an S3 server. SigV4 requires the server to know the secret key itself -in order to recompute a request signature, so secret keys **cannot** be hashed — -not here, and not in any S3-compatible implementation. The "password" a user -types into the Console is their S3 secret key. - -What protects it instead: +RustFS is an S3 server. SigV4 requires the server to know the secret key itself to recompute a request signature, so secret keys cannot be hashed, here or in any S3-compatible implementation. The "password" a user types into the Console is their S3 secret key. | Protection | Mechanism | | --- | --- | -| At rest | `RUSTFS_IAM_MASTER_KEY` + `encrypt_stream_io` (Argon2id → AES-GCM / ChaCha20-Poly1305) | +| At rest | `RUSTFS_IAM_MASTER_KEY` + `encrypt_stream_io` (Argon2id -> AES-GCM / ChaCha20-Poly1305) | | Length floor | `is_secret_key_valid` (`SECRET_KEY_MIN_LEN`) | +| Maximum length | None, deliberately; capping password length is an anti-pattern. | | Rotation | `POST /v3/account/password`, requiring the current secret | | Session cleanup | Every STS session minted from the identity is revoked on rotation | -There is deliberately **no maximum length**. The previous Console capped -passwords at 40 characters, which was a client-side invention with no server -constraint behind it; capping password length is an anti-pattern. - ## At-rest protection is mandatory for TOTP secrets -A TOTP secret is credential-equivalent: anyone holding it can mint valid codes -forever. So enrollment is **refused** when `RUSTFS_IAM_MASTER_KEY` is not -configured, rather than writing the secret in plaintext: +A TOTP secret is credential-equivalent: anyone holding it can mint valid codes forever. Enrollment is therefore refused when `RUSTFS_IAM_MASTER_KEY` is not configured, rather than writing the secret in plaintext: -``` +```text POST /v3/account/mfa/enroll → 501 NotImplemented "two-factor authentication requires RUSTFS_IAM_MASTER_KEY to be configured so the shared secret can be encrypted at rest" ``` -IAM *identities* tolerate a missing master key for backward compatibility with -existing deployments. A new feature has no such history to honour, and a second -factor that can be lifted off a disk is worse than none, because the user -believes they have one. - -`GET /v3/account/mfa` reports `enrollment_available: false` with the reason, so -the Console and `rc` explain the remedy instead of offering a control that fails. +IAM identities tolerate a missing master key for backward compatibility with existing deployments. A new feature has no such history to honour, and a second factor that can be lifted off a disk is worse than none because the user believes they have one. `GET /v3/account/mfa` reports `enrollment_available: false` with the reason, so the Console and `rc` explain the remedy instead of offering a control that fails. ## Root credentials cannot be changed at runtime -The root identity comes from `RUSTFS_ACCESS_KEY` / `RUSTFS_SECRET_KEY` and lands -in a process-wide `OnceLock` (`crates/credentials/src/credentials.rs`). It cannot -be rotated while the server runs, and the account surface reports this as -`credentials_source: "env"` with `mutable.password: false`. +The root identity comes from `RUSTFS_ACCESS_KEY` / `RUSTFS_SECRET_KEY` and lands in a process-wide `OnceLock` (`crates/credentials/src/credentials.rs`). It cannot be rotated while the server runs, and the account surface reports this as `credentials_source: "env"` with `mutable.password: false`. -This is not merely a missing feature. The root secret key feeds three things: +The root secret key feeds three things, so rotating it at runtime would invalidate every session cluster-wide and break node-to-node authentication: -1. **STS session token signing** (`root_credentials::token_signing_key`) — every - live session in the cluster is HMAC-signed with it. -2. **The internode RPC secret** (`derive_rpc_secret`), unless - `RUSTFS_RPC_SECRET` is set explicitly. -3. **Legacy IAM at-rest decryption** for blobs migrated from MinIO. +1. STS session token signing (`root_credentials::token_signing_key`): every live session in the cluster is HMAC-signed with it. +2. The internode RPC secret (`derive_rpc_secret`), unless `RUSTFS_RPC_SECRET` is set explicitly. +3. Legacy IAM at-rest decryption for blobs migrated from MinIO. -Rotating it at runtime would therefore invalidate every session cluster-wide and -break node-to-node authentication. Making root mutable is a separate piece of -work with those three couplings as prerequisites; it is not a side effect of -adding a profile page. - -**Operational recommendation:** treat root as a bootstrap identity. Create a -built-in IAM user with the `consoleAdmin` policy for day-to-day administration. -That identity has a working password change and full 2FA support. +Making root mutable is separate work with those three couplings as prerequisites. Operational recommendation: treat root as a bootstrap identity and create a built-in IAM user with the `consoleAdmin` policy for day-to-day administration; that identity has a working password change and full 2FA support. ## Rate limiting, replay and expiry -| Control | Value | Where | +| Control | Value | Where / why | | --- | --- | --- | -| Failed attempts before lockout | 5 | `mfa/record.rs` | -| First lockout | 15 minutes, doubling per further run | `mfa/record.rs` | -| Lockout ceiling | 1 hour | so a sustained attack cannot deny the owner indefinitely | -| TOTP clock skew | ±1 step (±30s) | three codes valid at once, no more | -| TOTP replay | Consumed time step is a high-water mark; `step <= last_used` is refused | closes the ~90s window a captured code would otherwise have | +| Failed attempts before lockout | 5 (`MAX_FAILED_ATTEMPTS`) | `crates/iam/src/mfa/record.rs` | +| First lockout | 15 minutes, doubling per further run | `record.rs` | +| Lockout ceiling | 1 hour | A sustained attack cannot deny the owner indefinitely. | +| TOTP clock skew | +/-1 step, +/-30s (`TOTP_SKEW_STEPS`) | Three codes valid at once, no more. | +| TOTP replay | Consumed time step is a high-water mark; `step <= last_used` is refused | Closes the ~90s window a captured code would otherwise have. | | Recovery code replay | `used_at` stamp, single use | | -| Login challenge TTL | 5 minutes | | -| Pending enrollment TTL | 10 minutes | an abandoned enrollment leaves no usable secret | +| Login challenge TTL | 5 minutes (`CHALLENGE_TTL_SECONDS`) | | +| Pending enrollment TTL | 10 minutes | An abandoned enrollment leaves no usable secret. | -A wrong code, a replayed code and a malformed code are **indistinguishable** on -the wire: all three return `AccessDenied` with the same message. The distinction -survives only in the audit trail, so an operator can tell a guessing attempt from -a replay without an attacker learning that a captured code was genuine. - -The lockout is stored in the record and updated under an optimistic -compare-and-set, so it holds across the cluster rather than per node. +A wrong code, a replayed code, and a malformed code are indistinguishable on the wire: all three return `AccessDenied` with the same message. The distinction survives only in the audit trail, so an operator can tell a guessing attempt from a replay without an attacker learning that a captured code was genuine. The lockout is stored in the record and updated under an optimistic compare-and-set, so it holds across the cluster rather than per node. ## Storage -``` +```text .rustfs.sys/config/mfa//totp.json (encrypted with the IAM master key) ``` -A sibling of `config/iam/`, not a child: the IAM cache loader walks the whole -`config/iam/` tree on startup and buckets what it finds by first path segment, so -a new prefix under there would be swept into that walk for no benefit. +A sibling of `config/iam/`, not a child: the IAM cache loader walks the whole `config/iam/` tree on startup and buckets what it finds by first path segment, so a new prefix under there would be swept into that walk for no benefit. -Records are **not cached**. Every verification reads from the store, because a -cache would need cluster-wide invalidation to keep the replay mark and the -lockout counter honest, and getting that wrong reopens exactly the holes this -design closes. Verifications are rare enough that the read is not worth -optimising. - -Writes are read-modify-write under an `If-Match` precondition with bounded -retries — the same optimistic scheme the IAM lazy-rewrite path uses. It degrades -to a retry rather than to a distributed lock a crashed node would have to time -out. +Records are not cached. Every verification reads from the store, because a cache would need cluster-wide invalidation to keep the replay mark and the lockout counter honest, and getting that wrong reopens exactly the holes this design closes. Verifications are rare enough that the read is not worth optimising. Writes are read-modify-write under an `If-Match` precondition with bounded retries, the same optimistic scheme the IAM lazy-rewrite path uses; it degrades to a retry rather than to a distributed lock a crashed node would have to time out. ## Login challenges are stateless -A challenge is `HMAC-SHA256(root_secret, "rustfs-mfa-challenge:v1" ‖ access_key ‖ -issued_at)`, base64url-encoded with its payload. - -The obvious alternative is a TTL cache, the way the OIDC flow stores its PKCE -verifiers. That store is node-local, which is fine for OIDC because the whole -authorization round trip returns to the node that started it. A second factor -does not: a cluster behind a load balancer without session affinity would issue -the challenge on one node and receive the code on another, and a node-local -challenge would fail there for reasons no operator could debug. - -Statelessness costs nothing, because the challenge is not what makes the exchange -single-use — the consumed TOTP time step is. +A challenge is `HMAC-SHA256(root_secret, "rustfs-mfa-challenge:v1" ‖ access_key ‖ issued_at)`, base64url-encoded with its payload. A node-local TTL cache (the way the OIDC flow stores PKCE verifiers) works for OIDC because the whole round trip returns to the node that started it; a second factor behind a load balancer without session affinity would issue the challenge on one node and receive the code on another. The challenge is not what makes the exchange single-use; the consumed TOTP time step is. ## Authorization model | Route | Gate | | --- | --- | | `GET /v3/account/info` | possession of the credential | -| `POST /v3/account/password` | credential **+ knowledge of the current secret** | +| `POST /v3/account/password` | credential + knowledge of the current secret | | `GET /v3/account/mfa` | possession of the credential | | `POST /v3/account/mfa/enroll` | credential, and the credential kind must be mutable | | `POST /v3/account/mfa/activate` | credential + a valid code from the pending secret | -| `POST /v3/account/mfa/disable` | credential + **a valid code and the account password** | +| `POST /v3/account/mfa/disable` | credential + a valid code and the account password | | `POST /v3/account/mfa/recovery-codes` | credential + a valid code | | `GET /v3/mfa/challenge` | possession of the credential | | `GET /v3/user/mfa` | `admin:GetUser` | | `DELETE /v3/user/mfa` | `admin:EnableUser` | | `PUT /v3/set-user-secret-key` | `admin:CreateUser` | -The self-service routes carry **no admin action**. Giving them one would be wrong -in both directions: it would stop an ordinary user from changing their own -password, and it would let any holder of that action change somebody else's. -They are registered as `CredentialOnly` in the route-policy matrix. +The self-service routes carry no admin action and are registered as `CredentialOnly` in the route-policy matrix. Giving them one would be wrong in both directions: it would stop an ordinary user from changing their own password, and it would let any holder of that action change somebody else's. -`POST /v3/account/password` and `POST /v3/account/mfa/disable` require a -proof-of-knowledge step because a signature only proves a credential was *used*. -The Console signs with a short-lived session, so without it a hijacked browser -tab could rewrite the account's credentials or strip its second factor. +`POST /v3/account/password` and `POST /v3/account/mfa/disable` require a proof-of-knowledge step because a signature only proves a credential was used. The Console signs with a short-lived session, so without it a hijacked browser tab could rewrite the account's credentials or strip its second factor; requiring the password makes disabling the factor as hard as the thing the factor protects. -### Why turning the factor off needs the password too - -Requiring only a code would mean a single shoulder-surfed number, in a session -someone walked away from, is enough to remove the protection. Requiring the -password makes disabling the factor as hard as the thing the factor protects. - -### Break-glass - -`DELETE /v3/user/mfa` clears another identity's factor, for a user who lost both -their authenticator and their recovery codes. It is gated on `admin:EnableUser` -rather than a bespoke action, because that is the same capability that can -already re-enable a disabled account — anyone who can do that can already take -the identity over, so a separate action would be a distinction without a security -difference. - -The record is deleted outright rather than disabled, so no stale lockout counter -survives to block the user's next enrollment. The acting administrator is -recorded in the audit entry. +Break-glass: `DELETE /v3/user/mfa` clears another identity's factor, for a user who lost both their authenticator and their recovery codes. It is gated on `admin:EnableUser` rather than a bespoke action because that capability can already re-enable a disabled account, and anyone who can do that can already take the identity over. The record is deleted outright rather than disabled, so no stale lockout counter survives to block the user's next enrollment. The acting administrator is recorded in the audit entry. ## Recovery codes -Ten codes, `XXXX-XXXX-XXXX-XXXX-XXXX`, 100 bits of uniform randomness each, in a -Crockford base32 alphabet with `I`, `L`, `O` and `U` removed so a handwritten -code cannot be ambiguous. +Ten codes (`RECOVERY_CODE_COUNT`), `XXXX-XXXX-XXXX-XXXX-XXXX`, 100 bits of uniform randomness each (`RECOVERY_CODE_ENTROPY_BITS`), in a Crockford base32 alphabet with `I`, `L`, `O` and `U` removed so a handwritten code cannot be ambiguous. -Stored as domain-separated SHA-256 digests, **not** a password KDF. With 100 bits -of uniform randomness there is no dictionary to try and no human-chosen pattern -to exploit, so the attacks a slow KDF defends against do not apply — while a -memory-hard KDF would have to run once per stored code on every verification -attempt, turning each guess into an attacker-controlled multiple of that cost. -This is the standard treatment for high-entropy bearer tokens, and the same -reasoning is why there is no per-code salt. +Stored as domain-separated SHA-256 digests, not a password KDF. With 100 bits of uniform randomness there is no dictionary to try and no human-chosen pattern to exploit, so the attacks a slow KDF defends against do not apply, while a memory-hard KDF would have to run once per stored code on every verification attempt, turning each guess into an attacker-controlled multiple of that cost. This is the standard treatment for high-entropy bearer tokens, and the same reasoning is why there is no per-code salt. -Codes are returned in plaintext exactly once. Activation always replaces the set: -reusing a previous one would leave codes valid for a secret they were never -issued against. Disabling clears them, so no live bypass survives a factor the -user believes is gone. +Codes are returned in plaintext exactly once. Activation always replaces the set: reusing a previous one would leave codes valid for a secret they were never issued against. Disabling clears them, so no live bypass survives a factor the user believes is gone. ## Audit Two `EventName` variants carry the whole surface: -- `iam:Identity:CredentialChanged` — password rotation, enrollment, activation, - disable, recovery-code regeneration, administrative reset. -- `iam:Identity:AuthChallenge` — challenge issuance and second-factor - verification. +| Event | Covers | +| --- | --- | +| `iam:Identity:CredentialChanged` | Password rotation, enrollment, activation, disable, recovery-code regeneration, administrative reset. | +| `iam:Identity:AuthChallenge` | Challenge issuance and second-factor verification. | -The per-operation detail lives in `api.name` and the `iamOperation` tag, which is -what a SIEM filters on. The enum is coarse because `EventName::mask()` gives every -variant its own bit in a `u64` and the budget is nearly spent — 63 of 64 used -after these two. Splitting these per-operation needs `mask()` widened first. +The per-operation detail lives in `api.name` and the `iamOperation` tag, which is what a SIEM filters on. The enum is coarse because `EventName::mask()` gives every variant its own bit in a `u64`; splitting these per operation needs `mask()` widened first. -**Redaction:** no secret key, TOTP secret, provisioning URI, submitted code or -recovery code enters an audit entry — not even hashed, and not on the failure -paths where the submitted value would be the most tempting thing to record. -Failures are described by a closed set of static strings -(`AccountAuditFailure`), so no caller-supplied bytes can reach a log target -through this module. +Redaction: no secret key, TOTP secret, provisioning URI, submitted code, or recovery code enters an audit entry, not even hashed, and not on the failure paths where the submitted value would be the most tempting thing to record. Failures are described by a closed set of static strings (`AccountAuditFailure`), so no caller-supplied bytes can reach a log target through this module. ## Known limitations -1. **2FA does not gate direct SigV4 access.** By design; see above. The fix is - the `aws:MultiFactorAuthPresent` policy condition. -2. **Root cannot rotate its own credentials at runtime.** By design; see above. -3. **GHSA-m77q-r63m-pj89 is unaffected.** STS session tokens are signed with the - root secret key, so anyone holding it can still forge a session token — - including one carrying `x-rustfs-mfa-verified`. 2FA does not close this; a - dedicated STS signing key does, and that advisory is tracked separately. -4. **Username changes are not supported for anyone.** The access key is the - primary key for policy mappings, group membership, service-account parents and - bucket-policy principals. A rename is a migration that orphans service - accounts and silently breaks bucket-policy ARNs, not an edit; - `mutable.username` is `false` for every identity. +| Limitation | Status | +| --- | --- | +| 2FA does not gate direct SigV4 access. | Design boundary (above). The fix is the `aws:MultiFactorAuthPresent` policy condition, not implemented yet. | +| Root cannot rotate its own credentials at runtime. | Design boundary (above); three couplings must be broken first. | +| GHSA-m77q-r63m-pj89 is unaffected. | STS session tokens are signed with the root secret key, so anyone holding it can still forge a session token, including one carrying `x-rustfs-mfa-verified`. A dedicated STS signing key closes this; tracked separately. | +| Username changes are not supported for anyone. | The access key is the primary key for policy mappings, group membership, service-account parents, and bucket-policy principals. A rename is a migration that orphans service accounts and silently breaks bucket-policy ARNs; `mutable.username` is `false` for every identity. | diff --git a/docs/operations/vault-kms-authentication.md b/docs/operations/vault-kms-authentication.md index adabd2918..02e826c94 100644 --- a/docs/operations/vault-kms-authentication.md +++ b/docs/operations/vault-kms-authentication.md @@ -1,6 +1,8 @@ # Vault KMS authentication runbook -This runbook covers how the RustFS Vault KMS backends (KV2 and Transit) authenticate to Vault, how to deploy AppRole and Vault Agent token-file authentication, and how the fail-closed credential window behaves in production. For what each backend stores in Vault and the KV2/Transit policy scopes, see [KMS backend security properties](kms-backend-security.md). +**Use this when:** configuring how the Vault KMS backends (KV2 and Transit) authenticate to Vault, rotating AppRole SecretIDs or agent tokens, or diagnosing `KMS credentials unavailable` errors. + +**Source of truth:** `crates/kms/src/backends/vault_credentials.rs` (`DEFAULT_TOKEN_FILE_POLL_INTERVAL_SECS`, `refresh_safety_window_secs`, the renewal loop and its log lines), `crates/kms/src/config.rs` (`RUSTFS_KMS_TIMEOUT_SECS` default). For what each backend stores in Vault and the KV2/Transit policy scopes, see [KMS backend security properties](kms-backend-security.md). ## Choosing an authentication method @@ -11,7 +13,7 @@ This runbook covers how the RustFS Vault KMS backends (KV2 and Transit) authenti | Kubernetes | `Kubernetes` | Lease-bound token obtained by login; renewed by RustFS | Renew at half TTL, re-login on failure | Production on Kubernetes, with no credential to distribute | | Agent token file | `TokenFile` | Owned by Vault Agent; RustFS only re-reads the sink file | File re-read once per poll interval | Production with a Vault Agent (or equivalent) managing auth | -Exactly one method must be configured. Setting `RUSTFS_KMS_VAULT_TOKEN_FILE` together with any other method, or `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` together with `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID`, is rejected at startup with a configuration error, because the effective identity would be ambiguous. A leftover `RUSTFS_KMS_VAULT_TOKEN` alongside a configured login method is tolerated and ignored, so a stale variable cannot silently downgrade the identity. +Exactly one method must be configured. Setting `RUSTFS_KMS_VAULT_TOKEN_FILE` together with any other method, or `RUSTFS_KMS_VAULT_KUBERNETES_ROLE` together with `RUSTFS_KMS_VAULT_APPROLE_ROLE_ID`, is rejected at startup with a configuration error because the effective identity would be ambiguous. A leftover `RUSTFS_KMS_VAULT_TOKEN` alongside a configured login method is tolerated and ignored, so a stale variable cannot silently downgrade the identity. All of these are read the same way whether the service is started with `RUSTFS_KMS_ENABLE=true` or configured later through `POST /rustfs/admin/v3/kms/configure`. @@ -55,11 +57,15 @@ RustFS logs in at startup, then renews the token at half its TTL in the backgrou ### SecretID delivery and rotation -Deliver the SecretID out of band — a secrets-manager-mounted file, an init-container writing `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE`, or Vault response wrapping unwrapped by your deployment tooling. Treat it like a password: owner-readable file permissions, never in logs or shell history. +Deliver the SecretID out of band (a secrets-manager-mounted file, an init-container writing `RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE`, or Vault response wrapping unwrapped by your deployment tooling). Treat it like a password: owner-readable file permissions, never in logs or shell history. -The secret_id file is re-read on every login attempt, so rotating the SecretID is a two-step operation with no restart: generate a new SecretID (`vault write -f auth/approle/role/rustfs-kms/secret-id`), atomically replace the file, then revoke the old SecretID accessor. The already-issued token keeps renewing; the new SecretID is only needed at the next full re-login. +The secret_id file is re-read on every login attempt, so rotating the SecretID needs no restart: -An empty or missing secret_id file fails the login attempt immediately (no Vault round trip). At startup the error is fatal — provider construction fails and the process exits — so a file missing at boot is recovered by restarting the process, not by an in-process retry. Once RustFS is running, the same failure is retried on the normal refresh cadence, so repairing the file mid-run heals the backend without a restart. +1. Generate a new SecretID: `vault write -f auth/approle/role/rustfs-kms/secret-id`. +2. Atomically replace the file. +3. Revoke the old SecretID accessor. The already-issued token keeps renewing; the new SecretID is only needed at the next full re-login. + +An empty or missing secret_id file fails the login attempt immediately (no Vault round trip). At startup the error is fatal: provider construction fails and the process exits, so a file missing at boot is recovered by restarting the process, not by an in-process retry. Once RustFS is running, the same failure is retried on the normal refresh cadence, so repairing the file mid-run heals the backend without a restart. ## Kubernetes authentication @@ -96,7 +102,7 @@ RUSTFS_KMS_VAULT_KUBERNETES_ROLE=rustfs RustFS logs in at startup and renews the token at half its TTL, falling back to a fresh login exactly as AppRole does. The ServiceAccount token is re-read from disk on every login rather than cached, so a projected token the kubelet rotates is picked up without a restart. -A missing or empty token file fails the login attempt immediately (no Vault round trip). At startup the error is fatal — provider construction fails and the process exits — so a token projected late during a slow pod start is recovered by the pod restart loop, not by an in-process retry. Once RustFS is running, a token file that goes missing or turns empty is retried on the normal refresh cadence and heals the backend on its own. +A missing or empty token file fails the login attempt immediately (no Vault round trip). At startup the error is fatal (provider construction fails and the process exits), so a token projected late during a slow pod start is recovered by the pod restart loop, not by an in-process retry. Once RustFS is running, a token file that goes missing or turns empty is retried on the normal refresh cadence and heals the backend on its own. ## Vault Agent token file @@ -130,22 +136,24 @@ RUSTFS_KMS_VAULT_ADDRESS=https://vault.example.com:8200 RUSTFS_KMS_VAULT_TOKEN_FILE=/run/vault-agent/token ``` -The poll interval (`poll_interval_secs` in the `TokenFile` auth configuration, default 30 seconds) controls how often the file is re-read. Each successful read grants the token an observed validity of twice the poll interval and installs a fresh client generation, so an agent-rotated token is picked up within one poll interval of the atomic replace. +The poll interval (`poll_interval_secs` in the `TokenFile` auth configuration, `DEFAULT_TOKEN_FILE_POLL_INTERVAL_SECS`, 30 seconds) controls how often the file is re-read. Each successful read grants the token an observed validity of twice the poll interval and installs a fresh client generation, so an agent-rotated token is picked up within one poll interval of the atomic replace. Requirements enforced at every read, each failing the refresh without contacting Vault: - The file must exist and be non-empty after trimming whitespace. - On Unix, the file must not be readable or writable by group or other (mode `0600` or stricter). Wider permissions are a hard error naming the offending mode, mirroring the SFTP host-key rule. RustFS must run as the file's owner. -If the agent stops refreshing the file that is fine — RustFS re-reads the same token and keeps going as long as the token itself is valid on the Vault side. If the file disappears or turns empty, RustFS keeps serving requests on the last-read token until the fail-closed window trips, and heals automatically once the file is restored. +If the agent stops refreshing the file, RustFS re-reads the same token and keeps going as long as the token itself is valid on the Vault side. If the file disappears or turns empty, RustFS keeps serving requests on the last-read token until the fail-closed window trips, and heals automatically once the file is restored. ## Fail-closed window For lease-bound credentials (AppRole and Kubernetes tokens, token files), `current()` refuses to hand out a token that is within the safety window of its expiry and has not been refreshed. Requests then fail with `KMS credentials unavailable: ...` instead of being sent with a token that could lapse mid-flight and fail unpredictably on the Vault side. -- Default window: one per-attempt timeout (`RUSTFS_KMS_TIMEOUT_SECS`, default 30s) — a request issued now can legitimately stay in flight that long, so the token must outlive it. -- Override: `refresh_safety_window_secs` on the `AppRole`, `Kubernetes` or `TokenFile` auth configuration. -- Static tokens never trip the window: they carry no lease and are assumed valid until Vault says otherwise. +| Aspect | Value | +| --- | --- | +| Default window | One per-attempt timeout (`RUSTFS_KMS_TIMEOUT_SECS`, default 30s): a request issued now can legitimately stay in flight that long, so the token must outlive it. | +| Override | `refresh_safety_window_secs` on the `AppRole`, `Kubernetes` or `TokenFile` auth configuration. | +| Static tokens | Never trip the window; they carry no lease and are assumed valid until Vault says otherwise. | The window is a symptom threshold, not the fault itself: by the time it trips, refresh has been failing for roughly half the token TTL (AppRole, Kubernetes) or two poll intervals (token file). @@ -153,12 +161,12 @@ The window is a symptom threshold, not the fault itself: by the time it trips, r | Symptom | Log line to look for | Likely cause and fix | | --- | --- | --- | -| Requests fail with `KMS credentials unavailable` | `Vault credential refresh failed; retrying until the credentials recover` (warn, repeated) | Vault unreachable/sealed, or the credential source is broken; the provider recovers on its own once refresh succeeds — fix the cause, no restart needed | -| Renewal succeeded but re-login later fails | `Vault token renewal failed; falling back to a fresh login` followed by login errors | SecretID expired/revoked or AppRole role changed; rotate the secret_id file | -| Token file mode error at startup or during polls | `has insecure permissions` in the error | Fix the sink `mode` (0600) and the file owner; the next poll heals the provider | -| Token file missing/empty errors | `Failed to read Vault token file` / `token file ... is empty` | Vault Agent down or sink misconfigured; restart the agent, the next poll heals the provider | -| Kubernetes login fails with a permission error | `Vault Kubernetes login failed` | The pod's ServiceAccount is not in the role's `bound_service_account_names`/`_namespaces`, or `auth/kubernetes/config` names the wrong API server | -| Kubernetes ServiceAccount token errors | `Failed to read Kubernetes ServiceAccount token` / `ServiceAccount token ... is empty` | The token is not projected into the pod (check `automountServiceAccountToken` and the volume mount); the next refresh cycle heals the provider | -| Startup fails immediately with a configuration error naming two env vars | — | Two auth methods configured at once; keep exactly one of token, AppRole, Kubernetes, token file | +| Requests fail with `KMS credentials unavailable` | `Vault credential refresh failed; retrying until the credentials recover` (warn, repeated) | Vault unreachable/sealed, or the credential source is broken; the provider recovers on its own once refresh succeeds. Fix the cause; no restart needed. | +| Renewal succeeded but re-login later fails | `Vault token renewal failed; falling back to a fresh login` followed by login errors | SecretID expired/revoked or AppRole role changed; rotate the secret_id file. | +| Token file mode error at startup or during polls | `has insecure permissions` in the error | Fix the sink `mode` (0600) and the file owner; the next poll heals the provider. | +| Token file missing/empty errors | `Failed to read Vault token file` / `token file ... is empty` | Vault Agent down or sink misconfigured; restart the agent, the next poll heals the provider. | +| Kubernetes login fails with a permission error | `Vault Kubernetes login failed` | The pod's ServiceAccount is not in the role's `bound_service_account_names`/`_namespaces`, or `auth/kubernetes/config` names the wrong API server. | +| Kubernetes ServiceAccount token errors | `Failed to read Kubernetes ServiceAccount token` / `ServiceAccount token ... is empty` | The token is not projected into the pod (check `automountServiceAccountToken` and the volume mount); the next refresh cycle heals the provider. | +| Startup fails immediately with a configuration error naming two env vars | (none) | Two auth methods configured at once; keep exactly one of token, AppRole, Kubernetes, token file. | When diagnosing, confirm three clocks/lifetimes in order: the Vault token TTL (`vault token lookup` with the token's accessor), the RustFS refresh cadence (half TTL or the poll interval), and the fail-closed window. The renewal task logs every failed cycle, so a silent gap in warnings combined with `CredentialsUnavailable` errors points at the process clock or a paused runtime rather than Vault. diff --git a/docs/testing/README.md b/docs/testing/README.md index 702c2c4c9..137285d4e 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -1,87 +1,30 @@ # RustFS Testing -> **Owner: backlog#1153 (infra-11).** This file is the authoritative home for -> the test-layer taxonomy, naming conventions, and serial/nextest rules. The -> event × budget × required-status matrix is owned separately by -> `docs/testing/ci-gates.md` (backlog#1149 ci-15); this file links to it rather -> than duplicating counts, timeouts, or required-check names. +**Use this when:** you need to pick a test layer for a change, name a test so a gate keeps selecting it, understand why `#[serial]` does nothing under nextest, or handle a flaky test. +**Source of truth:** `.config/nextest.toml` (profiles, test-groups, quarantine), `.config/make/tests.mak` (`make test`), `.github/workflows/*.yml` (what runs when; matrix in [ci-gates.md](ci-gates.md)). ## Test taxonomy -RustFS layers its tests from cheap-and-narrow to expensive-and-broad. Higher -layers catch what lower layers cannot but cost more wall-clock and setup, so -each layer has a clear entry point and a clear "when". Pick the lowest layer -that can prove your change; add a higher-layer test only when the behaviour is -not observable below it. +Pick the lowest layer that can prove the change; add a higher-layer test only when the behaviour is not observable below it. -| Layer | What it covers | Entry command | When it runs | +| Layer | What it covers | Entry command | When it runs (details: [ci-gates.md](ci-gates.md)) | |---|---|---|---| -| Unit & crate integration | Per-crate logic and in-process integration tests, run under nextest | `cargo nextest run --all --exclude e2e_test` (or `-p `) | Every PR (required) | -| ecstore black-box | Erasure-coded read/write/recovery validation of the ecstore stack | `scripts/run_ecstore_validation_suite.sh --profile quick` | Local / release validation (not in CI workflows) | -| e2e (`e2e_test` crate) | Full server spun up per test, driven over the S3 API | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR smoke lane + scheduled full/nightly lanes | -| s3s-e2e conformance | External S3 conformance tool run against a live rustfs server | `./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs-e2e-data` | Per-PR e2e gate (`e2e-tests` job) | -| S3 compatibility | Third-party suites: `ceph/s3-tests` (boto3) and MinIO `mint` (many SDKs) | `scripts/s3-tests/run.sh` (mint: `.github/workflows/mint.yml`) | s3-tests: per-PR gate; mint: scheduled, report-only | -| Chaos / fault-injection | Multi-node, power-loss, and disk-fault harness | — (harness planned) | Planned — tracked in backlog#1100 | -| Fuzz | `cargo-fuzz` targets over untrusted parsing/validation surfaces | `./scripts/fuzz/run.sh` (or `cd fuzz && cargo +nightly fuzz run `) | PR smoke + nightly corpus (`.github/workflows/fuzz.yml`) | -| Benchmarks | Criterion micro/throughput benchmarks | `cargo bench -p ` | On-demand / local | +| Unit & crate integration | Per-crate logic and in-process integration tests | `cargo nextest run --all --exclude e2e_test` (or `-p `); `make test` wraps it | Every PR, required (`Test and Lint`, `ci` profile) | +| ecstore black-box | Erasure-coded read/write/recovery validation; profiles `quick` / `full` / `destructive` / `fuzz` | `scripts/run_ecstore_validation_suite.sh --profile quick` | Local and release validation only; not wired into any workflow. Contract: [ecstore-validation-suite-design.md](ecstore-validation-suite-design.md) | +| e2e (`e2e_test` crate) | A real `rustfs` binary per test, driven over the S3, admin, and protocol APIs | `cargo nextest run --profile e2e-smoke -p e2e_test` | PR: `e2e-smoke` (report-only); merge queue / main push: `e2e-full`; nightly: `e2e-repl-nightly`, `e2e-nightly`, `e2e-protocols`. Guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md) | +| s3s-e2e conformance | External S3 conformance tool against a live server | `./scripts/e2e-run.sh ./target/debug/rustfs ` | PR, report-only (second half of the `End-to-End Tests` job) | +| S3 compatibility | `ceph/s3-tests` (boto3; allow-list `scripts/s3-tests/implemented_tests.txt`) and MinIO `mint` | `scripts/s3-tests/run.sh`; mint via `.github/workflows/mint.yml` | s3-tests: PR report-only plus a weekly full sweep; mint: weekly, report-only | +| Chaos / fault-injection | Single-node disk fault injection (`crates/e2e_test/src/chaos.rs`, `crates/e2e_test/src/fault_proxy.rs`) used by the reliability and heal e2e modules | Part of the e2e crate (`e2e-reliability` test-group) | With the `e2e-full` and nightly e2e lanes. A multi-node power-loss harness is not in tree | +| Fuzz | `cargo-fuzz` targets over untrusted parsing surfaces; isolated sub-workspace under `fuzz/` | `./scripts/fuzz/run.sh` (see [`fuzz/README.md`](../../fuzz/README.md)) | PR smoke on the paths listed in `.github/workflows/fuzz.yml`, plus nightly corpus | +| Benchmarks | Criterion benches under each crate's `benches/` | `cargo bench -p ` | On demand; never a gate | -> The **When it runs** column is a qualitative pointer only. The authoritative -> event × timeout × required-status matrix lives in `docs/testing/ci-gates.md` -> (ci-15) — do not duplicate its numbers here. - -Layer notes: - -- **Unit & crate integration** — the primary gate. `make test` wraps - `cargo nextest run --all --exclude e2e_test`; CI runs the same set under the - strict `ci` profile (`.config/nextest.toml`). Requires `cargo-nextest` (see - [Serial execution & nextest profiles](#serial-execution--nextest-profiles)). -- **ecstore black-box** — `scripts/run_ecstore_validation_suite.sh` has four - profiles (`quick` / `full` / `destructive` / `fuzz`); `quick` is the - PR-smoke-sized core read/write/recovery pass. It is a local and - release-validation tool, not wired into a CI workflow. -- **e2e** — each test spawns its own single-node rustfs server on a random port - with an isolated temp dir, so the suite is parallel-safe. The `e2e-smoke` - profile is the single PR wiring mechanism; slow/cross-process suites run in - scheduled lanes (`e2e-repl-nightly`, and the reliability group). Full - contributor guide: [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md); - per-module counts: [`e2e-suite-inventory.md`](e2e-suite-inventory.md). -- **s3s-e2e** — an external black-box conformance tool installed in CI; locally - run it against a freshly built binary with `scripts/e2e-run.sh - `. -- **S3 compatibility** — `ceph/s3-tests` exercises S3 semantics through boto3 - and is gated on a committed allow-list - (`scripts/s3-tests/implemented_tests.txt`); MinIO `mint` runs many real - client SDKs and is report-only until suites pass reliably (see the header of - `.github/workflows/mint.yml`). -- **Chaos / fault-injection** — multi-node, power-loss, and disk-fault - scenarios are out of scope for this repo's in-tree suites; the harness is - tracked in backlog#1100. (Single-node disk-fault e2e tests already live in the - e2e crate's `e2e-reliability` group.) -- **Fuzz** — the `cargo-fuzz` harness is an isolated sub-workspace under - `fuzz/` (kept out of the root workspace on purpose); see - [`fuzz/README.md`](../../fuzz/README.md) for targets and corpus rules. -- **Benchmarks** — Criterion benches live under each crate's `benches/`. They - are not a gate; run them locally to compare before/after on a specific crate. - -Script inventory: every entry under `scripts/` — including the runners above — -is indexed in [`scripts/README.md`](../../scripts/README.md) with its status -(ci-gate / dev-tool / archived) and wiring. - -### Security advisory regression tests - -Fixed GHSA advisories map to named, discoverable regression tests. The -advisory -> test map lives in -[`docs/testing/security-regressions.md`](security-regressions.md). sec-14 -(backlog#1151) formalizes the written admission policy in `AGENTS.md`. +Every script named above is indexed with status and wiring in [`scripts/README.md`](../../scripts/README.md). Fixed GHSA advisories map to named regression tests in [security-regressions.md](security-regressions.md). ## Naming conventions ### Reserved test-name substrings (migration gate) -The migration-critical CI gate selects tests **by name substring** rather than -by module path, so a rename that drops the substring silently thins the gate. -These substrings are therefore reserved: keep them in the test name when a test -proves migration-critical behaviour. +`scripts/check_migration_gate_count.sh` (runs in `Test and Lint`) selects migration-critical tests by name substring and fails when the count drops below `.config/migration-gate-floor.txt`. A rename that drops a substring silently thins the gate, so these substrings are reserved: | Substring | Guards | |---|---| @@ -91,138 +34,55 @@ proves migration-critical behaviour. | `source_cleanup` | Post-migration source cleanup | | `delete_marker` | Delete-marker handling across migration | -The gate is enforced by -[`scripts/check_migration_gate_count.sh`](../../scripts/check_migration_gate_count.sh), -which counts the selected tests and fails if the count drops below the committed -floor in -[`.config/migration-gate-floor.txt`](../../.config/migration-gate-floor.txt). -A deliberate reduction must lower the floor in the same PR, so the change is -reviewable in the diff (backlog#1153 infra-12). The substring list above is the -same one the gate uses; keep the two in sync when adding a reserved word. +A deliberate reduction lowers the floor in the same PR. The list above mirrors the script; change both together. ### General naming -- Name a regression test after what it pins: the issue or advisory number - (`..._regression_test`, `..._issue_NNNN_...`) or the invariant it protects, so - a reviewer can find the guard for a past bug by grepping. -- e2e lane membership is driven by test-name patterns in `.config/nextest.toml` - (for example the `_real_dual_node` / `_real_single_node` markers route - replication tests into the nightly lane). Follow the existing marker when - adding a test to an established suite; see - [`crates/e2e_test/README.md`](../../crates/e2e_test/README.md). -- Follow the Rust API Guidelines for symbol naming (see `AGENTS.md`). +- Name a regression test after what it pins (issue or advisory number, or the invariant) so `rg` finds the guard for a past bug. +- e2e lane membership is selected by test-name patterns in `.config/nextest.toml` (for example `_real_dual_node` / `_real_single_node` route replication tests to the nightly lane). Follow the existing marker of the suite you extend. +- Symbol naming follows the Rust API Guidelines (see `AGENTS.md`). -## Serial execution & nextest profiles +## nextest and `#[serial]` -**`cargo-nextest` is the runner.** `make test` requires it and CI installs it; -plain `cargo test` is not a faithful substitute because the two runners execute -tests differently. +`cargo-nextest` is the runner: `make test` requires it and CI installs it. nextest runs every test in its own process, so `serial_test`'s in-process `#[serial]` mutex does **not** serialize tests against each other; it only affects the plain `cargo test` fallback. Cross-test serialization under nextest comes from a `[test-groups]` entry with `max-threads = 1` in `.config/nextest.toml` (for example `ecstore-serial-flaky`, `e2e-reliability`) or from a `-j 1` lane. Prefer making tests self-isolating (per-test instance context, random port, own temp dir) over adding serialization. `RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1 make test` runs plain `cargo test`; its results are not authoritative because `[test-groups]` do not apply. -- **Install:** `cargo install cargo-nextest --locked`, or a prebuilt binary - (faster) from . -- **Escape hatch:** `RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1 make test` runs the - plain `cargo test` fallback, but the results are **not authoritative** — - serialization semantics differ from CI and `[test-groups]` do not apply. +Time-driven tests use paused tokio time (`start_paused` plus `tokio::time::advance`) or explicit synchronization instead of fixed `sleep` windows. -**Why `#[serial]` is mostly a no-op under nextest.** nextest runs every test in -its own process, so `serial_test`'s in-process `#[serial]` mutex does **not** -serialize tests against each other. The mechanism that actually serializes -across nextest's process boundary is a nextest `[test-groups]` entry with -`max-threads = 1` (for example `ecstore-serial-flaky` and `e2e-reliability` in -`.config/nextest.toml`). Consequences: +### Profiles -- Do not add `#[serial]` expecting cross-test isolation under nextest. If two - tests genuinely share process/global state or a fixed external resource, - serialize them with a `[test-groups]` entry, or make each test self-isolating - (per-test instance context — see backlog#1153 infra-7 / infra-8). -- A large share of the repo's existing `#[serial]` markers only affect the - `cargo test` fallback. The serial-debt census and removal plan are tracked in - backlog#1153 infra-7. +All profiles are defined in `.config/nextest.toml`; its block comments hold the filters and rationale. -**Profiles** (all defined in `.config/nextest.toml`): +| Profile | Role | +|---|---| +| `default` | Local runs; never retries | +| `ci` | PR gate for everything except `e2e_test`; global `retries = 0` plus the quarantine list | +| `e2e-smoke` | PR subset of `e2e_test` | +| `e2e-full` | Merge-queue / main-push single-node e2e lane | +| `e2e-repl-nightly` | Nightly slow / cross-process replication lane | +| `e2e-nightly` | Nightly serial multi-process cluster fault lane | +| `e2e-protocols` | Nightly fixed-port FTPS/SFTP/WebDAV lane, run with `-j 1` | -- `default` — local runs. **Never retries**: a red test locally is a real - failure to investigate, not noise to retry away. -- `ci` — the strict CI gate: global `retries = 0` plus a narrowly-scoped - quarantine list (`retries = 2`) for tests with a tracked OPEN flake issue. -- `e2e-smoke` — the PR smoke subset of the `e2e_test` crate (the single wiring - mechanism for e2e in PR CI). -- `e2e-repl-nightly` — the scheduled slow/cross-process replication lane. +Membership of each e2e profile is pinned by a digest in `.config/e2e--selection.txt` and checked by `scripts/check_test_wiring.py --check-profile ` before the lane runs. To list what a profile selects on your platform (the result is platform-dependent because some modules are linux-only): -### Time control (paused vs real clock) - -Time-driven tests should prefer paused time (`tokio::time` with `start_paused` -and `advance`) or explicit event synchronization over fixed `sleep` race -windows. The written convention and the `docs/testing/time-control.md` guide are -added by backlog#1153 infra-4. - -## Coverage - -Workspace line coverage is measured weekly. Pull requests that touch iam, kms, -policy, or crypto also run a non-required, report-only comparison against -`.config/coverage-baselines.toml`. During calibration, a regression is recorded -in the job summary without failing the job; missing or malformed coverage -evidence still fails closed (backlog#1153 infra-6). - -- **CI**: `.github/workflows/coverage.yml` runs every Sunday and on manual - dispatch: `cargo llvm-cov nextest --workspace --exclude e2e_test` under the - `ci` nextest profile — the same scope and profile as the PR test gate. The - per-crate line-coverage table lands in the run's job summary; the lcov + - JSON exports are uploaded as a `coverage-lcov-` artifact kept for - 90 days. Scheduled failures open/append the `[scheduled-failure] coverage` - issue via the shared alert action (ci-8). -- **Local**: `make coverage` is the equivalent (slow — instrumented rebuild - plus the full suite). It prints the same per-crate table via - `scripts/coverage_per_crate.py` and writes `target/llvm-cov/lcov.info` and - `coverage.json`. -- **Security-critical ratchet**: relevant pull requests compare iam / kms / - policy / crypto line coverage with the versioned baseline. Drops greater than - the configured one-percentage-point calibration threshold are marked - `REGRESSION (report-only)`. The weekly summary runs the same comparison so - calibration continues even when no relevant pull request is open. Baseline - changes require a linked coverage run and a reviewed explanation. -- **Trend comparison**: each run's job summary is the weekly per-crate - snapshot — open two runs from the Actions history (workflow "coverage") and - compare their tables. For line-level diffs, download the two runs' - `coverage-lcov-*` artifacts and compare the `lcov.info` files with your lcov - tooling of choice. -- **Not measured**: doctests (ci.yml runs them uninstrumented; covering them - would require a nightly toolchain) and the `e2e_test` crate (excluded from - the unit gate; its lanes are described in the taxonomy above). +```bash +cargo nextest list -p e2e_test --profile e2e-smoke --message-format json \ + | jq -r '.["rust-suites"][].testcases | to_entries[] | select(.value["filter-match"].status == "matches") | .key | split("::")[0]' \ + | sort | uniq -c +``` ## Flake policy -A flaky test is one that fails non-deterministically without a corresponding -code change. Flakes erode trust in the gate and block tightening required -checks, so they are handled on a strict, time-boxed loop. +A flaky test fails non-deterministically without a corresponding code change. Retry semantics live in `.config/nextest.toml`: `default` never retries, `ci` has global `retries = 0`, and only quarantined tests get `retries = 2` under `ci`. A quarantined test that passes on retry is marked `flaky` in `target/nextest/ci/junit.xml` (uploaded as a CI artifact); that marker, not a green check, is how a live flake stays visible. -**Retry semantics (source of truth: `.config/nextest.toml`):** +1. **Discover** — a non-deterministic failure (CI or local) or a `flaky` JUnit marker. +2. **Open an issue within 24h** describing symptom, suspected cause, and affected suite. No silent re-runs. +3. **Quarantine** — add a `[[profile.ci.overrides]]` entry with `retries = 2` and a comment linking exactly one OPEN issue. The current quarantine list is the `[[profile.ci.overrides]]` block in `.config/nextest.toml`. +4. **Fix or delete within 30 days** — make the test robust and remove the entry, or delete the test. An entry without a live OPEN issue link is a policy violation. -- The **local `default` profile never retries.** A red test on your machine is - a real failure to investigate, not noise to paper over. -- The **CI `ci` profile runs with global `retries = 0`.** A new race must fail - on its first occurrence so the first crime scene is never masked. -- Only tests on the **quarantine list** get `retries = 2`, and only under the - `ci` profile. Each quarantine entry MUST link exactly one OPEN issue. -- **JUnit flaky markers are the observable.** A quarantined test that passes - only after a retry is marked `flaky` in `target/nextest/ci/junit.xml` - (uploaded as a CI artifact). That marker — not a green check — is how we see - a flake is still live. +## Coverage -**Lifecycle of a flake:** - -1. **Discover** — a test fails non-deterministically (CI or local), or shows a - `flaky` marker in the JUnit report. -2. **Open an issue within 24h** — file/track an issue describing the flake - (symptom, suspected cause, affected suite). No silent re-runs. -3. **Quarantine** — add the test to the quarantine override block in - `.config/nextest.toml` with a comment linking that OPEN issue. This grants - `retries = 2` under CI so the flake stops reddening unrelated PRs, while the - `flaky` marker keeps it visible. -4. **Fix or delete within 30 days** — make the test robust (then remove the - quarantine entry) or delete the test. A quarantine entry may not outlive its - fix window; an entry without a live OPEN issue link is a policy violation. - -First quarantine members: the two backlog#937 ecstore groups -(`concurrent_resend_same_part_commits_one_generation` and -`store::bucket::tests::bucket_delete_*`). +- `.github/workflows/coverage.yml` measures workspace line coverage on its schedule and on manual dispatch: `cargo llvm-cov nextest --workspace --exclude e2e_test` under the `ci` profile, the same scope as the PR gate. The per-crate table lands in the job summary; lcov and JSON exports are uploaded as an artifact (retention set in the workflow). +- PRs touching the paths listed in `coverage.yml` also run a report-only comparison against `.config/coverage-baselines.toml` via `scripts/check_security_coverage.py`: a regression is recorded in the summary without failing the job; missing or malformed coverage evidence fails closed. +- `make coverage` (`.config/make/coverage.mak`) is the local equivalent; it writes `target/llvm-cov/lcov.info` and `coverage.json` and prints the same table via `scripts/coverage_per_crate.py`. +- Not measured: doctests (`ci.yml` runs them uninstrumented) and the `e2e_test` crate. +- A baseline change needs a linked coverage run and a reviewed explanation in the PR. diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 24de4b606..c12050a40 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -1,154 +1,102 @@ # CI gate matrix -This file is the source of truth for which validation runs on each event, its -configured wall-clock budget, and whether it can block a merge. Test taxonomy, -naming, and nextest serialization rules remain in [README.md](README.md); e2e -membership and counts remain in -[e2e-suite-inventory.md](e2e-suite-inventory.md). +**Use this when:** a check is red and you need to know whether it blocks the merge, which workflow and job produced it, and how to reproduce it locally. +**Source of truth:** the live `main` ruleset (command below) for required status; `.github/workflows/.yml` for triggers, `paths`, `timeout-minutes`, and cron; `.config/nextest.toml` for e2e profile filters; `.github/scheduled-validations.json` for the freshness-watchdog list. -The distinction between **required** and **report-only** is load-bearing: -a failing job blocks a merge only when its exact check name is present in the -live `main` ruleset. A workflow name, a `merge_group` trigger, or a red PR check -does not make a job required by itself. +A job blocks a merge only when its exact check name is in the live `main` ruleset. A workflow name, a `merge_group` trigger, or a red PR check does not make a job required by itself. ## Required merge checks -The live `main` ruleset (`6436880`) currently requires exactly these contexts: +The `main` ruleset (`6436880`) requires exactly these contexts, with `strict_required_status_checks_policy=false`: | Required context | Producer | Validation | |---|---|---| -| `CLA Check` | `.github/workflows/cla.yml` | Contributor agreement | -| `Quick Checks` | `.github/workflows/ci.yml` | Formatting and repository guard scripts | -| `Test and Lint` | `.github/workflows/ci.yml` | Clippy, workspace nextest excluding `e2e_test`, doctests, and migration proofs | +| `CLA Check` | `cla.yml` | Contributor agreement | +| `Quick Checks` | `ci.yml` job `quick-checks` | Formatting and repository guard scripts | +| `Test and Lint` | `ci.yml` job `test-and-lint` | Clippy, workspace nextest (`ci` profile, excluding `e2e_test`), doctests, migration-gate count (`scripts/check_migration_gate_count.sh`) | -For pull requests limited to the paths excluded by the main CI workflow, -`.github/workflows/ci-docs-only.yml` reports `Quick Checks` and -`Test and Lint` under the same names. It runs the real quick checks and the -planning-document guard; it does not claim that Rust compilation or runtime -tests ran. Despite the workflow name, these paths also include selected deploy, -workflow, and lock files. +For PRs limited to the `paths-ignore` list in `ci.yml`, `ci-docs-only.yml` reports `Quick Checks` and `Test and Lint` under the same names; it runs the quick checks and `scripts/check_no_planning_docs.sh`, not a Rust build or tests. `scripts/check_ci_paths_sync.sh` keeps the two path lists aligned. -Verify the live rule rather than trusting this snapshot before changing merge -policy: +Verify the live rule before changing merge policy: ```bash gh api repos/rustfs/rustfs/rulesets/6436880 \ --jq '.rules[] | select(.type == "required_status_checks") | .parameters' ``` -The ruleset currently has `strict_required_status_checks_policy=false`. -`Continuous Integration` accepts `merge_group` events and runs `e2e-full` for -them, but `End-to-End Tests (full merge gate)` is not currently a required -context. Therefore the repository is prepared to test a merge-queue SHA, but -the workflow alone does not prove that every merge passed that lane. +Promotion rule: never promote a report-only lane to required from one green run. Require at least 14 days and 30 representative PRs with at least 99% complete execution, then update the ruleset and this file together. ## Pull request and merge matrix -Budgets below are job `timeout-minutes`, not typical runtimes. “Report-only” -means the result is visible and actionable but is not in the live required -context list. +"Report-only" means visible and actionable but not in the required list. Budgets are each job's `timeout-minutes` in the named workflow and are not copied here. -| Event | Validation | Budget | Merge status | Reproduction | -|---|---|---:|---|---| -| PR, non-doc change | `Quick Checks` | 10 min | Required | `make pre-commit` (broader local umbrella) | -| PR, non-doc change | `Test and Lint` | 90 min | Required | `cargo nextest run --profile ci --all --exclude e2e_test` | -| PR, non-doc change | `Typos` | 10 min | Report-only | `typos` | -| PR, non-doc change | `ILM Integration (serial)` | 90 min | Report-only | Use the exact command in `.github/workflows/ci.yml` | -| PR, non-doc change | rio-v2 / swift / sftp test-and-lint variants | 90 min each | Report-only | `cargo nextest run` with the workflow's feature set | -| PR, non-doc change | `Build RustFS Debug Binary` | 30 min | Report-only; prerequisite for black-box lanes | `cargo build -p rustfs --bins` | -| PR, non-doc change | `io_uring Integration (real)` | 30 min | Report-only | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` | -| PR, non-doc change | `End-to-End Tests` (`e2e-smoke` plus `s3s-e2e`) | 30 min | Report-only | `cargo nextest run --profile e2e-smoke -p e2e_test`; then `./scripts/e2e-run.sh ./target/debug/rustfs ` | -| PR, non-doc change | `S3 Implemented Tests` | 60 min | Report-only | Build `rustfs`, then run `scripts/s3-tests/run.sh` with `DEPLOY_MODE=binary`, `TEST_MODE=single`, and `MAXFAIL=0` | -| PR, non-doc change | `S3 Lifecycle Behavior Tests` | 30 min | Report-only | Use the accelerated scanner environment in `.github/workflows/ci.yml` with `scripts/s3-tests/run.sh` | -| PR touching dependency or workflow inputs | Cargo Deny / Workflow Pin Report / Dependency Review | 20 / 5 / 30 min | Report-only | `cargo deny check`; `scripts/security/check_workflow_pins.sh` | -| PR touching architecture rules or architecture docs | `Architecture Migration Rules` | 10 min | Report-only | `scripts/check_architecture_migration_rules.sh` | -| PR touching Nix or workspace manifests | `Nix Build & Check` | 60 min | Report-only | `nix flake check` | -| PR limited to main-CI-excluded paths | companion `Quick Checks` and `Test and Lint` | 10 min each | Required | `git diff --check`; `make doc-paths-check` when documentation paths changed | -| `merge_group` | Standard CI plus `e2e-full` | 55 min for `e2e-full` | Standard required contexts only; `e2e-full` report-only | `cargo nextest run --profile e2e-full -p e2e_test` | -| Push to `main` | Standard CI plus `e2e-full` | 55 min for `e2e-full` | Post-merge detection | Same as `merge_group` | -| PR touching fuzz inputs or harness paths | Build plus five 60-second fuzz smoke targets | 60 min build; 30 min per target | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` | -| PR touching selected ecstore disk/format paths | `Rename Safety` on Windows | 60 min | Report-only | Run the four `cargo test -p rustfs-ecstore --lib ` commands in `windows-filesystem.yml` on Windows | +| Event | Check name | Workflow / job | Merge status | Reproduce | +|---|---|---|---|---| +| PR, non-doc change | `Quick Checks` | `ci.yml` `quick-checks` | Required | `make pre-commit` | +| PR, non-doc change | `Test and Lint` | `ci.yml` `test-and-lint` | Required | `cargo clippy --all-targets -- -D warnings`; `cargo nextest run --profile ci --all --exclude e2e_test`; `cargo test --all --doc`; `scripts/check_migration_gate_count.sh` | +| PR, non-doc change | `Typos` | `ci.yml` `typos` | Report-only | `typos` | +| PR, non-doc change | `ILM Integration (serial)` | `ci.yml` `test-ilm-integration-serial` | Report-only | exact command in the job | +| PR, non-doc change | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Report-only | `cargo nextest run` with the job's `--features` | +| PR, non-doc change | `Connect Short Credential Boundary` | `ci.yml` `connect-short-credential-boundary` | Report-only | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `cargo check -p rustfs --release --features connect-e2e-short-credentials` must fail | +| PR, non-doc change | `Build RustFS Debug Binary` | `ci.yml` `build-rustfs-debug-binary` | Report-only; prerequisite for the black-box jobs | `cargo build -p rustfs --bins` | +| PR, non-doc change | `io_uring Integration (real)` | `ci.yml` `uring-integration` | Report-only | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` | +| PR, non-doc change | `End-to-End Tests` | `ci.yml` `e2e-tests` | Report-only | `cargo nextest run --profile e2e-smoke -p e2e_test`, then `./scripts/e2e-run.sh ./target/debug/rustfs `; membership guards `scripts/check_test_wiring.py --check-profile e2e-smoke ` and `scripts/check_security_smoke_count.sh check ` | +| PR, non-doc change | `S3 Implemented Tests` | `ci.yml` `s3-implemented-tests` | Report-only | build `rustfs`, then `scripts/s3-tests/run.sh` with the job's `DEPLOY_MODE` / `TEST_MODE` / `MAXFAIL` env | +| PR, non-doc change | `S3 Lifecycle Behavior Tests` | `ci.yml` `s3-lifecycle-behavior-tests` | Report-only | `scripts/s3-tests/run.sh` with the job's accelerated-scanner env | +| PR touching `paths` in `audit.yml` | `Cargo Deny`, `Workflow Pin Report`, `Dependency Review` | `audit.yml` `cargo-deny`, `workflow-pin-report`, `dependency-review` | Report-only | `cargo deny check`; `scripts/security/check_workflow_pins.sh` | +| PR touching `paths` in `architecture-migration-rules.yml` | `Architecture Migration Rules` | `architecture-migration-rules.yml` `architecture-migration-rules` | Report-only | `scripts/check_architecture_migration_rules.sh` | +| PR touching `paths` in `nix.yml` | `Nix Build & Check` | `nix.yml` `nix-validation` | Report-only | `nix flake check` | +| PR touching `paths` in `fuzz.yml` | `Build Fuzz Harness`, `Smoke / ` | `fuzz.yml` `fuzz-build`, `pr-fuzz-smoke` | Report-only | `MAX_TOTAL_TIME=60 ./scripts/fuzz/run.sh` | +| PR touching `paths` in `windows-filesystem.yml` | `Rename Safety` | `windows-filesystem.yml` `rename-safety` | Report-only | the `cargo test -p rustfs-ecstore --lib ` commands in the job, on Windows | +| PR touching `paths` in `coverage.yml` | `Workspace line coverage` | `coverage.yml` `coverage` | Report-only | `make coverage`; `python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json` | +| PR touching `paths` in `e2e-upgrade.yml` | `Direct upgrade from rc.2` | `e2e-upgrade.yml` `direct-upgrade` | Report-only | the `cargo test --locked -p e2e_test` command in the job with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release | +| PR touching `paths` in `oidc-keycloak.yml` | `OIDC Keycloak live gate` | `oidc-keycloak.yml` `oidc-keycloak-live` | Report-only | `cargo build --locked -p rustfs --bin rustfs`, then `bash scripts/test/oidc_keycloak_live.sh ./target/debug/rustfs` | +| PR touching `paths` in `targets-integration.yml` | `PostgreSQL, MySQL, AMQP, and NATS` | `targets-integration.yml` `targets-live` | Report-only | start the containers as in the job, export the `RUSTFS_TEST_*` DSNs, then the job's `cargo test --locked -p rustfs-targets --test -- --ignored --test-threads=1` commands | +| PR limited to main-CI-excluded paths | `Quick Checks`, `Test and Lint` | `ci-docs-only.yml` `quick-checks`, `test-and-lint` | Required | `git diff --check`; `make doc-paths-check`; `scripts/check_no_planning_docs.sh` | +| `merge_group`; push to `main` | `End-to-End Tests (full merge gate)` | `ci.yml` `e2e-full` | Report-only | `cargo nextest run --profile e2e-full -p e2e_test` | -The authoritative e2e filters live in `.config/nextest.toml`; extend a profile -instead of adding a second ad-hoc selector. Before a profile runs, -`scripts/check_test_wiring.py` compares its exact membership to the committed -digest so a silent test drop fails closed. +e2e filters live in `.config/nextest.toml`; extend a profile instead of adding a second selector. Before a profile runs, `scripts/check_test_wiring.py` compares its listing to the committed digest in `.config/e2e--selection.txt`, so a silent test drop fails closed. -## Scheduled and manual validation +## Scheduled validation -Scheduled lanes are independent fault domains. They do not block a pull -request, but their workflow-local gate can fail the run and scheduled failures -are routed to the shared failure-issue action. The scheduled-validation -watchdog and freshness workflow separately detect incomplete runs and missing -schedules. +Scheduled lanes never block a PR. Their workflow-local gate fails the run, scheduled failures route to the shared failure-issue action, and `scheduled-validation-freshness.yml` fails when a workflow listed in `.github/scheduled-validations.json` has not run within its `max_age_hours` (a `never_ran_grace_until` entry covers the window before a newly enabled cron's first slot). Cadence is qualitative here; the cron lives in each workflow's `on.schedule`. -| Cadence (UTC unless noted) | Workflow / validation | Budget | Verdict and artifacts | Reproduction | -|---|---|---:|---|---| -| Daily 02:17 | Fuzz: five nightly corpus targets | 60 min build; 60 min per target | Gate; corpus/crash artifacts, scheduled failure alert | `MAX_TOTAL_TIME= ./scripts/fuzz/run.sh` | -| Dormant (cron 03:17 once re-enabled) | MinIO interop (EC + SSE read parity) | 40 min | Manually disabled in the Actions settings (backlog#1603) and therefore outside the freshness list; re-add it to `.github/scheduled-validations.json` when re-enabling | Follow the pinned Docker fixture steps in `minio-interop.yml` | -| Daily 04:29 | Replication / cluster-fault / protocol e2e | 45 / 90 / 90 min | Three independent gates; JUnit, membership, and server logs | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` | -| Daily 06:31 | Warp performance A/B | 180 min | Regression budget gate; A/B summaries and server logs | `bash scripts/run_hotpath_warp_abba.sh --help` | -| Daily 00:07 Asia/Shanghai (16:07 UTC previous day) | Nightly GNU build and Vault lanes | 150 / 90 / 60 min | Build, live Vault, and HA failover gates | Use the commands and pinned Vault images in `nightly-gnu.yml` | -| Daily 03:23 | Security Audit | 20 / 5 min, plus 30 min on PR dependency review | Cargo Deny and workflow-pin gates; scheduled failure alert | `cargo deny check`; `scripts/security/check_workflow_pins.sh` | -| Daily 23:47 | Scheduled Validation Freshness | 10 min | Fails when a critical schedule was never created or is stale; an entry may carry `never_ran_grace_until` to cover the window before a newly enabled cron's first slot | Dispatch `scheduled-validation-freshness.yml` | -| Sunday 00:11 | Full `Continuous Integration` matrix | Per-job budgets above | Weekly variant coverage, including dormant rio-v2 binary/e2e lanes | Dispatch `ci.yml` | -| Sunday 01:13 | Seven-platform build matrix | 150 min per platform | Build/package integrity; scheduled failure alert | Dispatch `build.yml` with an exact platform set | -| Sunday 02:19 | Ceph s3-tests full sweep: single and real four-node, four shards each | 180 min per shard | Compatibility gate; report, JUnit, exact node IDs, and server logs | `scripts/s3-tests/run.sh` against an existing single or distributed target | -| Sunday 06:41 | Mint | 120 min | **Report-only by design**; per-suite PASS/FAIL/NA and raw `log.json` | Reproduce the pinned Docker sequence in `mint.yml` or dispatch it | -| Sunday 07:43 | Workspace line coverage | 120 min | Report-only trend; lcov and JSON retained 90 days | `make coverage` | -| Monthly, day 1 06:37 | Runner Hygiene | 15 min | Validates runner ephemerality; scheduled failure alert | Dispatch `runner-hygiene.yml` | +| Workflow (cadence) | Jobs | Verdict and artifacts | In freshness list | Reproduce | +|---|---|---|---|---| +| `ci.yml` (weekly) | full matrix, including the schedule/dispatch-only rio-v2 jobs `build-rustfs-debug-binary-rio-v2` and `e2e-tests-rio-v2` | per-job | yes | dispatch `ci.yml` | +| `build.yml` (weekly) | `build-rustfs` over the six-target platform matrix in `prepare-platform-matrix` (four Linux, macOS aarch64, Windows x86_64) | build/package integrity | yes | dispatch `build.yml` with an exact platform set | +| `e2e-replication-nightly.yml` (nightly) | `repl-nightly`, `cluster-nightly`, `protocols-nightly` | three independent gates; JUnit, membership listing, server logs | yes | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` | +| `e2e-s3tests.yml` (weekly) | `s3tests` (single and distributed, four shards each), `upstream-head-canary` | compatibility gate; report, JUnit, node IDs, server logs | yes | `scripts/s3-tests/run.sh` against an existing single or distributed target | +| `fuzz.yml` (nightly) | `nightly-fuzz-corpus` per target | gate; corpus and crash artifacts | yes | `MAX_TOTAL_TIME= ./scripts/fuzz/run.sh` | +| `minio-interop.yml` (nightly) | `minio-interop` | EC + SSE read-parity gate | yes, with `never_ran_grace_until` | pinned Docker fixture steps in the workflow | +| `performance-ab.yml` (nightly) | `warp-ab` | regression-budget gate; A/B summaries and server logs | yes | `bash scripts/run_hotpath_warp_abba.sh --help` | +| `nightly-gnu.yml` (nightly) | `build`, `kms-vault-lane`, `kms-vault-ha-failover` | build, live Vault, and HA failover gates | yes | commands and pinned Vault images in the workflow | +| `audit.yml` (nightly) | `cargo-deny`, `workflow-pin-report` | dependency and workflow-pin gates | yes | `cargo deny check`; `scripts/security/check_workflow_pins.sh` | +| `mint.yml` (weekly) | `mint` | report-only by design; per-suite PASS/FAIL/NA and raw `log.json` | yes | pinned Docker sequence in the workflow | +| `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes | `make coverage` | +| `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch | +| `e2e-upgrade.yml` (weekly) | `direct-upgrade` | upgrade gate; server logs | no | see the PR row | +| `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | no | see the PR row | +| `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | no | see the PR row | +| `scheduled-validation-freshness.yml` (nightly) | `check-freshness` | fails on a never-created or stale schedule | n/a | dispatch | -Manual `workflow_dispatch` exists for the scheduled workflows above. Manual -runs are debugging evidence and intentionally do not open scheduled-failure -issues. A manual performance run may explicitly allow a known regression; that -override must not be treated as an ordinary passing baseline. +Manual `workflow_dispatch` runs are debugging evidence and do not open scheduled-failure issues. A manual performance run may explicitly allow a known regression; that override is not a passing baseline. ## Release validation -Release validation is post-merge and tag-driven; it does not substitute for a -pull-request gate. +Post-merge and tag-driven; not a substitute for a PR gate. -| Event | Validation | Budget | Result | -|---|---|---:|---| -| Push to `main` or weekly schedule | `Build and Release` platform matrix | 150 min per platform | Build artifacts for all selected targets; no release publication on a main push | -| Valid release or preview tag | `Build and Release` plus asset checks | 150 min per platform | Draft release, checksummed assets, and publish step | -| Successful non-preview release-tag build | Docker image build and image scan | 60 min build; 30 min scan | Multi-architecture images plus vulnerability report | -| Successful release-tag build | DEB/RPM packaging | 30 min per architecture | Packages and checksum files uploaded to the release | -| Successful non-preview release-tag build | Helm template test and package | 30 min build; 30 min publish | Versioned chart and repository index | +| Trigger | Workflow / job | Result | +|---|---|---| +| Push to `main`, weekly schedule, dispatch | `build.yml` `build-rustfs` (a development build on a main push restricts the matrix to the Linux targets) | build artifacts; no release publication | +| Valid release or preview tag | `build.yml` `build-rustfs`, `create-release`, `upload-release-assets`, `publish-release` | draft release, checksummed assets, publish | +| Successful non-preview release-tag build (`workflow_run`) | `docker.yml` `build-docker`, `scan-docker-image` | multi-architecture images and vulnerability report | +| Successful release-tag build (`workflow_run`) | `package.yml` `package` | DEB/RPM packages and checksums uploaded to the release | +| Successful non-preview release-tag build (`workflow_run`) | `helm-package.yml` `build-helm-package`, `publish-helm-package` | versioned chart and repository index | +| Final tag's release published | `build.yml` `cleanup-preview-releases` | deletes every `-preview.` Release for that target; the tags are kept | -Use an exact preview tag for end-to-end release rehearsal. Manual dispatches -are backfill/debug paths and do not prove the automatic `workflow_run` chain. - -A preview Release is internal validation state, not a deliverable: after the -final tag's release is published, `cleanup-preview-releases` deletes every -`-preview.` Release for that target. The tags themselves are kept, so -the validated commit stays traceable. - -## Evidence requirements - -A green check is useful only when it proves the intended behavior ran: - -- Record the exact commit SHA and run URL. -- Separate product failure from runner prerequisites, service readiness, and - cancellation. Repair the precondition, then rerun the exact workload. -- Preserve membership manifests, JUnit, raw compatibility logs, seeds, and - server logs where the workflow provides them. -- For a bug fix or a new fault checker, provide sensitivity evidence: the old - behavior or an intentional mutation must fail the new oracle, and the fixed - behavior must pass it. -- Never promote a report-only lane to required from one green run. Require at - least 14 days and 30 representative pull requests with at least 99% complete - execution, then update the ruleset and this table together. +Use an exact preview tag for an end-to-end release rehearsal. Manual dispatches are backfill/debug paths and do not prove the automatic `workflow_run` chain. ## Change checklist -Update this file in the same pull request when any of these change: - -- workflow triggers, job names, timeouts, or nextest profile ownership; -- required status contexts or strict/merge-queue policy; -- scheduled cadence, alert routing, artifact contract, or local reproduction; -- report-only versus gating semantics. - -Do not copy per-module test counts here. Update -[e2e-suite-inventory.md](e2e-suite-inventory.md) and its enforced membership -digest instead. +Update this file in the same PR when a job or check name changes, a workflow gains or loses a `pull_request` or `schedule` trigger, required contexts or strict/merge-queue policy change, report-only vs gating semantics change, or `.github/scheduled-validations.json` membership changes. Do not copy timeouts, crons, or test counts here. diff --git a/docs/testing/e2e-suite-inventory.md b/docs/testing/e2e-suite-inventory.md deleted file mode 100644 index 3ef58aeba..000000000 --- a/docs/testing/e2e-suite-inventory.md +++ /dev/null @@ -1,107 +0,0 @@ -# e2e_test suite inventory - -> Authoritative per-module test counts for the `e2e_test` crate (backlog#1149 -> ci-4), generated from `cargo nextest list -p e2e_test`. Regenerate with: -> ```bash -> cargo nextest list -p e2e_test --message-format json | jq -r '.["rust-suites"][]?.testcases | to_entries[] | select(.value.ignored == false) | .key | split("::")[0]' | sort | uniq -c -> ``` -> Modules marked ✅ are in the PR smoke profile `e2e-smoke`; 🌙 marks the -> cluster, protocol, and replication subsets in the consolidated nightly -> workflow. The `e2e-full` merge/main profile covers the remaining default -> single-node tests. Committed test-ID digests are enforced before each run. -> Note: counts exclude `#[ignore]`d tests (nextest lists them separately). -> Managed-SSE (SSE-S3/SSE-KMS) replication contracts assert successful -> re-encryption on the target (backlog#1783); SSE-C replication still pins a -> fail-closed FAILED contract until ciphertext passthrough lands. - -| module | tests | PR smoke | -|---|---|---| -| admin_auth_test | 4 | ✅ | -| admin_iam_crud_test | 3 | ✅ | -| admin_mfa_test | 4 | | -| admin_pools_test | 1 | ✅ | -| admin_timeout_regression_test | 1 | 🌙 | -| anonymous_access_test | 4 | ✅ | -| api_rate_limit_test | 3 | | -| archive_download_integrity_test | 13 | | -| bucket_logging_test | 3 | | -| bucket_policy_check_test | 1 | ✅ | -| bucket_stats_regression_test | 3 | | -| chaos | 2 | | -| checksum_upload_test | 7 | | -| cluster_concurrency_test | 3 | 🌙 | -| cluster_multidrive_pool_test | 4 | 🌙 | -| common | 17 | | -| compression_test | 6 | ✅ | -| connection_cap_test | 2 | | -| console_smoke_test | 1 | ✅ | -| content_encoding_test | 3 | ✅ | -| copy_object_checksum_test | 7 | | -| copy_object_metadata_test | 4 | ✅ | -| copy_object_tagging_test | 2 | ✅ | -| copy_object_version_restore_test | 2 | | -| copy_source_invalid_date_test | 1 | ✅ | -| create_bucket_region_test | 2 | ✅ | -| data_usage_test | 2 | | -| degraded_listing_availability_test | 1 | 🌙 | -| degraded_read_eof_regression_test | 3 | | -| delete_marker_migration_semantics_test | 2 | ✅ | -| delete_object_no_content_length_test | 1 | | -| delete_objects_versioning_test | 2 | ✅ | -| delete_regression_test | 5 | | -| distributed_startup_regression_test | 3 | | -| existing_object_tag_policy_test | 4 | | -| fake_s3_target | 6 | ✅ | -| fault_proxy | 7 | | -| get_codec_streaming_compat_test | 1 | | -| get_stream_failure_observability_test | 1 | | -| group_delete_test | 4 | | -| head_object_consistency_test | 1 | ✅ | -| head_object_range_test | 1 | ✅ | -| heal_erasure_disk_rebuild_test | 6 | 🌙 | -| inline_fast_path_cluster_test | 16 | | -| internode_rpc_signature_e2e_test | 5 | | -| kms | 50 | | -| leading_slash_key_test | 2 | ✅ | -| lifecycle_regression_test | 4 | | -| list_buckets_auth_test | 1 | ✅ | -| list_buckets_double_slash_test | 3 | ✅ | -| list_buckets_iam_filter_test | 1 | ✅ | -| list_object_versions_metadata_extension_test | 1 | | -| list_object_versions_regression_test | 2 | ✅ | -| list_objects_duplicates_test | 3 | ✅ | -| list_objects_v2_metadata_extension_test | 1 | | -| list_objects_v2_pagination_test | 12 | ✅ | -| listing_regression_test | 4 | | -| mc_mirror_small_bucket_test | 1 | | -| multipart_auth_test | 75 | | -| multipart_storage_class_test | 3 | ✅ | -| namespace_lock_quorum_test | 2 | 🌙 | -| negative_sigv4_test | 7 | ✅ | -| notification_startup_regression_test | 2 | | -| notification_webhook_test | 3 | ✅ | -| object_lambda_test | 16 | 🌙 | -| object_lock | 34 | | -| overwrite_cleanup_regression_test | 1 | | -| policy | 6 | | -| presigned_negative_test | 7 | ✅ | -| protocols | 16 | 🌙 | -| quota_test | 14 | | -| reliability_disk_fault_test | 4 | | -| reliant | 43 | 20 ✅ | -| replication_extension_test | 76 | 20 ✅ +56 🌙 | -| replication_lww_receiver_test | 1 | | -| security_boundary_test | 4 | | -| server_startup_failfast_test | 1 | | -| snowball_auto_extract_test | 6 | | -| special_chars_test | 14 | ✅ | -| ssec_copy_test | 2 | ✅ | -| stale_multipart_cleanup_cluster_test | 1 | 🌙 | -| storage_class_capability_test | 4 | ✅ | -| sts_query_compat_test | 6 | ✅ | -| tier_transition_regression_test | 3 | | -| tls_gen | 3 | | -| tls_hot_reload_test | 1 | ✅ | -| version_id_regression_test | 10 | ✅ | - -**Total listed: 622 tests across 86 modules · PR smoke: 165 tests / 36 modules · merge/main full: 495 tests / 77 modules · nightly replication: 56 tests · nightly cluster faults: 32 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-31. diff --git a/docs/testing/ecstore-validation-suite-design.md b/docs/testing/ecstore-validation-suite-design.md index 62e95f340..e9bec1fe4 100644 --- a/docs/testing/ecstore-validation-suite-design.md +++ b/docs/testing/ecstore-validation-suite-design.md @@ -1,294 +1,62 @@ -# ECStore Validation Suite Design +# ECStore Validation Suite -This document defines the validation suite RustFS should use before claiming -ECStore erasure-coding correctness, durability, and fault-resilience coverage. -It is a suite design, not a claim that all tests already exist. +**Use this when:** you run or extend `scripts/run_ecstore_validation_suite.sh`, or add an erasure-coding test and need the scenario row it must satisfy. +**Source of truth:** `scripts/run_ecstore_validation_suite.sh` (profiles, flags, step commands, coverage scope, default thresholds); `docs/architecture/erasure-coding.md` (the invariants the rows enforce). -## Goal - -Provide one command that runs the full ECStore confidence suite and produces a -single pass/fail result plus artifacts. The command should exercise both: - -- white-box invariants in `rustfs-ecstore`, `rustfs-filemeta`, and related - helpers; -- black-box S3 and admin behavior against real single-node and distributed - erasure deployments. - -The suite can reduce release risk; it cannot prove the absence of all defects. -It must therefore combine deterministic matrices, negative tests, fuzz/corpus -tests, chaos tests, and explicit artifact review. - -## Proposed Entry Point - -Use the top-level runner for repeatable local, CI, and release checks: +## Runner contract ```bash -scripts/run_ecstore_validation_suite.sh --profile full +scripts/run_ecstore_validation_suite.sh --profile \ + [--out-dir ] [--skip-e2e] [--skip-s3-tests] [--skip-coverage] [--require-fixtures] \ + [--unit-coverage-min ] [--unit-coverage-scope ] [--dry-run] ``` -Profiles: +The runner is a local and release-validation tool; no CI workflow invokes it. -| Profile | Purpose | Expected cost | -| --- | --- | --- | -| `quick` | PR smoke for EC logic and existing single-node reliability tests. | minutes | -| `full` | Release gate: white-box, e2e, chaos, S3 compatibility subset, coverage. | hours | -| `destructive` | Manual/nightly gate: distributed 4-node/16-disk, crash/restart, rebalance/decommission fault injection. | hours+ | -| `fuzz` | Malformed metadata/RPC/corpus fuzzing with fixed seed output. | bounded by budget | +| Profile | Steps (each profile includes the one above it) | Cost | +|---|---|---| +| `quick` | `rustfs-filemeta` lib tests; focused `rustfs-ecstore` lib filters (`erasure`, `set_disk::read`, `set_disk::core::io_primitives`, the rename-rollback test, `disk::local`); the two isolated global-state tests named at the top of the runner; the whole `rustfs-ecstore --lib` with `--test-threads=1`; e2e `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `namespace_lock_quorum_test` | minutes | +| `full` | fixture gate; legacy-bitrot and MinIO generated-read fixture tests; s3-tests subset `TESTEXPR="multipart or range or versioning or delete"` with `DEPLOY_MODE=build MAXFAIL=0`; unit coverage gate | hours | +| `destructive` | `disk::local::test::crash_consistency` (power loss at each pre-commit step; object reopens as old or new, never mixed); e2e `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `delete_marker_migration_semantics_test` | hours+ | +| `fuzz` | `scripts/fuzz/run.sh` bounded by `MAX_TOTAL_TIME` (default in the runner) | bounded | -The current runner wires the existing high-signal checks. The matrix below -still tracks required follow-up coverage before `full` can be treated as a -complete release gate. +Artifacts under `` (default `target/ecstore-validation//`): -The runner writes artifacts under -`target/ecstore-validation//`: +| File | Content | +|---|---| +| `run-metadata.env` | profile, flags, thresholds, environment | +| `summary.tsv` | one row per step: pass / fail / skip with reason | +| `blackbox-matrix.tsv` | every black-box and fixture row with command, fixture env, and status `enabled` / `disabled` / `missing-optional` / `missing-required`; written on every invocation regardless of profile | +| `logs/*.log` | per-step transcripts | +| `coverage/ecstore/lcov.info`, `summary.tsv`, `files.tsv` | coverage export, gate summary, per-file table (`full` and `destructive`) | -- command transcript and environment; -- randomized seeds; -- object manifests with SHA256; -- per-disk layout snapshots before and after mutation; -- server logs; -- junit/nextest output; -- `blackbox-matrix.tsv` with selected black-box and fixture gates; -- coverage report; -- failure reproducer instructions. +Fixture rows: -## Acceptance Rules +| Row | Profile | Fixture env | +|---|---|---| +| legacy bitrot read (`crates/ecstore/tests/legacy_bitrot_read_test.rs`) | `full` | `RUSTFS_LEGACY_TEST_ROOT`, `RUSTFS_LEGACY_TEST_DISK` | +| MinIO generated encrypted read and negative restore (`storage::minio_generated_read_test` in `rustfs/src/storage/mod.rs`, `--features rio-v2`, `--ignored`) | `full` | `RUSTFS_MINIO_FIXTURE_ROOT`, `RUSTFS_MINIO_STATIC_KMS_KEY_B64` | -The suite passes only when all selected profile commands pass and every -scenario asserts both API-visible behavior and on-disk state where applicable. +A missing fixture is recorded as a `missing-optional` skip. `--require-fixtures` turns it into an early `ecstore-fixture-gate` failure before the expensive black-box steps run. -Required fail-closed rules: +## Acceptance rules + +A run passes only when every selected step passes. Every scenario asserts API-visible behaviour and, where applicable, on-disk state; a test that only checks constants, helper calls, deleted branches, or implementation details does not satisfy a row. + +Fail-closed invariants every row enforces: - never return corrupted object bytes; - never silently accept forged or split-brain metadata; - never downgrade write quorum to read quorum; -- never leave a mixed old/new object after partial commit; +- never leave a mixed old/new object after a partial commit; - never panic on malformed EC metadata; - return typed errors or quorum failures for invalid states. -Unit-test coverage is a hard release-gate input: `rustfs-ecstore` unit line -coverage for the EC-critical scope must be at least 95%, with 100% as the -target for EC read, write, decode, heal, metadata quorum, and rollback paths. -A lower threshold is only acceptable for a temporary, explicitly documented -exception tied to missing testability or unreachable code. Full-crate coverage -is still reported as an observation metric, but it must not hide EC regressions -behind unrelated modules. +Fault injection is explicit and deterministic: local disk mocks for unit tests, process-level disk manipulation (`crates/e2e_test/src/chaos.rs`) for e2e tests. Property tests replay a fixed seed for payload, range, and missing-shard selection. -Tests must not pass by only checking constants, helper calls, deleted branches, -or implementation details. Every test needs a reader-facing or storage-state -assertion. +### Coverage gate -Fixture-backed checks are optional for local smoke runs, but explicit in the -artifact stream. `--require-fixtures` turns missing legacy or MinIO generated -fixtures into an early `ecstore-fixture-gate` failure before expensive black-box -steps run. The MinIO generated fixture gate requires both -`RUSTFS_MINIO_FIXTURE_ROOT` and `RUSTFS_MINIO_STATIC_KMS_KEY_B64`. - -## White-Box Matrix - -### Erasure Algorithm - -Target files: - -- `crates/ecstore/src/erasure/coding/erasure.rs` -- `crates/ecstore/src/erasure/coding/encode.rs` -- `crates/ecstore/src/erasure/coding/decode.rs` -- `crates/ecstore/src/erasure/coding/decode_reader.rs` -- `crates/ecstore/src/erasure/codec/bridge.rs` - -Coverage: - -| Area | Scenarios | Assertions | -| --- | --- | --- | -| Shard geometry | legacy/current shard-size formulas; lengths `0`, `1`, `block-1`, `block`, `block+1`, multi-block tail | no divide-by-zero; shard/file/range offsets match expected | -| Encode/decode | `(data, parity)` sets `2+2`, `4+2`, `8+8`; random payloads; missing shards up to parity | reconstructed data equals original | -| Negative reconstruction | missing shards above parity; inconsistent shard lengths; corrupt surplus parity | typed error, no partial success | -| Source verification | missing data shard plus extra parity source | rebuilt parity must match source parity | -| Legacy compatibility | old shard formula and legacy checksum data | legacy files decode and heal correctly | -| Streaming decode | legacy engine vs RustFS codec engine on same stripe stream | bytes and errors are equivalent | -| Range output | head/middle/tail/suffix; cross-block and final-short-stripe ranges | exact byte range, no over-read/under-read | - -Add property tests with fixed replay seeds for payload, range, and missing-shard -selection. - -### Bitrot and Reader Alignment - -Target files: - -- `crates/ecstore/src/erasure/coding/bitrot.rs` -- `crates/ecstore/src/erasure/coding/decode.rs` -- `crates/ecstore/src/set_disk/core/io_primitives.rs` -- `crates/ecstore/src/set_disk/shard_source.rs` - -Coverage: - -| Area | Scenarios | Assertions | -| --- | --- | --- | -| Hash framing | valid hash+data; wrong hash; truncated hash; truncated data | invalid data never succeeds | -| Short shard | short read under normal hash, `skip_verify`, and hash-none | `UnexpectedEof` or equivalent typed error | -| Lockstep reads | mid-stream data shard failure; pending/timeout reader; final short stripe | each live reader advances exactly one stripe; failed reader retires | -| Adaptive reads | hedged parity fallback and timeout retirement | no shard desync; reconstructed bytes match original | -| Shard source order | out-of-order read completion and missing slots | slots resolve by shard index | -| Deferred readers | data-blocks-first setup opens deferred parity at correct offset | parity fallback uses aligned data | - -Use instrumented readers that record shard index, stripe index, read count, -offset, and retirement reason. - -### Metadata, Quorum, and Commit Atomicity - -Target files: - -- `crates/filemeta/src/filemeta/version.rs` -- `crates/ecstore/src/set_disk/read.rs` -- `crates/ecstore/src/set_disk/metadata.rs` -- `crates/ecstore/src/set_disk/ops/object.rs` -- `crates/ecstore/src/set_disk/core/io_primitives.rs` -- `crates/ecstore/src/disk/local.rs` - -Coverage: - -| Area | Scenarios | Assertions | -| --- | --- | --- | -| Metadata tamper | same `version_id`/`mod_time`, divergent data dir, parts, ETag, size, checksum, inline flag, erasure distribution | previous committed version or read quorum error; no arbitrary latest | -| Early stop | valid quorum, stale quorum, corrupt trailing disks, slow trailing disks | early-stop only on safe identity | -| Quorum downgrade | read quorum vs write quorum; delete marker quorum; version-not-found quorum | no mutation below write quorum | -| Rename atomicity | failure before data rename, after data rename, after metadata rename, cleanup failure | object is old or new; never mixed | -| Rollback | failed commit quorum and stale temp data | rollback preserves old metadata/data | -| Malformed metadata | oversized lengths, bad CRC, invalid versions, invalid UUID/timestamp/enum, huge parts | bounded memory; typed error; no panic | - -Fault injection should be explicit and deterministic, preferably through local -disk mocks for unit tests and process-level disk manipulation for e2e tests. - -## Black-Box Matrix - -Use `crates/e2e_test` for real S3/admin behavior and extend -`crates/e2e_test/src/chaos.rs` rather than duplicating ad hoc helpers. - -The current runner emits `blackbox-matrix.tsv` for every invocation. It is a -machine-readable manifest of selected black-box scenarios, their commands, -required fixture environment, and whether each row is enabled, disabled, or -missing an optional/required fixture. - -Current runner rows: - -| Profile | Scenario | Gate | Fixture env | -| --- | --- | --- | --- | -| `quick` | single-node disk fault read/write | e2e black box | none | -| `quick` | degraded erasure disk rebuild | e2e black box | none | -| `quick` | namespace lock quorum under EC ops | e2e black box | none | -| `full` | legacy bitrot read fixture restore | fixture black box | `RUSTFS_LEGACY_TEST_ROOT`, `RUSTFS_LEGACY_TEST_DISK` | -| `full` | MinIO generated encrypted read and negative restore fixture | fixture black box | `RUSTFS_MINIO_FIXTURE_ROOT`, `RUSTFS_MINIO_STATIC_KMS_KEY_B64` | -| `full` | S3 multipart/range/versioning/delete subset | S3 black box | none | -| `destructive` | distributed cluster concurrency | e2e black box | none | -| `destructive` | stale multipart cleanup cluster | e2e black box | none | -| `destructive` | delete marker migration semantics | e2e black box | none | - -### Single-Node 4-Disk EC - -| Scenario | Required assertions | -| --- | --- | -| baseline PUT/GET/HEAD/List for tiny, inline, block-boundary, multi-block, multipart objects | SHA256 manifest matches; metadata is consistent on all disks | -| one disk offline during read | existing objects readable; no corrupted bytes | -| one disk offline during write | write succeeds only when write quorum holds; restored disk is healed | -| above-parity disk loss | GET/PUT fails with quorum error; no partial bytes accepted | -| corrupt data shard and parity shard | GET returns original bytes or fails closed; read-repair/heal restores | -| corrupt inline `xl.meta` | fail closed or heal; no forged inline data | -| range read with offline/corrupt shard | exact range bytes; invalid ranges produce expected S3 errors | -| multipart part resend and concurrent same-part writes | final complete object matches chosen committed parts | -| crash during multipart complete/put/delete | after restart only old or new full version is visible | - -Existing anchors: - -- `crates/e2e_test/src/reliability_disk_fault_test.rs` -- `crates/e2e_test/src/heal_erasure_disk_rebuild_test.rs` -- `crates/e2e_test/src/chaos.rs` - -### Distributed 4-Node / 16-Disk EC - -| Scenario | Required assertions | -| --- | --- | -| node/disk outage while reading large objects | no EOF/truncation; SHA256 manifest matches | -| write while one remote node is down | write follows quorum; later heal reconstructs remote disk | -| remote shard bitrot | degraded read uses clean shards; no bad bytes | -| concurrent GET/PUT/DELETE/List on same key | no 500 for expected conflicts; no dirty reads | -| range GET matrix for large objects | sequential and parallel ranges match full-object hash | -| internode timeout/slow disk | typed error or fallback; no desync | - -This layer is mandatory because single-process unit tests cannot prove RPC, -HTTP/2, timeout, and distributed quorum behavior. - -### Versioning, Delete Markers, and Migration - -| Scenario | Required assertions | -| --- | --- | -| latest delete marker | GET/HEAD/ListObjectVersions match S3 semantics | -| explicit `versionId` for old versions | exact old bytes and metadata | -| suspended/null version | no version ordering regression | -| delete marker during heal/rebalance/decommission | marker visibility and history are preserved | -| orphan directory cleanup | real objects are not purged; tombstones behave correctly | - -### Heal, Rebalance, and Decommission - -| Scenario | Required assertions | -| --- | --- | -| auto heal and admin deep heal | data hash unchanged; `xl.meta` and format data rebuilt | -| heal interruption/restart | idempotent recovery; no dangling temp objects | -| two-pool rebalance with versioned/multipart objects | source and target pools have consistent versions | -| decommission cancel/restart/finalize | target readable; source cleanup safe | -| rebalance/decommission with node outage | progress resumes; no duplicate or missing versions | - -Existing scripts under `scripts/test/decommission_*.sh` should be wrapped into -the `destructive` profile only after they emit machine-readable pass/fail -artifacts. - -## S3 Compatibility and Large Object Gates - -The full profile should include a targeted S3 compatibility subset, not the -entire compatibility suite by default: - -```bash -TESTEXPR="multipart or range or versioning or delete" \ -DEPLOY_MODE=build \ -MAXFAIL=0 \ -./scripts/s3-tests/run.sh -``` - -Large object gates: - -- `scripts/run_get_codec_streaming_smoke.sh` for legacy/codec GET parity; -- `scripts/run_gt1g_get_http_matrix.sh` for sequential and parallel range GET; -- `scripts/run_gt1g_multipart_put_matrix.sh` for multipart PUT paths. - -These should be artifact-producing optional stages in `full` or `destructive` -profiles, not hidden local-only commands. - -## Fuzz and Corpus Gates - -Add bounded fuzz/corpus tests for: - -- `xl.meta` MessagePack and legacy filemeta versions; -- protobuf/RPC payload decoding; -- checksum and bitrot headers; -- range offset/length overflow; -- object names with path traversal, encoded separators, and symlink components; -- huge inline metadata and huge part counts. - -Each corpus failure must save the input bytes and the minimized reproducer under -the suite artifact directory. - -## Coverage Snapshot Target - -The runner enforces unit line coverage for `rustfs-ecstore` in `full` and -`destructive` profiles. The default threshold is 95%, and the target remains -100% for EC-critical paths: - -```bash -scripts/run_ecstore_validation_suite.sh --profile full --unit-coverage-min 95 -scripts/run_ecstore_validation_suite.sh --profile full --unit-coverage-min 100 -scripts/run_ecstore_validation_suite.sh --profile full --unit-coverage-scope crate -``` - -The default hard gate scope is `ec-critical`, covering: +`full` and `destructive` run `cargo llvm-cov -p rustfs-ecstore --lib` and fail when line coverage of the gate scope is below `--unit-coverage-min`. The default minimum and the 100% target for EC read, write, decode, heal, metadata-quorum, and rollback paths are the `UNIT_COVERAGE_*` constants at the top of the runner. `cargo-llvm-cov` must be installed unless `--skip-coverage` is passed explicitly. The default scope `ec-critical` is: - `crates/ecstore/src/erasure/**` - `crates/ecstore/src/set_disk/read.rs` @@ -298,79 +66,103 @@ The default hard gate scope is `ec-critical`, covering: - `crates/ecstore/src/set_disk/core/io_primitives.rs` - `crates/ecstore/src/disk/local.rs` -Minimum release-gate target: +`--unit-coverage-scope crate` measures the whole crate instead; that number is an observation metric and must not hide EC regressions behind unrelated modules. Lowering the minimum requires a documented exception tied to missing testability or unreachable code. Uncovered branches in reconstruction, quorum, and error paths are either intentionally unreachable or tracked. -- `cargo-llvm-cov` must be installed unless `--skip-coverage` is explicitly set; -- `rustfs-ecstore` EC-critical unit line coverage is at least 95%; -- EC read/write/decode/heal/quorum/rollback code should trend toward 100%; -- full-crate unit line coverage is recorded for visibility but is not the EC - hard gate; -- all HIGH rows in this document have positive and negative tests; -- changed EC/read/write/heal lines are covered; -- branch coverage is reviewed for reconstruction, quorum, and error paths; -- uncovered branches are either intentionally unreachable or tracked. +## White-box scenario matrix -The `quick` profile also runs -`cargo test -p rustfs-ecstore --lib -- --test-threads=1` so the white-box smoke -includes all current `rustfs-ecstore` library unit tests without parallel test -context cross-talk, not only focused EC filters. +Each row is a scenario the unit layer covers; a new EC unit test names the row it satisfies. -Coverage artifacts: +### Erasure algorithm (`crates/ecstore/src/erasure/`) -- `target/ecstore-validation//coverage/ecstore/lcov.info` -- `target/ecstore-validation//coverage/ecstore/summary.tsv` -- `target/ecstore-validation//coverage/ecstore/files.tsv` - -Local validation on 2026-07-07 found that the current suite is not yet at the -target: full-crate unit line coverage was 69.32%; the EC-critical scope was -84.32% after the first negative read/write/recovery additions. The lowest -EC-critical files were `set_disk/ops/object.rs`, `io_primitives.rs`, -`disk/local.rs`, `bitrot.rs`, and `encode.rs`. These are gaps to close before -the default 95% gate can pass. - -## Initial Command Set - -Use the runner first: - -```bash -scripts/run_ecstore_validation_suite.sh --profile quick -scripts/run_ecstore_validation_suite.sh --profile full -scripts/run_ecstore_validation_suite.sh --profile destructive -scripts/run_ecstore_validation_suite.sh --profile fuzz -``` - -The split-run equivalent for the quick profile is: - -```bash -cargo test -p rustfs-filemeta --lib -cargo test -p rustfs-ecstore --lib erasure -cargo test -p rustfs-ecstore --lib set_disk::read -cargo test -p rustfs-ecstore --lib set_disk::core::io_primitives -cargo test -p rustfs-ecstore --lib set_disk::tests::test_rename_data_quorum_failure_rolls_back_destination_object -cargo test -p rustfs-ecstore --lib disk::local -cargo test -p rustfs-ecstore --lib -- --test-threads=1 -cargo test --package e2e_test reliability_disk_fault_test -- --nocapture -cargo test --package e2e_test heal_erasure_disk_rebuild_test -- --nocapture -cargo test --package e2e_test namespace_lock_quorum_test -- --nocapture -``` - -Fixture-backed tests should run when the fixture path is present: - -```bash -cargo test -p rustfs-ecstore --test legacy_bitrot_read_test -- --nocapture -cargo test -p rustfs --features rio-v2 storage::minio_generated_read_test --lib -- --ignored --nocapture -``` - -## Multi-Expert Adversarial Review Summary - -Three independent read-only reviews were run for this design: - -| Reviewer | Main challenge | Resulting requirement | +| Area | Scenarios | Assertions | | --- | --- | --- | -| Algorithm correctness | A single run only samples one timing/layout/hash combination. | matrix/property tests plus instrumented reader alignment checks | -| Black-box reliability | Existing tests miss distributed, shard-desync, range+fault, and migration-fault combinations. | add 4-node/16-disk and destructive profiles | -| Security and fault review | Metadata tamper, quorum downgrade, bitrot bypass, and rename atomicity must be end-to-end. | HIGH matrix rows block completion claims | +| Shard geometry | legacy/current shard-size formulas; lengths `0`, `1`, `block-1`, `block`, `block+1`, multi-block tail | no divide-by-zero; shard/file/range offsets match expected | +| Encode/decode | `(data, parity)` sets `2+2`, `4+2`, `8+8`; random payloads; missing shards up to parity | reconstructed data equals original | +| Negative reconstruction | missing shards above parity; inconsistent shard lengths; corrupt surplus parity | typed error, no partial success | +| Source verification | missing data shard plus extra parity source | rebuilt parity matches source parity | +| Legacy compatibility | old shard formula and legacy checksum data | legacy files decode and heal correctly | +| Streaming decode | legacy engine vs RustFS codec engine on the same stripe stream | bytes and errors are equivalent | +| Range output | head/middle/tail/suffix; cross-block and final-short-stripe ranges | exact byte range, no over-read or under-read | -Current verdict: design accepted as a target. The runner exists and captures -artifacts; implementation remains incomplete until the missing matrix rows have -artifact-backed evidence. +### Bitrot and reader alignment (`erasure/coding/bitrot.rs`, `decode.rs`, `set_disk/core/io_primitives.rs`, `set_disk/shard_source.rs`) + +| Area | Scenarios | Assertions | +| --- | --- | --- | +| Hash framing | valid hash+data; wrong hash; truncated hash; truncated data | invalid data never succeeds | +| Short shard | short read under normal hash, `skip_verify`, and hash-none | `UnexpectedEof` or equivalent typed error | +| Lockstep reads | mid-stream data shard failure; pending/timeout reader; final short stripe | each live reader advances exactly one stripe; failed reader retires | +| Adaptive reads | hedged parity fallback and timeout retirement | no shard desync; reconstructed bytes match original | +| Shard source order | out-of-order read completion and missing slots | slots resolve by shard index | +| Deferred readers | data-blocks-first setup opens deferred parity at the correct offset | parity fallback uses aligned data | + +Instrumented readers record shard index, stripe index, read count, offset, and retirement reason. + +### Metadata, quorum, and commit atomicity (`crates/filemeta/src/filemeta/version.rs`, `set_disk/read.rs`, `set_disk/metadata.rs`, `set_disk/ops/object.rs`, `disk/local.rs`) + +| Area | Scenarios | Assertions | +| --- | --- | --- | +| Metadata tamper | same `version_id`/`mod_time`, divergent data dir, parts, ETag, size, checksum, inline flag, erasure distribution | previous committed version or read-quorum error; no arbitrary latest | +| Early stop | valid quorum, stale quorum, corrupt trailing disks, slow trailing disks | early-stop only on safe identity | +| Quorum downgrade | read quorum vs write quorum; delete-marker quorum; version-not-found quorum | no mutation below write quorum | +| Rename atomicity | failure before data rename, after data rename, after metadata rename, cleanup failure | object is old or new; never mixed | +| Rollback | failed commit quorum and stale temp data | rollback preserves old metadata and data | +| Malformed metadata | oversized lengths, bad CRC, invalid versions, invalid UUID/timestamp/enum, huge parts | bounded memory; typed error; no panic | + +## Black-box scenario matrix + +Real S3 and admin behaviour runs through `crates/e2e_test`. Extend `crates/e2e_test/src/chaos.rs` rather than adding ad hoc fault helpers. + +### Single-node 4-disk EC (`reliability_disk_fault_test.rs`, `heal_erasure_disk_rebuild_test.rs`, `chaos.rs`) + +| Scenario | Required assertions | +| --- | --- | +| baseline PUT/GET/HEAD/List for tiny, inline, block-boundary, multi-block, multipart objects | SHA256 manifest matches; metadata is consistent on all disks | +| one disk offline during read | existing objects readable; no corrupted bytes | +| one disk offline during write | write succeeds only when write quorum holds; restored disk is healed | +| above-parity disk loss | GET/PUT fails with a quorum error; no partial bytes accepted | +| corrupt data shard and parity shard | GET returns original bytes or fails closed; read-repair/heal restores | +| corrupt inline `xl.meta` | fail closed or heal; no forged inline data | +| range read with offline/corrupt shard | exact range bytes; invalid ranges produce the expected S3 errors | +| multipart part resend and concurrent same-part writes | final object matches the chosen committed parts | +| crash during multipart complete/put/delete | after restart only the old or the new full version is visible | + +### Distributed 4-node / 16-disk EC (`cluster_concurrency_test.rs`, `namespace_lock_quorum_test.rs`, `stale_multipart_cleanup_cluster_test.rs`) + +Single-process unit tests cannot prove RPC, HTTP/2, timeout, or distributed-quorum behaviour, so this layer is mandatory. + +| Scenario | Required assertions | +| --- | --- | +| node/disk outage while reading large objects | no EOF/truncation; SHA256 manifest matches | +| write while one remote node is down | write follows quorum; later heal reconstructs the remote disk | +| remote shard bitrot | degraded read uses clean shards; no bad bytes | +| concurrent GET/PUT/DELETE/List on the same key | no 500 for expected conflicts; no dirty reads | +| range GET matrix for large objects | sequential and parallel ranges match the full-object hash | +| internode timeout / slow disk | typed error or fallback; no desync | + +### Versioning, delete markers, and migration (`delete_marker_migration_semantics_test.rs`) + +| Scenario | Required assertions | +| --- | --- | +| latest delete marker | GET/HEAD/ListObjectVersions match S3 semantics | +| explicit `versionId` for old versions | exact old bytes and metadata | +| suspended/null version | no version-ordering regression | +| delete marker during heal/rebalance/decommission | marker visibility and history are preserved | +| orphan directory cleanup | real objects are not purged; tombstones behave correctly | + +### Heal, rebalance, and decommission + +| Scenario | Required assertions | +| --- | --- | +| auto heal and admin deep heal | data hash unchanged; `xl.meta` and format data rebuilt | +| heal interruption/restart | idempotent recovery; no dangling temp objects | +| two-pool rebalance with versioned/multipart objects | source and target pools have consistent versions | +| decommission cancel/restart/finalize | target readable; source cleanup safe | +| rebalance/decommission with node outage | progress resumes; no duplicate or missing versions | + +`scripts/test/decommission_*.sh` cover parts of this table but are not runner steps because they emit no machine-readable pass/fail artifact. + +## Large-object and fuzz gates + +Stand-alone harnesses indexed in `scripts/README.md`, not runner steps: `scripts/run_get_codec_streaming_smoke.sh` (legacy vs codec GET parity), `scripts/run_gt1g_get_http_matrix.sh` (sequential and parallel range GET above 1 GiB), `scripts/run_gt1g_multipart_put_matrix.sh` (multipart PUT above 1 GiB). + +The `fuzz` profile delegates to `scripts/fuzz/run.sh`; targets, corpus rules, and crash-reproducer locations are in `fuzz/README.md`. Malformed-storage-input surfaces the corpus must cover: `xl.meta` MessagePack and legacy filemeta versions, RPC payload decoding, checksum and bitrot headers, range offset/length overflow, object names with path traversal or encoded separators, huge inline metadata and part counts. diff --git a/docs/testing/issue-2007-coalescer-delay.md b/docs/testing/issue-2007-coalescer-delay.md deleted file mode 100644 index e0eb484aa..000000000 --- a/docs/testing/issue-2007-coalescer-delay.md +++ /dev/null @@ -1,58 +0,0 @@ -# Backlog #2007 Coalescer Delay Validation - -`scripts/issue_2007_coalescer_prometheus_report.py` is a read-only Prometheus -report helper for validating whether the GET metadata `ReadVersion` coalescer -default can move from `200us` to `50us`. - -The benchmark itself is intentionally external to this helper: use the same -main build, bucket/object set, workload, and -`RUSTFS_BATCH_READ_VERSION_SERVER_PARALLELISM=4` for both cells. Only switch: - -```bash -RUSTFS_GET_METADATA_READ_VERSION_COALESCE=auto -RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS=200 -RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS=50 -``` - -After each measured workload window, collect a report from Prometheus: - -```bash -scripts/issue_2007_coalescer_prometheus_report.py \ - --query-url http://prometheus.example:9090 \ - --profile delay-200us \ - --window 180s \ - --rustfs-selector 'server=~"node[5-8]"' \ - --node-selector 'instance=~"node[5-8].*"' - -scripts/issue_2007_coalescer_prometheus_report.py \ - --query-url http://prometheus.example:9090 \ - --profile delay-50us \ - --window 180s \ - --rustfs-selector 'server=~"node[5-8]"' \ - --node-selector 'instance=~"node[5-8].*"' -``` - -The output is Markdown and is suitable for attaching to the issue alongside the -warp throughput, average latency, p95, p99, and TTFB p99 from the fixed -workload run. - -Required RustFS signals: - -- `grpc_read_version` and `grpc_batch_read_version` outgoing request increases. -- Coalescer batch distribution from - `rustfs_get_metadata_read_version_coalescer_total{event="attempted_batch"}`. -- `batch_read_version_coalescer_wait`, `batch_read_version_rpc_roundtrip`, - `batch_read_version_disk_read`, and `batch_read_version_response_map` p99. - -Required host-cost signals: - -- CPU busy from `node_cpu_seconds_total`. -- Network RX/TX from `node_network_receive_bytes_total` and - `node_network_transmit_bytes_total`. -- Disk read await, average queue depth, and utilization from node-exporter disk - counters. - -If a section reports `UNAVAILABLE`, treat that evidence as missing rather than -zero. Do not use a default-change PR until the `50us` cell has stable -throughput/latency benefit and CPU, network, and disk cost are available and -acceptable. diff --git a/docs/testing/metrics-1649-prometheus-smoke.md b/docs/testing/metrics-1649-prometheus-smoke.md deleted file mode 100644 index 6065f3e45..000000000 --- a/docs/testing/metrics-1649-prometheus-smoke.md +++ /dev/null @@ -1,44 +0,0 @@ -# Backlog #1649 Prometheus smoke - -`scripts/prometheus_metrics_1649_smoke.py` is a read-only environment check for -the metric dimensions delivered by backlog #1649 and issues #1650-#1653. It -uses Prometheus' instant-query API and does not start, stop, reconfigure, or -load RustFS nodes. - -Run the parser and selector self-test without a live environment: - -```bash -python3 scripts/prometheus_metrics_1649_smoke.py --self-test -``` - -For a live cluster, pass a Prometheus base URL (or its `/api/v1/query` -endpoint), one or more expected server label values, and the built-in profile: - -```bash -python3 scripts/prometheus_metrics_1649_smoke.py \ - --query-url http://prometheus.example:9090 \ - --profile backlog-1649 \ - --server rustfs-node1 \ - --server rustfs-node2 -``` - -The profile checks the disk, scanner, ILM, audit, and notification series and -their required labels. It also requires the legacy aggregate audit and -notification series, so an additive label change cannot silently break -existing dashboards. - -Dynamic series retirement is checked with an exact label set after the -scheduler retirement window has elapsed: - -```bash -python3 scripts/prometheus_metrics_1649_smoke.py \ - --query-url http://prometheus.example:9090 \ - --retired 'rustfs_scanner_bucket_drive_result_total|server=node1,bucket=removed,drive=d1,result=success' \ - --retired 'rustfs_audit_total_messages_by_server|server=node1,target_id=removed' -``` - -`--require METRIC|key=value,...` requires a matching series; -`--require-labels METRIC|key1,key2` requires every returned series to carry -the named labels. Use `--bearer` for a bearer token or `--basic` for a -`username:password` credential when Prometheus is protected. Do not put -credentials in committed commands, logs, or issue comments. diff --git a/docs/testing/security-regressions.md b/docs/testing/security-regressions.md index d853dcadd..4abaa342d 100644 --- a/docs/testing/security-regressions.md +++ b/docs/testing/security-regressions.md @@ -1,93 +1,41 @@ # Security Advisory Regression Tests -Every fixed RustFS GitHub Security Advisory (GHSA) should map to at least one -named, discoverable regression test. The convention is: name the test (or a -helper / doc comment on the exact assertion) after the advisory so that +**Use this when:** fixing or reviewing a GHSA, or checking that an advisory's guard actually executes in CI. +**Source of truth:** the test files named below (`rg -i ghsa` finds them); `.config/nextest.toml` for lane membership; `.github/workflows/ci.yml` and `.github/workflows/e2e-replication-nightly.yml` for execution. -```bash -rg -i "ghsa|3p3x|r5qv" -``` +Every fixed RustFS GitHub Security Advisory maps to at least one named regression test. Name the test (or the helper / doc comment on the exact assertion) after the advisory so `rg -i "ghsa|"` finds the guard, and a future fix of a still-open advisory is forced to update its pinned test (red -> green). -finds the guard for any advisory, and a future fix of a still-open advisory is -forced to update its pinned test (red -> green). +## Advisory -> test map -> This is the lightweight inventory. sec-14 (backlog#1151) formalizes the -> written admission policy in `AGENTS.md`; keep this file as the map. - -## Advisory -> test mapping - -| Advisory | Class | Fix PR | Named regression tests | Layer | +| Advisory | Class | Fix | Named regression tests | Layer | | --- | --- | --- | --- | --- | -| [GHSA-3p3x-734c-h5vx](https://github.com/rustfs/rustfs/security/advisories/GHSA-3p3x-734c-h5vx) | Constant-time secret comparison on WebDAV/FTPS password login | rustfs/rustfs#4403 | `assert_ftps_ghsa_3p3x_wrong_credentials_rejected` (`crates/e2e_test/src/protocols/ftps_core.rs`); `GHSA-3p3x` auth-failure block in `test_webdav_core_operations` (`crates/e2e_test/src/protocols/webdav_core.rs`) | e2e (protocols suite) | +| [GHSA-3p3x-734c-h5vx](https://github.com/rustfs/rustfs/security/advisories/GHSA-3p3x-734c-h5vx) | Constant-time secret comparison on WebDAV/FTPS password login | rustfs/rustfs#4403 | `assert_ftps_ghsa_3p3x_wrong_credentials_rejected` (`crates/e2e_test/src/protocols/ftps_core.rs`); `GHSA-3p3x` auth-failure block in `test_webdav_core_operations` (`crates/e2e_test/src/protocols/webdav_core.rs`) | e2e (`e2e-protocols`) | | [GHSA-r5qv-rc46-hv8q](https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q) | Internode RPC authentication must fail closed | rustfs/rustfs#4402 | `ghsa_r5qv_resolve_shared_secret_rejects_default_fallback`, `ghsa_r5qv_verify_rpc_signature_fails_closed_on_missing_or_invalid_auth` (`crates/ecstore/src/cluster/rpc/http_auth.rs`) | unit | -| [GHSA-m77q-r63m-pj89](https://github.com/rustfs/rustfs/security/advisories/GHSA-m77q-r63m-pj89) | STS JWTs signed with shared root secret (intentionally unfixed) | n/a | `test_ghsa_m77q_sts_session_token_signed_with_root_secret` (flow-level pin: signing key == root secret, root-only decode, authorizes) and `test_created_sts_credentials_authorize_with_session_token_claims` (`crates/iam/src/sys.rs`); `token_signing_key` doc (`crates/iam/src/root_credentials.rs`) — pin current by-design behavior; fixing m77q must update red -> green | unit | -| [GHSA-5354-r3w2-34m8](https://github.com/rustfs/rustfs/security/advisories/GHSA-5354-r3w2-34m8) | Service-account parent must stay within caller scope — a non-owner holding `CreateServiceAccountAdminAction` could parent a service account to the root credential and authenticate as owner | rustfs/rustfs#5141 | `ghsa_5354_non_owner_service_account_parent_confined_to_scope` and the `add_service_account_parent_within_scope` invariant it pins (`rustfs/src/admin/handlers/service_account.rs`) | unit | -| [GHSA-3ppv-fx5m-m749](https://github.com/rustfs/rustfs/security/advisories/GHSA-3ppv-fx5m-m749) | Versioned object reads must be authorized against `s3:GetObjectVersion`, not `s3:GetObject` (`get_object`, CopyObject source, UploadPartCopy source) | rustfs/rustfs#5142 | `ghsa_3ppv_versioned_read_selects_get_object_version_action` and the `versioned_read_action` helper it pins (`rustfs/src/storage/access.rs`) | unit | -| [GHSA-v9cp-qfw9-9pfp](https://github.com/rustfs/rustfs/security/advisories/GHSA-v9cp-qfw9-9pfp) | `ForAllValues:`/`ForAnyValue:` negated string operators were transposed — negation was applied to the aggregate quantified result instead of the per-value predicate | pending | `ghsa_v9cp_for_all_values_not_equals_partial_overlap`, `ghsa_v9cp_for_any_value_not_equals_partial_overlap` and the absent-key/positive-quantifier cases beside them (`crates/policy/tests/quantified_negation.rs`) — the value set must partially overlap the policy set, contained or disjoint sets cannot tell the two quantifiers apart | unit (crate test) | -| [GHSA-6r96-hmgc-726c](https://github.com/rustfs/rustfs/security/advisories/GHSA-6r96-hmgc-726c) | Request headers must not populate server-derived IAM condition keys (`userid`, `groups`, `jwt:`/`ldap:` claim names) | pending | `ghsa_6r96_identity_condition_keys_ignore_spoofed_headers`, `ghsa_6r96_claim_condition_keys_ignore_spoofed_headers`, and `test_request_headers_still_reach_conditions` which guards the reserved set from growing too broad (`rustfs/src/auth.rs`) | unit | -| [GHSA-x298-9x87-fvjq](https://github.com/rustfs/rustfs/security/advisories/GHSA-x298-9x87-fvjq) | The anonymous ListObjectVersions -> `s3:ListBucket` fallback must reach the same public-access gates as a direct grant | pending | `ghsa_x298_anonymous_list_object_versions_denied_when_restrict_public_buckets_enabled` (`crates/e2e_test/src/anonymous_access_test.rs`) — asserts 200 before the public-access block is applied so the test proves the gate rather than a broken fallback | e2e | -| [GHSA-g3vq-vv42-f647](https://github.com/rustfs/rustfs/security/advisories/GHSA-g3vq-vv42-f647) | FTPS `MKD` must clear the `s3:CreateBucket` authorization boundary before reaching the backend | pending | `ghsa_g3vq_mkd_denied_before_reaching_backend` (`crates/protocols/src/ftps/driver.rs`) — primes `create_bucket` to succeed so the assertion distinguishes "denied at authorization" from "backend refused" | unit | +| [GHSA-m77q-r63m-pj89](https://github.com/rustfs/rustfs/security/advisories/GHSA-m77q-r63m-pj89) | STS JWTs signed with the shared root secret (intentionally unfixed) | n/a; tests pin the by-design behaviour and must flip red -> green when m77q is fixed | `test_ghsa_m77q_sts_session_token_signed_with_root_secret`, `test_created_sts_credentials_authorize_with_session_token_claims` (`crates/iam/src/sys.rs`); `token_signing_key` doc (`crates/iam/src/root_credentials.rs`) | unit | +| [GHSA-5354-r3w2-34m8](https://github.com/rustfs/rustfs/security/advisories/GHSA-5354-r3w2-34m8) | Service-account parent must stay within caller scope; a non-owner with `CreateServiceAccountAdminAction` could parent a service account to root | rustfs/rustfs#5141 | `ghsa_5354_non_owner_service_account_parent_confined_to_scope`, `ghsa_5354_scope_guard_matches_owner_or_self_scope_for_derived_credentials`, and the `add_service_account_parent_within_scope` invariant they pin (`rustfs/src/admin/handlers/service_account.rs`) | unit | +| [GHSA-3ppv-fx5m-m749](https://github.com/rustfs/rustfs/security/advisories/GHSA-3ppv-fx5m-m749) | Versioned reads (`get_object`, CopyObject source, UploadPartCopy source) authorize against `s3:GetObjectVersion`, not `s3:GetObject` | rustfs/rustfs#5142 | `ghsa_3ppv_versioned_read_selects_get_object_version_action` and the `versioned_read_action` helper it pins (`rustfs/src/storage/access.rs`) | unit | +| [GHSA-v9cp-qfw9-9pfp](https://github.com/rustfs/rustfs/security/advisories/GHSA-v9cp-qfw9-9pfp) | `ForAllValues:`/`ForAnyValue:` negated string operators applied negation to the aggregate instead of the per-value predicate | fixed, GHSA private-fork merge | `ghsa_v9cp_for_all_values_not_equals_partial_overlap`, `ghsa_v9cp_for_any_value_not_equals_partial_overlap` and the absent-key/positive-quantifier cases beside them (`crates/policy/tests/quantified_negation.rs`); the value set must partially overlap the policy set, since contained or disjoint sets cannot tell the quantifiers apart | crate test | +| [GHSA-6r96-hmgc-726c](https://github.com/rustfs/rustfs/security/advisories/GHSA-6r96-hmgc-726c) | Request headers must not populate server-derived IAM condition keys (`userid`, `groups`, `jwt:`/`ldap:` claims) | fixed, GHSA private-fork merge | `ghsa_6r96_identity_condition_keys_ignore_spoofed_headers`, `ghsa_6r96_claim_condition_keys_ignore_spoofed_headers`, and `test_request_headers_still_reach_conditions`, which keeps the reserved set from growing too broad (`rustfs/src/auth.rs`) | unit | +| [GHSA-x298-9x87-fvjq](https://github.com/rustfs/rustfs/security/advisories/GHSA-x298-9x87-fvjq) | Anonymous ListObjectVersions -> `s3:ListBucket` fallback must reach the same public-access gates as a direct grant | fixed, GHSA private-fork merge | `ghsa_x298_anonymous_list_object_versions_denied_when_restrict_public_buckets_enabled` (`crates/e2e_test/src/anonymous_access_test.rs`); asserts 200 before the public-access block is applied so it proves the gate, not a broken fallback | e2e (`e2e-smoke`) | +| [GHSA-g3vq-vv42-f647](https://github.com/rustfs/rustfs/security/advisories/GHSA-g3vq-vv42-f647) | FTPS `MKD` must clear the `s3:CreateBucket` authorization boundary before reaching the backend | fixed, GHSA private-fork merge | `ghsa_g3vq_mkd_denied_before_reaching_backend` (`crates/protocols/src/ftps/driver.rs`); primes `create_bucket` to succeed so the assertion distinguishes "denied at authorization" from "backend refused" | unit (`ftps` feature) | -## Where these run (CI-execution map) +## Where these run -Every security regression must land where CI actually runs it — a named test in -an unexecuted suite is theater. The suites split across three execution paths by -topology: +| Layer | Command | Lane | Guard | +| --- | --- | --- | --- | +| Unit and crate tests (`ghsa_r5qv_*`, the m77q pins, `ghsa_5354_*`, `ghsa_3ppv_*`, `ghsa_6r96_*`, `ghsa_v9cp_*`, `ghsa_g3vq_*`) | `cargo nextest run --profile ci --all --exclude e2e_test` | every PR, `Test and Lint` (required) | none needed; the workspace pass runs every unit and crate test | +| S3-API negative-auth e2e (`negative_sigv4_test`, `presigned_negative_test`, `admin_auth_test`) | `cargo nextest run --profile e2e-smoke -p e2e_test` | every PR, `End-to-End Tests` (report-only) | `scripts/check_security_smoke_count.sh` with the floor in `.config/security-smoke-floor.txt`, run in the `e2e-tests` job; fails when a rename drops one of these modules out of the smoke filter | +| Other S3 e2e guards (`anonymous_access_test`) | `cargo nextest run --profile e2e-smoke -p e2e_test` | every PR, `End-to-End Tests` (report-only) | `scripts/check_test_wiring.py --check-profile e2e-smoke` digest | +| Protocol e2e (`protocols::test_protocol_core_suite`, GHSA-3p3x) | `RUSTFS_BUILD_FEATURES=ftps,webdav,sftp cargo nextest run -j 1 --profile e2e-protocols -p e2e_test` | nightly, `e2e-replication-nightly.yml` job `protocols-nightly`; not PR-gated | `scripts/check_test_wiring.py --check-profile e2e-protocols` digest | -- **Unit tests** — `ghsa_r5qv_*` (`crates/ecstore`), the GHSA-m77q STS - pinning (`crates/iam`), `ghsa_5354_*` / `ghsa_3ppv_*` / `ghsa_6r96_*` - (`rustfs` lib), `ghsa_v9cp_*` (`crates/policy` crate test), and - `ghsa_g3vq_*` (`crates/protocols`) run automatically in the default CI pass - (`cargo nextest run --profile ci --all --exclude e2e_test`) — no special - wiring. This is the CI-executed regression for the RPC fail-closed (r5qv), - STS-signing (m77q), service-account parent-scope (5354), versioned-read - authorization (3ppv), condition-key injection (6r96), quantified-negation - (v9cp), and FTPS MKD authorization (g3vq) advisories. +Notes: - `ghsa_g3vq_*` sits behind `rustfs-protocols`' `ftps` feature, which is off by - default for that crate alone. It still runs in the workspace pass because the - `rustfs` crate defaults to `["ftps", "webdav"]` and cargo unifies features - across the build. Running `cargo test -p rustfs-protocols` on its own silently - skips it — use `--features ftps` in that case. The `protocol-features` CI - matrix covers `swift` and `sftp` only, for the same reason: those are not in - any default feature set, so they need explicit jobs. -- **S3-API negative-auth e2e (e2e-smoke, PR-gated)** — the attacker-facing S3 - auth-rejection suites run on every PR via the `e2e-smoke` nextest profile - (`.config/nextest.toml`), which each spawns its own server on a random port - and is parallel-safe: - - `negative_sigv4_test` — tampered/wrong-key/skewed header SigV4 (sec-1) - - `presigned_negative_test` — expired/tampered/wrong-key presigned URLs (sec-2) - - `admin_auth_test` — non-admin denial + root-credential lifecycle (sec-4) - - A count-floor guard (`scripts/check_security_smoke_count.sh`, floor in - `.config/security-smoke-floor.txt`) runs in the `e2e-tests` CI job and fails if - a rename drops any of these out of the smoke filter (infra-12 mechanism). This - is the sec-5 wiring: those merged suites were dead weight until listed here. -- **Protocol e2e (WebDAV/FTPS constant-time login, GHSA-3p3x)** — lives in the - `e2e_test` protocols suite (`test_protocol_core_suite`), which binds **fixed - ports**, needs the `ftps,webdav` build features, and is `#[serial]`. Those - three properties make it **structurally incompatible** with the random-port, - default-feature, parallel `e2e-smoke` profile: it cannot be a filterset change, - so sec-5 (scoped to a filterset change, global ruling G5) does not wire it. - It runs manually today: - - ```bash - RUSTFS_BUILD_FEATURES=ftps,webdav cargo test --package e2e_test \ - test_protocol_core_suite -- --test-threads=1 --nocapture - ``` - - **Open gap:** GHSA-3p3x's *e2e* layer has no PR-gated CI execution. A protocols - e2e CI lane is a ci-domain concern (a new profile/job, not a filterset change); - it is deliberately out of sec-5's boundary and left as a follow-up. +- `test_protocol_core_suite` is a single `#[tokio::test]` (not `#[serial]`) that binds fixed ports and needs the `ftps,webdav` build features; the nightly job serializes it with `-j 1`. It cannot join the random-port, default-feature `e2e-smoke` profile. Targeted local run: `crates/e2e_test/src/protocols/README.md`. +- `ghsa_g3vq_*` sits behind the `ftps` feature of `rustfs-protocols`, which is off by default for that crate alone. It still runs in the workspace pass because the `rustfs` crate defaults to `["ftps", "webdav"]` and cargo unifies features across the build; `cargo test -p rustfs-protocols` on its own skips it, so pass `--features ftps`. The `protocol-features` matrix in `ci.yml` covers only `swift` and `sftp` for the same reason. ## Adding a new advisory guard 1. Reproduce the advisory's bypass form as a focused negative test. -2. Name the test (or the helper/assertion) `ghsa__*`, or attach a - `GHSA-` doc comment with the advisory URL and fix PR. +2. Name the test (or the helper/assertion) `ghsa__*`, or attach a `GHSA-` doc comment with the advisory URL. 3. Add a row to the table above. -4. Land it where CI runs it (see the map above): a unit guard in the default CI - pass; an S3-API negative-auth e2e in the `e2e-smoke` filter (add the module to - `.config/nextest.toml` and bump `.config/security-smoke-floor.txt`); a - fixed-port protocol e2e in the protocols suite (still manual — see the open - gap above). +4. Land it where it runs (table above): a unit guard needs nothing extra; an S3 e2e guard joins the `e2e-smoke` filter in `.config/nextest.toml` with its digest updated, and a negative-auth module also bumps `.config/security-smoke-floor.txt`; a fixed-port protocol guard goes into the protocols suite (nightly lane). diff --git a/rustfs/src/storage/rpc/http_service.rs b/rustfs/src/storage/rpc/http_service.rs index d910ff42e..41b1719d7 100644 --- a/rustfs/src/storage/rpc/http_service.rs +++ b/rustfs/src/storage/rpc/http_service.rs @@ -747,8 +747,7 @@ async fn handle_walk_dir(req: Request) -> Response { return response_with_status(StatusCode::PAYLOAD_TOO_LARGE, message); } }; - // RUSTFS_COMPAT_TODO(#4648): old clients retry terminal stream failures on an already-used writer. - // Remove after every supported peer version advertises walk-dir stream completion v1. + // RUSTFS_COMPAT_TODO(rustfs-4648): old clients retry terminal stream failures on an already-used writer. Remove after every supported peer version advertises walk-dir stream completion v1. let propagate_completion_errors = match validate_walk_dir_completion_request(&query, &body) { Some(propagate_completion_errors) => propagate_completion_errors, None => { diff --git a/scripts/README.md b/scripts/README.md index 88dd17818..da2bd119c 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -64,7 +64,8 @@ their issue closes. | Entry | Status | Purpose | Wiring / docs | |---|---|---|---| -| `run_hotpath_warp_ab.sh` | ci-gate | Linux warp A/B rig for the hotpath series | performance-ab.yml (scheduled); `docs/operations/hotpath-warp-ab-runbook.md` | +| `run_hotpath_warp_ab.sh` | dev-tool | Linux warp A/B rig for the hotpath series (quick local A/B) | `docs/operations/hotpath-warp-ab-runbook.md` | +| `run_hotpath_warp_abba.sh` | ci-gate | Formal ABBA warp runner (baseline/candidate interleaved, `--rounds >= 3`); the runner performance-ab.yml executes | performance-ab.yml (scheduled); `docs/operations/hotpath-warp-ab-runbook.md` | | `hotpath_warp_ab_gate.sh` | dev-tool | Relative-budget gate evaluated over the warp A/B results | used by `run_hotpath_warp_ab.sh`; hotpath runbook | | `run_internode_grpc_ab_bench.sh` | dev-tool | One-click A/B driver for the internode gRPC optimization stages | `docs/operations/internode-grpc-benchmark-runbook.md` | | `run_internode_transport_baseline.sh` | dev-tool | Internode transport baseline runner | internode runbook; `crates/io-metrics/README.md` | @@ -74,7 +75,8 @@ their issue closes. | `run_pinned_paired_abba_bench.sh` | dev-tool | Pinned RustFS/MinIO paired ABBA benchmark orchestrator for backlog#1432 | `test_pinned_paired_abba_bench.sh` | | `run_get_codec_streaming_smoke.sh` | dev-tool | Local GET benchmark harness for the codec streaming read path | `docs/testing/ecstore-validation-suite-design.md` | | `run_get_1mib_abba_stage_metrics.sh` | dev-tool | Exact-1MiB isolated-host GET ABBA/stage-metrics harness for backlog#1434 | `test_get_1mib_abba_stage_metrics.sh` | -| `issue_2007_coalescer_prometheus_report.py` | dev-tool | Read-only Prometheus report for GET metadata coalescer delay cost validation | `test_issue_2007_coalescer_prometheus_report.sh`; `docs/testing/issue-2007-coalescer-delay.md` | +| `issue_2007_coalescer_prometheus_report.py` | dev-tool | Read-only Prometheus report for GET metadata coalescer delay cost validation; usage in the module docstring | `test_issue_2007_coalescer_prometheus_report.sh` | +| `prometheus_metrics_1649_smoke.py` | dev-tool | Read-only Prometheus instant-query smoke check for the backlog#1649 metric dimensions, required labels, and retired series; usage in the module docstring | `--self-test` | | `run_gt1g_get_http_matrix.sh` | dev-tool | >1 GiB GET HTTP matrix | `docs/testing/ecstore-validation-suite-design.md` | | `run_gt1g_multipart_put_matrix.sh` | dev-tool | >1 GiB multipart PUT matrix | `docs/testing/ecstore-validation-suite-design.md` | | `sample_remote_rustfs_rss.sh` | dev-tool | Remote RustFS PID CPU/RSS TSV sampler for hotpath profiling runs | `test_sample_remote_rustfs_rss.sh`; backlog#1647 | diff --git a/scripts/check_architecture_migration_rules.sh b/scripts/check_architecture_migration_rules.sh index cbc3ccfac..d6f1a600f 100755 --- a/scripts/check_architecture_migration_rules.sh +++ b/scripts/check_architecture_migration_rules.sh @@ -82,7 +82,7 @@ require_source_contains "docs/architecture/global-state-crate-split-plan.md" "ru require_source_contains "docs/architecture/global-state-crate-split-plan.md" "global-state-inventory.md" "global state inventory plan link" require_source_contains "docs/architecture/global-state-inventory.md" "## Global State Classification" "global state inventory classification section" require_source_contains "docs/architecture/global-state-inventory.md" "## Runtime Migration Inventory" "global state inventory migration section" -require_source_contains "docs/architecture/global-state-inventory.md" "GLOBAL_EXPIRY_STATE" "global state inventory first candidate" +require_source_contains "docs/architecture/global-state-inventory.md" "expiry_state_handle" "global state inventory lifecycle expiry-state runtime handle" require_source_contains "docs/architecture/global-state-inventory.md" "## RustFS Owner-Local Static Inventory" "global state inventory RustFS owner-local static section" require_source_contains "docs/architecture/global-state-inventory.md" "KEYSTONE_AUTH" "global state inventory RustFS auth static inventory" require_source_contains "docs/architecture/global-state-inventory.md" "DEADLOCK_DETECTOR" "global state inventory RustFS storage static inventory" diff --git a/scripts/check_doc_paths.sh b/scripts/check_doc_paths.sh index 62fb1b82c..c800ce219 100755 --- a/scripts/check_doc_paths.sh +++ b/scripts/check_doc_paths.sh @@ -2,13 +2,14 @@ # # Fail when an agent-instruction or architecture document references a # repository file path that no longer exists. Keeps CLAUDE.md / AGENTS.md / -# ARCHITECTURE.md / docs/architecture honest after refactors move code. +# ARCHITECTURE.md / docs/ honest after refactors move code. # -# Checked files: all tracked AGENTS.md, CLAUDE.md, ARCHITECTURE.md, and -# docs/architecture/*.md. +# Checked files: all tracked AGENTS.md, CLAUDE.md, ARCHITECTURE.md, and every +# Markdown file under docs/ (architecture, operations, testing, and the index). # # A reference is any token starting with crates/, rustfs/, scripts/, docs/, -# .github/ or .config/ that ends in a known file extension. Tokens containing +# helm/, deploy/, .github/ or .config/ (at a word boundary, so helm/rustfs/... +# is not misread as rustfs/...) that ends in a known file extension. Tokens containing # globs or placeholders, extensionless tokens (HTTP routes, directory # references, org/repo shorthands), and lines containing URLs are skipped. @@ -20,7 +21,7 @@ cd "$ROOT_DIR" FAILURES=0 doc_files() { - git ls-files 'AGENTS.md' '*/AGENTS.md' 'CLAUDE.md' 'ARCHITECTURE.md' 'docs/architecture/*.md' + git ls-files 'AGENTS.md' '*/AGENTS.md' 'CLAUDE.md' 'ARCHITECTURE.md' 'docs/*.md' 'docs/architecture/*.md' 'docs/operations/*.md' 'docs/testing/*.md' } check_file() { @@ -30,8 +31,8 @@ check_file() { # external, not repo paths. # grep exits 1 on no matches; that must not kill the script under pipefail. { grep -vE 'https?://' "$doc" || true; } \ - | { grep -oE '(crates|rustfs|scripts|docs|\.github|\.config)/[A-Za-z0-9_./-]+' || true; } \ - | sed -e 's/[.,;:)]*$//' -e 's:/$::' \ + | { grep -oE '(^|[^A-Za-z0-9_./-])(crates|rustfs|scripts|docs|helm|deploy|\.github|\.config)/[A-Za-z0-9_./-]+' || true; } \ + | sed -E -e 's/^[^A-Za-z0-9_.]//' -e 's/[.,;:)]*$//' -e 's:/$::' \ | sort -u \ | while IFS= read -r ref; do case "$ref" in diff --git a/scripts/issue_2007_coalescer_prometheus_report.py b/scripts/issue_2007_coalescer_prometheus_report.py index 64fc18a84..a7e743d81 100755 --- a/scripts/issue_2007_coalescer_prometheus_report.py +++ b/scripts/issue_2007_coalescer_prometheus_report.py @@ -4,6 +4,23 @@ The script queries Prometheus' instant-query API and prints a Markdown summary for one already-completed workload window. It never writes to RustFS, Prometheus, or scrape targets. + +Benchmark both cells with the same build, object set, workload, and +``RUSTFS_BATCH_READ_VERSION_SERVER_PARALLELISM``; switch only +``RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS`` (200 vs 50), then +report each measured window:: + + scripts/issue_2007_coalescer_prometheus_report.py \ + --query-url http://prometheus.example:9090 --profile delay-200us \ + --window 180s --rustfs-selector 'server=~"node[5-8]"' \ + --node-selector 'instance=~"node[5-8].*"' + scripts/issue_2007_coalescer_prometheus_report.py \ + --query-url http://prometheus.example:9090 --profile delay-50us \ + --window 180s --rustfs-selector 'server=~"node[5-8]"' \ + --node-selector 'instance=~"node[5-8].*"' + +A section printed as ``UNAVAILABLE`` is missing evidence, not zero. Self-test: +``scripts/test_issue_2007_coalescer_prometheus_report.sh``. """ from __future__ import annotations diff --git a/scripts/prometheus_metrics_1649_smoke.py b/scripts/prometheus_metrics_1649_smoke.py index f933ddcab..95e7d1461 100755 --- a/scripts/prometheus_metrics_1649_smoke.py +++ b/scripts/prometheus_metrics_1649_smoke.py @@ -5,6 +5,19 @@ The harness queries Prometheus' instant-query API. It never writes to RustFS, Prometheus, or the scrape targets. A check is ``metric|label=value,...``; ``--require-labels`` accepts ``metric|label1,label2``. ``--retired`` checks that an exact label set is absent after the scheduler's retirement window. + +Examples:: + + python3 scripts/prometheus_metrics_1649_smoke.py --self-test + python3 scripts/prometheus_metrics_1649_smoke.py \ + --query-url http://prometheus.example:9090 --profile backlog-1649 \ + --server rustfs-node1 --server rustfs-node2 + python3 scripts/prometheus_metrics_1649_smoke.py \ + --query-url http://prometheus.example:9090 \ + --retired 'rustfs_scanner_bucket_drive_result_total|server=node1,bucket=removed,drive=d1,result=success' + +Use ``--bearer`` or ``--basic user:pass`` for a protected Prometheus; never put +credentials in committed commands, logs, or issue comments. """ import argparse