Compare commits

..

129 Commits

Author SHA1 Message Date
github-actions[bot] 37808c4bbb chore: update star history 2026-11-17 03:20:24 +00:00
github-actions[bot] b32aa24d47 chore: update star history 2026-11-16 03:24:04 +00:00
github-actions[bot] da9e6c006f chore: update star history 2026-11-15 03:20:40 +00:00
github-actions[bot] f1791b6212 chore: update star history 2026-11-14 03:18:34 +00:00
github-actions[bot] b5c139ea58 chore: update star history 2026-11-13 03:19:58 +00:00
github-actions[bot] cf958e5e9d chore: update star history 2026-11-12 03:19:49 +00:00
github-actions[bot] b0db2eb6f4 chore: update star history 2026-11-11 03:20:08 +00:00
github-actions[bot] 5c666cc502 chore: update star history 2026-11-10 03:19:33 +00:00
github-actions[bot] 3ee0d74aeb chore: update star history 2026-11-09 03:21:54 +00:00
github-actions[bot] f77fdbcc0e chore: update star history 2026-11-08 03:20:01 +00:00
github-actions[bot] 11a53b910f chore: update star history 2026-11-07 03:18:40 +00:00
github-actions[bot] f2c2336f12 chore: update star history 2026-11-06 03:19:46 +00:00
github-actions[bot] 95e86d6555 chore: update star history 2026-11-05 03:20:15 +00:00
github-actions[bot] 773f11be84 chore: update star history 2026-11-04 03:18:46 +00:00
github-actions[bot] dc0eeb0d7e chore: update star history 2026-11-03 03:19:33 +00:00
github-actions[bot] 1a95ce76e9 chore: update star history 2026-11-02 03:21:34 +00:00
github-actions[bot] 7ab8909585 chore: update star history 2026-11-01 03:21:27 +00:00
github-actions[bot] 6bb2e2c55a chore: update star history 2026-10-31 03:18:29 +00:00
github-actions[bot] 4c5ee747a5 chore: update star history 2026-10-30 03:23:05 +00:00
github-actions[bot] 006582aaa0 chore: update star history 2026-10-29 03:23:25 +00:00
github-actions[bot] aa45b71952 chore: update star history 2026-10-28 03:22:53 +00:00
github-actions[bot] 2bdff2ac90 chore: update star history 2026-10-27 03:31:42 +00:00
github-actions[bot] ef2dced7e8 chore: update star history 2026-10-26 03:34:28 +00:00
github-actions[bot] 9345e75673 chore: update star history 2026-10-25 03:30:27 +00:00
github-actions[bot] a2010e8369 chore: update star history 2026-10-24 03:28:36 +00:00
github-actions[bot] d9cedf9141 chore: update star history 2026-10-23 03:30:30 +00:00
github-actions[bot] 88a9523384 chore: update star history 2026-10-22 03:30:08 +00:00
github-actions[bot] 1c92ce2278 chore: update star history 2026-10-21 03:30:30 +00:00
github-actions[bot] 4ee5e25ca7 chore: update star history 2026-10-20 03:29:43 +00:00
github-actions[bot] 2904cddca3 chore: update star history 2026-10-19 03:34:09 +00:00
github-actions[bot] 2b01be62e8 chore: update star history 2026-10-18 03:28:46 +00:00
github-actions[bot] 56b9c193b9 chore: update star history 2026-10-17 03:27:28 +00:00
github-actions[bot] 021e6f437b chore: update star history 2026-10-16 03:29:31 +00:00
github-actions[bot] c81fc59ba8 chore: update star history 2026-10-15 03:29:21 +00:00
github-actions[bot] 4bd4c0bb61 chore: update star history 2026-10-14 03:29:44 +00:00
github-actions[bot] 849194cc64 chore: update star history 2026-10-13 03:37:56 +00:00
github-actions[bot] 02733d400d chore: update star history 2026-10-12 03:42:50 +00:00
github-actions[bot] f7ffe0ebd4 chore: update star history 2026-10-11 03:37:14 +00:00
github-actions[bot] 10fed191ef chore: update star history 2026-10-10 03:33:17 +00:00
github-actions[bot] 026f4f0da6 chore: update star history 2026-10-09 03:35:32 +00:00
github-actions[bot] b79fa68776 chore: update star history 2026-10-08 03:34:35 +00:00
github-actions[bot] 2d13353a66 chore: update star history 2026-10-07 03:34:37 +00:00
github-actions[bot] f1fa52a652 chore: update star history 2026-10-06 03:32:13 +00:00
github-actions[bot] 13eaef780f chore: update star history 2026-10-05 03:39:49 +00:00
github-actions[bot] 3302904287 chore: update star history 2026-10-04 04:31:58 +00:00
github-actions[bot] 2a0c88e415 chore: update star history 2026-10-03 03:39:02 +00:00
github-actions[bot] f6a95a1318 chore: update star history 2026-10-02 03:31:41 +00:00
github-actions[bot] 17f104a6e1 chore: update star history 2026-10-01 03:37:07 +00:00
github-actions[bot] 29b439bc5c chore: update star history 2026-09-30 03:32:58 +00:00
github-actions[bot] aa5d4c6b1b chore: update star history 2026-09-29 03:32:30 +00:00
github-actions[bot] 036b5871fc chore: update star history 2026-09-28 03:35:53 +00:00
github-actions[bot] b95d6f0934 chore: update star history 2026-09-27 03:31:30 +00:00
github-actions[bot] 8dd6cb4884 chore: update star history 2026-09-26 03:28:36 +00:00
github-actions[bot] e73e0e6d06 chore: update star history 2026-09-25 13:50:41 +00:00
github-actions[bot] ab8f0af01e chore: update star history 2026-09-24 03:29:42 +00:00
github-actions[bot] 11dbde2ecd chore: update star history 2026-09-23 03:30:18 +00:00
github-actions[bot] cf79592e09 chore: update star history 2026-09-22 03:29:55 +00:00
github-actions[bot] 0dd09b53c3 chore: update star history 2026-09-21 03:32:14 +00:00
github-actions[bot] 4850b7cb91 chore: update star history 2026-09-20 03:29:29 +00:00
github-actions[bot] d1d5d93e6e chore: update star history 2026-09-19 03:28:07 +00:00
github-actions[bot] df9fc42380 chore: update star history 2026-09-18 03:28:36 +00:00
github-actions[bot] b2bec0ce67 chore: update star history 2026-09-17 03:29:27 +00:00
github-actions[bot] 4ed75c8f6d chore: update star history 2026-09-16 03:29:22 +00:00
github-actions[bot] c7ca5b68a6 chore: update star history 2026-09-15 03:29:59 +00:00
github-actions[bot] 7b6a036412 chore: update star history 2026-09-14 03:31:57 +00:00
github-actions[bot] 4f759d6b2f chore: update star history 2026-09-13 03:29:08 +00:00
github-actions[bot] f36199d46f chore: update star history 2026-09-12 03:27:24 +00:00
github-actions[bot] f33207fcab chore: update star history 2026-09-11 03:28:39 +00:00
github-actions[bot] b382228604 chore: update star history 2026-09-10 03:29:07 +00:00
github-actions[bot] fea1619f01 chore: update star history 2026-09-09 03:29:12 +00:00
github-actions[bot] 16a0b45521 chore: update star history 2026-09-08 03:28:34 +00:00
github-actions[bot] 5913b0cb87 chore: update star history 2026-09-07 13:43:14 +00:00
github-actions[bot] aced3e39ad chore: update star history 2026-09-07 03:30:53 +00:00
github-actions[bot] 1f0c780a5a chore: update star history 2026-09-06 03:28:22 +00:00
github-actions[bot] 87245040b3 chore: update star history 2026-09-05 03:27:31 +00:00
github-actions[bot] a02777cced chore: update star history 2026-09-04 03:28:50 +00:00
github-actions[bot] 86bb6d6631 chore: update star history 2026-09-03 04:42:32 +00:00
github-actions[bot] d7c25e3f99 chore: update star history 2026-09-02 03:29:53 +00:00
github-actions[bot] 6891756e81 chore: update star history 2026-09-01 03:31:22 +00:00
github-actions[bot] 313d13b1ff chore: update star history 2026-08-31 03:30:47 +00:00
github-actions[bot] dcec1a196d chore: update star history 2026-08-30 03:28:19 +00:00
github-actions[bot] 8f3e1540e9 chore: update star history 2026-08-29 03:25:48 +00:00
github-actions[bot] 03bd3960a7 chore: update star history 2026-08-28 05:54:36 +00:00
github-actions[bot] 17ba215c01 chore: update star history 2026-08-27 05:02:15 +00:00
github-actions[bot] bbc45cef2f chore: update star history 2026-08-26 03:35:37 +00:00
github-actions[bot] 069fa0c5d5 chore: update star history 2026-08-25 03:31:39 +00:00
github-actions[bot] 5032bfff50 chore: update star history 2026-08-24 03:34:58 +00:00
github-actions[bot] 40b7c2dbd0 chore: update star history 2026-08-23 03:33:33 +00:00
github-actions[bot] 17f8ed6419 chore: update star history 2026-08-22 03:28:40 +00:00
github-actions[bot] 9ede21b481 chore: update star history 2026-08-21 03:33:54 +00:00
github-actions[bot] c594b59b98 chore: update star history 2026-08-20 03:30:43 +00:00
github-actions[bot] a7f8982a5a chore: update star history 2026-08-19 03:30:42 +00:00
github-actions[bot] 9b3f0512d7 chore: update star history 2026-08-18 03:29:51 +00:00
github-actions[bot] c849178146 chore: update star history 2026-08-17 03:32:32 +00:00
github-actions[bot] 13abaaec40 chore: update star history 2026-08-16 03:32:17 +00:00
github-actions[bot] 08e015dc86 chore: update star history 2026-08-15 03:26:53 +00:00
github-actions[bot] 7e420b6633 chore: update star history 2026-08-14 03:56:23 +00:00
github-actions[bot] e41ecfc4aa chore: update star history 2026-08-13 03:57:04 +00:00
github-actions[bot] 4a4d70a28b chore: update star history 2026-08-12 03:56:05 +00:00
github-actions[bot] 0c9bcc24f2 chore: update star history 2026-08-11 03:51:32 +00:00
github-actions[bot] 531c477eb0 chore: update star history 2026-08-10 03:54:31 +00:00
github-actions[bot] df15f3a51a chore: update star history 2026-08-09 03:50:32 +00:00
github-actions[bot] ec537f8a00 chore: update star history 2026-08-08 03:45:26 +00:00
github-actions[bot] ab8039e8d5 chore: update star history 2026-08-07 04:09:46 +00:00
github-actions[bot] 3531aca97c chore: update star history 2026-08-06 04:22:05 +00:00
github-actions[bot] 58b7735e9a chore: update star history 2026-08-05 04:21:15 +00:00
github-actions[bot] 8352eaaea3 chore: update star history 2026-08-04 04:21:47 +00:00
github-actions[bot] 02bcf54ea8 chore: update star history 2026-08-03 04:25:38 +00:00
github-actions[bot] 0ee565c7a6 chore: update star history 2026-08-02 04:24:13 +00:00
github-actions[bot] ff5ea0b1fc chore: update star history 2026-08-01 04:23:41 +00:00
github-actions[bot] 4fbfc77851 chore: update star history 2026-07-31 04:24:04 +00:00
github-actions[bot] 3bf4fbde1a chore: update star history 2026-07-30 04:18:53 +00:00
github-actions[bot] c43ed581d2 chore: update star history 2026-07-29 04:21:42 +00:00
github-actions[bot] 686d0e8df7 chore: update star history 2026-07-28 04:20:51 +00:00
github-actions[bot] d0193a9407 chore: update star history 2026-07-27 04:28:05 +00:00
github-actions[bot] 5a1b4bbd45 chore: update star history 2026-07-26 04:24:45 +00:00
github-actions[bot] 70c4282a5f chore: update star history 2026-07-25 04:21:07 +00:00
github-actions[bot] 03fc8e81cc chore: update star history 2026-07-24 04:22:29 +00:00
github-actions[bot] aa537f8a7b chore: update star history 2026-07-23 04:22:49 +00:00
github-actions[bot] cf3d4d663e chore: update star history 2026-07-22 04:23:13 +00:00
github-actions[bot] c0d454e6b7 chore: update star history 2026-07-21 04:22:23 +00:00
github-actions[bot] 31fbde559c chore: update star history 2026-07-20 04:28:07 +00:00
github-actions[bot] 8cf5d6906d chore: update star history 2026-07-19 04:24:01 +00:00
github-actions[bot] 125ac23907 chore: update star history 2026-07-19 02:23:45 +00:00
github-actions[bot] e72cdfe581 chore: update star history 2026-07-18 09:50:15 +00:00
Zhengchao An af5a5f847e chore: update repository visuals 2026-07-18 17:21:35 +08:00
github-actions[bot] 0a9ecc3cbf chore: update star history 2026-07-18 08:50:41 +00:00
Zhengchao An 75aab297ca chore: update star history 2026-07-18 13:17:05 +08:00
Zhengchao An 4571fd76ad chore: initialize star history 2026-07-18 12:48:47 +08:00
3901 changed files with 4440 additions and 2338790 deletions
@@ -1,13 +0,0 @@
# Adversarial Review Shape
Use when root `AGENTS.md` triggers adversarial validation or for a substantial PR review. Paths below are repository-relative. The root finding standard and completion rule apply; selecting a lens does not require finding a defect.
Risk and review shape:
- **Exempt:** documentation, comments, formatting, or typos with no runtime, build, test, or agent-execution effect.
- **Mechanical:** changes shown to preserve behavior, such as a symbol rename with unchanged callers or a file move with equivalent wiring. Run correctness and simplicity lenses. Classify tests, tooling, CI, and agent instructions by the behavior and authority they change, not their file type; deleting regression protection or changing failure propagation, publishing, or destructive actions can be high risk.
- **Standard:** localized behavior changes. Run one integrated final-diff pass covering correctness, simplicity, and test coverage; add only domain lenses matched by the diff.
- **High risk / substantial PR review:** high risk includes locking, erasure/quorum/heal, replication, multipart, RPC, lifecycle/tiering, persistence/fsync, IAM/KMS/auth, cryptography, on-disk/on-wire formats, and S3-visible semantics. Cover the applicable lenses from two perspectives: first trace the intended contract and implementation, then independently challenge its assumptions, failure paths, and evidence. Use separate reviewers when delegation is authorized and useful, or fresh sequential passes otherwise. Choose staffing and split lenses to fit the risk; a fixed reviewer count or duplicated review is not the goal.
- **Outbound client defaults:** what `TargetClient`, `PutObjectOptions`, or the remote SDK configuration sends to every replication or migration target is high risk for every target class even when the change fixes one. Follow the SOP in `docs/postmortems/2026-09-03-replication-checksum-default-regression.md`: run the outbound target matrix, document each new env knob in the same PR, and list verified and unverified target classes in the PR Impact section.
Available domain lenses are security, concurrency/durability, compatibility, and performance; select only those matched by the change.
-79
View File
@@ -1,79 +0,0 @@
# Implementation Rules
These implementation, style, and quality rules apply to all contributions, regardless of author or editor. Paths below are repository-relative. Read only the relevant sections during read-only review; select verification from the root `AGENTS.md`.
## Shared Style Contract
- Use the toolchain declared in `rust-toolchain.toml`, respect the minimum supported Rust version in `Cargo.toml`, and let `rustfmt.toml` and the configured Rust/Clippy lints define mechanical style. Do not override repository settings with personal formatter or lint preferences.
- Follow the owning module's organization, imports, and domain terminology where repository rules leave a choice. Existing violations do not justify new ones; avoid unrelated cleanup in a focused change.
- Rules using "must" or "do not" are requirements. Rules using "prefer" are defaults; depart when correctness, ownership, performance, or readability benefits justify it. Explain consequential tradeoffs, not every idiomatic choice. Where several approaches meet the contract equally well, the implementer chooses.
- Follow the declared toolchain locally and the affected CI lane's toolchain for reproduction. The repository currently tracks `stable`; for a formatter/lint mismatch, compare `rustc -Vv`, `cargo fmt --version`, and `cargo clippy --version` with the failing run before changing code. Toolchain upgrades must keep local and CI settings aligned and pass their affected gates; do not silently raise `rust-version` or change lint policy to make a check pass.
## Worktree and Disk Hygiene
- For a new task, start from the latest `origin/main` unless a different base is requested, and confirm the requested change is not already present. When continuing existing work or a PR, preserve its branch, base, and task-owned changes; do not restart it from `main`.
- An existing clean, isolated task worktree is sufficient. Create another worktree only when the current checkout is shared, dirty with unrelated work, or belongs to another task.
- Never commit from a shared checkout.
- Use a task-specific branch named `<type>/<topic>`, such as `fix/...`, `feat/...`, `test/...`, or `docs/...`, unless the user specifies a name.
- Do not include agent, tool, contributor, account, or organization names in branch names.
- Push to the user-requested remote or the repository's configured push remote. Do not hard-code or infer a remote from an account name.
- Check free space before artifact-heavy builds, tests, coverage, or downloads. Re-check before a broad gate when space is tight.
- Before handoff, remove the Cargo target directories and temporary worktrees the task created unless they stay intentionally or hold unmerged work. Identify build output by Cargo's markers (`.rustc_info.json`, `CACHEDIR.TAG`), not by directory name: `target` is also a source directory here.
- Remove only task-owned temporary/build artifacts. Never delete another task's worktree or uncommitted data.
- At handoff, mention disk or cleanup details only when they affected execution or artifacts/worktrees remain intentionally.
- For Docker verification, ensure the tested image contains the intended binary. `buildx build` without `--load` leaves the image in the build cache; use the Makefile's loaded image and rebuild without cache only when stale artifacts are evidenced.
## Change Style
- Preserve unrelated control flow. Restructure touched code only when needed for the requested behavior or an explicitly requested refactor; style preferences alone do not justify changing execution order or cleanup behavior.
- Prefer a direct local edit over new files, wrappers, managers, or speculative abstractions.
- Add a helper only when it removes current duplication, names a real domain boundary, or isolates a non-trivial invariant. Keep helpers near the code they support when the module's organization permits it.
- Remove an in-scope path superseded by the change. If compatibility requires it, adapt at the boundary to one canonical core and use the repository's `RUSTFS_COMPAT_TODO` policy.
- Comments explain non-obvious invariants or reasons. Do not narrate code or record change history.
- Mention unrelated problems when useful; do not fix them in a narrow task.
## Rust Style
These repository-owned rules incorporate selected ideas from the [OpenCode style guide](https://github.com/anomalyco/opencode/blob/dev/AGENTS.md#style-guide), adapted to RustFS. That link is background, not an additional rule source. Apply these defaults to new or changed code; correctness and domain invariants take priority.
- Default to immutable `let` bindings; use `mut` for actual state changes. Prefer `if` or `match` expressions when branches compute a value instead of initializing a placeholder and reassigning it.
- Use type inference for obvious local values. Add annotations when inference needs help or the numeric, collection, or error type matters to understanding the code. Keep explicit types in function signatures and public contracts.
- Keep the main path readable with guard clauses, `let ... else`, and `?` when immediate propagation is correct. Preserve explicit handling for recovery, cleanup, aggregation, and quorum decisions; `if`/`else` expressions and exhaustive `match` remain idiomatic.
- Prefer readable iterator chains for straightforward mapping and filtering. Keep loops when they make state changes, early exits, or async operations clearer. Do not add intermediate collections or clones just to use iterators.
- Inline trivial single-use expressions only when a binding adds no meaning. Keep names that explain domain concepts, aid debugging, or make ownership and guard lifetimes clear. Do not change lock release or drop timing for style.
- Keep pure parsing, validation, and option-building helpers synchronous unless an async interface contract requires otherwise. This does not make blocking I/O or CPU-heavy work safe to run on async executor threads; offload it when needed to keep those threads responsive.
## Reuse and Boundary Rules
- Before adding helpers, constants, fixtures, or wrappers, search the touched code and domain owner; check `crates/utils`, `crates/common`, and direct dependencies when they could own the behavior. Stop once the relevant reuse boundary is clear.
- Reuse requires matching semantics: normalization, error types, deadlines, durability, and compatibility must fit the call site. A narrowly named local helper is better than forced reuse with different semantics.
- Validate untrusted input at the boundary that owns it, then pass validated types inward without repeating checks whose guarantees still hold. Deserializing into a Rust type alone does not establish domain validity. Values newly read from disk, RPC, persistence, or another version must be validated at that consuming boundary.
- For destructive actions such as delete or overwrite and for quorum decisions, check the relevant identity, generation, and authority under the operation's existing synchronization or precondition contract. A separate unguarded recheck does not prevent a check/use race.
- Every new control-flow branch needs a concrete triggering input/state. For decoded or peer data, corruption and mixed-version input are valid triggers.
- Required values must return a typed error when absent or corrupt; do not use a default that converts corruption into a plausible result.
- Attach error context once where it is actionable. Do not erase typed errors below aggregation or quorum layers.
## Rust Boundaries
- Production `unwrap`/`expect` must follow a type guarantee or checked invariant; explain only non-obvious guarantees. Stricter scoped prohibitions, including `crates/AGENTS.md`, still apply. Invalid client, disk, or peer data must return errors rather than trigger panics.
- Public APIs do not return `Result<_, String>`. Library APIs use domain errors unless intentional erasure is part of the boundary contract; expose a stored inner error through `Error::source()`.
- Numeric conversions must not silently truncate. Use fallible conversions for untrusted values; validate finiteness, sign, and range before float-to-integer conversion. Clamp/saturate only when required by the domain.
- Do not add crate-root `#![allow(dead_code)]`.
## Naming
Use Rust API naming: `SCREAMING_SNAKE_CASE` constants/statics, `snake_case` functions/variables/modules, and `PascalCase` types/traits. Name domain concepts and include units where a value would otherwise be ambiguous; do not optimize for short names. Do not rename public, serialized, or persisted fields merely to match a local style preference, or rename unrelated existing violations.
## API and Operational Contracts
- For new or changed public APIs, document the caller-visible behavior and relevant `# Errors`, `# Panics`, and `# Safety` conditions. Include a useful example or link to one when usage is non-obvious; avoid boilerplate sections that say nothing. Follow the [Rust API documentation guidelines](https://rust-lang.github.io/api-guidelines/documentation.html) within the scope of the change.
- For async or storage APIs, state cancellation, retry/idempotency, partial-completion, ownership, and durability guarantees when callers rely on them. Make failure cleanup and commit points explicit in code and tests; do not imply atomicity that the implementation does not provide.
- For protocol, persisted-format, configuration, dependency, or operational changes, check affected consumers and supported versions. Update relevant docs/examples and describe compatibility, upgrade/rollback limits, and new configuration defaults when they change. New dependencies must solve a current need and fit the workspace's license, toolchain, and feature policy.
- For hot-path or resource-management changes, check allocation/copying, blocking, unbounded work, and lock duration where affected. Support performance claims with an appropriate measurement or clearly label them as expectations; a benchmark is not required for unrelated changes.
## Behavioral Tests
- For behavior changes, add or update the narrowest test that calls the real implementation and asserts an observable result. Existing tests can provide coverage when they directly exercise the changed behavior; do not duplicate production logic as the test oracle.
- Cover the changed contract's relevant failure and boundary cases, not only success. A bug regression should fail without the fix; do not broaden assertions or rely only on a successful call when the result or side effect is what matters.
- Follow `docs/testing/README.md` for test layers, runners, isolation, synchronization, and flaky tests. Use controlled clocks or explicit synchronization for time/concurrency tests instead of fixed sleeps, and keep waits bounded.
- Report the checks actually run and their outcomes, including skipped or blocked checks and the behavior they leave unverified. Do not describe focused local checks as full CI coverage.
-31
View File
@@ -1,31 +0,0 @@
# Git and Pull Request Rules
Applies to commits, pushes, PRs, and issue/discussion actions. Paths below are repository-relative. User authorization and root `AGENTS.md` still govern scope.
## Final PR Preflight
Before creating or updating a PR, reuse completed review and verification:
- Verify the actual base (normally `origin/main`) and the complete task diff, including file names and whitespace. Exclude secrets, logs, generated artifacts, and unrelated edits. Retain an existing PR's base unless requested.
- Confirm the final diff passed the root verification tier; fix task-owned failures and run missing scoped checks. Report unresolved required checks or authority without expanding the task. Do not start another general review.
- Keep the English Conventional Commit title at most 72 characters. Use the template headings, actual checks, material risks, and rollback notes.
- Immediately before writing to GitHub, confirm the head and task diff are unchanged. Rerun only checks invalidated by edits or relevant state changes.
## Pull Request Lifecycle
- Creating or updating a PR includes one immediate snapshot of checks, mergeability, reviews, and unresolved threads.
- Unless the user explicitly requests monitoring, a release workflow requires it, or an automation already owns it, hand off after the PR is open with the current state and next event to watch. Do not delay ordinary handoff with fixed quiet-period sleeps.
- For requested monitoring, use event-driven or bounded waits. Report only state changes, actionable failures, or a meaningful prolonged delay.
- Investigate failures/comments before changing code. Monitoring alone authorizes observation and diagnosis; fixes, pushes, replies, and thread resolution require their own existing task authority. When authorized, fix task-attributable issues, rerun affected verification, and resume the requested monitor. Otherwise report the concrete failure and next action without expanding scope.
- Merge only with existing user authorization and satisfied applicable reviewer, check, and branch-protection requirements. Reviewer approval does not grant merge authority; user authorization does not bypass repository protections.
- After an observed merge, verify the commit reached the base, then clean the task worktree/branch when safe. Preserve unmerged work for closed PRs unless deletion was explicitly authorized.
## Git and PR Baseline
- Follow Conventional Commits; keep the subject at most 72 characters.
- Source comments, commits, PR titles, and PR bodies are in English.
- Keep every heading from `.github/pull_request_template.md`; use `N/A` where needed and include commands actually run.
- Use `--body-file` for multiline `gh pr create`/`gh pr edit` content.
- PR/issue/discussion content must not contain the literal sequence `\n` or hard-wrapped prose paragraphs.
- Do not include local absolute paths or tool-specific labels/prefixes in GitHub content.
- Resolve review threads after the underlying issue is fixed. If declining a suggestion, reply with a short evidence-based reason.
-13
View File
@@ -1,13 +0,0 @@
---
name: issue-triage
description: Verify issue completion against current implementation and merged work. Use for issue triage; comment, close, or change labels only when authorized.
---
# Issue Triage
- Resolve the issue repository separately from the implementation repository: `rustfs/backlog` tracks work in `rustfs/rustfs`. Pass `--repo` explicitly to GitHub queries.
- Read the issue body, relevant comments, linked PRs, checklist, and sub-issues with `gh issue view`. Search explicit links, the full issue URL, qualified references, and subject keywords; an empty search result does not prove implementation is absent.
- Fetch the implementation base and inspect its current code. For each merged candidate, verify `git merge-base --is-ancestor <merge-commit> <remote>/<base>`; a title, commit message, or merged state alone is insufficient. A local checkout may be stale or on another branch.
- Verify every checklist/sub-issue before declaring completion. For batches, paginate the entire requested issue scope, exclude PR entries, and filter by author only when requested.
- Recommend closing only when all requested behavior is present or evidence shows the issue is superseded. Otherwise name what remains. A status request is read-only; reuse existing authority for comments/closing/labels, and follow [Git and PR rules](../../references/pull-requests.md) before posting. Use Chinese for `rustfs/backlog`, existing repository labels only, and a closing comment naming the verified behavior and PRs.
Report the issue, current state, implementation evidence, remaining work, verdict, and action actually taken. Use a table for batches. Write multiline comments via `--body-file`; when closing, post the prepared comment before closing without an inline multiline body.
@@ -1,43 +0,0 @@
---
name: plugin-contract-guard
description: Guard changes to target-plugin manifests, extension schemas, admin catalog/instance contracts, secret redaction, and external-plugin install policy. Use when a diff changes those contracts in crates/targets, crates/extension-schema, or admin plugin/extension handlers; path membership alone, comments, and unrelated runtime internals do not trigger it.
---
# Plugin & Extension Contract Guard
The "plugin system" spans four surfaces that must stay consistent:
| Surface | Location |
|---|---|
| Manifests & registry | `crates/targets/src/{manifest,plugin}.rs` |
| Install/enable planning (control plane) | `crates/targets/src/control_plane.rs` |
| Extension schemas | `crates/extension-schema/src/lib.rs`, `crates/targets/src/catalog/extension.rs` |
| Admin API contract | `rustfs/src/admin/plugin_contract.rs`, `handlers/{plugins_catalog,plugins_instances,extensions,target_descriptor}.rs` |
## Hard invariants (verify before merging)
1. **Secrets have one source of truth.** Secret config keys are declared only in the plugin manifest (`TargetPluginManifest.secret_fields`, `crates/targets/src/manifest.rs`) and flow to admin via `AdminTargetSpec.secret_fields`. Never add a hand-maintained per-service secret table in a handler; if redaction misses a field, fix the manifest.
2. **Redaction must round-trip.** Instance GET responses replace secret values with `***redacted***` (`REDACTED_SECRET_VALUE` in `plugins_instances.rs`). Instance PUT restores the stored secret when it receives that placeholder back (`restore_redacted_secret_values`). Any new read or write path for target config must keep both halves: redact on the way out, restore the placeholder on the way in. The placeholder literal must never be persisted.
3. **Fixtures never reach production responses.** `example_external_webhook_plugin()` (`crates/targets/src/catalog/mod.rs`) is a test/demo fixture for control-plane planning tests. Production catalog/extension handlers must not include it; regression tests (`plugin_catalog_never_exposes_example_or_external_fixtures`, `extension_catalog_never_exposes_example_or_external_fixtures`) enforce it.
4. **External plugin flow is planning-only and deny-by-default.** `plan_external_target_plugin_action` returns decisions, it executes nothing. `TargetPluginExternalFlowGate::default()` is fully closed and `TargetPluginInstallPolicy::default().allowed_download_hosts` is empty — keep it that way; tests opt in via explicit policies. Install validation requires https, an allowlisted host, a full 64-hex-char sha256 digest, signature and provenance URIs, and an artifact matching the host `target_triple`.
5. **Custom target types must not collide.** Unknown target types get an interned unique `custom:<type>` plugin id (`custom_plugin_id` in `manifest.rs`). Custom plugins with secrets must register via `TargetPluginDescriptor::with_manifest` and declare `secret_fields`; `::new` derives a manifest with no secrets.
## Changing the admin JSON contract
- Shapes are locked twice in `plugin_contract.rs` tests: insta snapshots (`rustfs/src/admin/snapshots/`) plus literal `json!` assertions. Update both deliberately; a shape change is a console-facing API change.
- Field naming is `snake_case`, except discovery blocks (`runtimeCapabilities`, `clusterSnapshot`, `extensionsCatalog`) which are camelCase **by cross-endpoint convention** (same shape in `system.rs`, `console.rs`, `pools.rs`). Do not "fix" that inconsistency locally.
- Contract types deliberately duplicate `rustfs_targets` types (anti-corruption layer). Add a `From` impl; do not serialize internal types directly.
## Handler conventions
- Every new admin plugin/extension route needs authorization at the top of `call` and an `include_str!` guard test asserting it (repo-wide pattern — see `plugin_instance_handlers_require_admin_authorization_contract`). The source guard checks wiring, not runtime authorization; use focused behavior evidence for changed allow/deny semantics.
- Reads use `GetBucketTargetAction` (instances) or `ServerInfoAdminAction` (catalogs); writes use `SetBucketTargetAction`.
- Refresh persisted module switches once per request (`refresh_persisted_module_switches`), then evaluate the sync `module_disabled_block_reason` per domain — do not re-read the store per domain or per instance.
## Generic bounds
Event-payload generics use the `PluginEvent` blanket trait (`crates/targets/src/plugin.rs`). Do not respell `Send + Sync + 'static + Clone + Serialize + DeserializeOwned`.
-20
View File
@@ -1,20 +0,0 @@
---
name: pr-review
description: Review a GitHub PR using its exact base/head and repository risk policy. Use for requested PR reviews; publish or fix only within conversation authorization.
---
# PR Review
1. Resolve the PR repository and remote; the current checkout may belong elsewhere. Read the PR purpose and linked issues, then fetch the actual base and PR head:
```bash
gh pr view <N> --repo <owner/repo> --json title,author,state,body,baseRefName,headRefName,baseRefOid,headRefOid
git fetch <repo-remote> <baseRefName> refs/pull/<N>/head
git diff <baseRefOid>...<headRefOid>
```
Record both SHAs; refresh if either moved during fetching. Follow relevant [implementation boundaries](../../references/implementation.md).
2. Apply the [risk policy](../../references/adversarial-validation.md), including its two perspectives for high-risk work. Ordinary reviews use the root finding standard directly. File count does not determine risk or authorize delegation; keep any delegated review within the authorized scope.
3. Check `gh pr checks <N> --repo <owner/repo>`. Investigate failures when relevant to a finding or a requested CI/readiness diagnosis. Distinguish pending, unavailable, pre-existing, flaky, and PR-caused states using evidence; a code review does not authorize unrelated repairs.
4. Report the reviewed SHAs, risk tier, supported findings or `No findings`, material verification limits, check state, and verdict. Set severity from evidenced impact, triggering conditions, reach, and urgency: P0 is a critical issue requiring immediate action with broad or unavoidable impact; P1 is a high-impact defect requiring prompt correction; P2 is an ordinary concrete correctness, compatibility, performance, or maintainability defect; P3 is an optional improvement only when requested. A remote trigger, crash, or deadlock does not by itself establish P0. P0/P1 findings block approval; evaluate other findings and verification gaps against the actual acceptance requirements rather than approving automatically below that threshold.
5. When posting is authorized, follow [Git and PR rules](../../references/pull-requests.md), recheck the remote base and head, and review any changed diff or integration assumptions before submitting. Reuse unaffected evidence. Use `gh pr review` with `--approve`, `--request-changes`, or `--comment` and `--body-file`; use [the API example](references/posting.md) for authorized inline comments. Reuse existing authorization.
6. Follow the repository PR lifecycle and explicit monitoring scope. Fetch new heads before comparing to the reviewed SHA; never rely on an unfetched `origin/pull/<N>/head`. Push fixes or resolve threads only when authorized; check `maintainerCanModify` before a fork push.
@@ -1,23 +0,0 @@
# Inline PR Review Submission
Read only when an inline review is authorized. Recheck the PR head before posting and bind the review to the reviewed commit.
For inline comments on specific lines, use the GitHub API:
```bash
REVIEW_PAYLOAD=$(mktemp "${TMPDIR:-/tmp}/pr-review.XXXXXX")
cat > "$REVIEW_PAYLOAD" <<'EOF'
{
"commit_id": "<reviewed-head-sha>",
"body": "review body",
"event": "REQUEST_CHANGES",
"comments": [
{
"path": "crates/<crate>/src/<file>.rs",
"line": 42,
"body": "finding description"
}
]
}
EOF
gh api --method POST /repos/{owner}/{repo}/pulls/<N>/reviews --input "$REVIEW_PAYLOAD" && rm -f "$REVIEW_PAYLOAD"
```
@@ -1,38 +0,0 @@
---
name: rustfs-release-publish
description: Publish a RustFS version through frozen Linux candidate acceptance, desktop completion, manifest approval, byte-preserving publication and post-release follow-up. Use only for an explicit release request.
---
# RustFS Release
Require the exact final SemVer target before version edits or publication. If unspecified, inspect releases/tags and ask with concrete candidates; do not guess the channel or bump. Reuse the user's confirmed target and authorized delivery scope. Independent read-only preflight may continue.
## Invariants
- Cargo and every candidate executable contain the final product version. `<target>-preview.<run_number>` is an immutable candidate identifier, not a public tag or displayed binary version.
- Freeze the exact source, reviewed workflow recipe, toolchain, Console, CLI, dependency lock, previous stable version and validation plan. Newer unrelated main or Console releases do not change those inputs.
- Retain `release/<target>` history when a release branch is used; backport only reviewed fixes. Pin `CANDIDATE_SHA` to its exact CI-validated commit. Never force-push a release branch or move a public tag.
- Build Linux first. Complete field acceptance before macOS/Windows and packaging. Full certification and final acceptance are required before publication; a partial Linux PASS grants no release authority.
- Promote the accepted bytes. Final publication never recompiles binaries or rebuilds packages/images. Infrastructure retries reuse successful artifacts; changed source or inputs require a new candidate and affected acceptance.
- Phase failures block dependents. No latest/default-channel update before versioned delivery. Use the [pipeline runbook](../../../docs/operations/release-pipeline.md) for executable entry points, locks and recovery.
## Read by phase
| Phase | Procedure |
|---|---|
| 0 | Clean tree, fetch main/tags, verify access, then complete the mandatory [Console gate](references/console-gate.md). |
| 1–2 | [Prepare the exact source and run branch CI](references/release-candidate.md). Version edits use `rustfs-release-version-bump`. |
| 3 | [Build and verify the Linux candidate](references/preview-release.md). |
| 4–5 | [Run artifact, field, Console and rc acceptance](references/preview-acceptance.md), then complete desktop/package/image/chart validation. |
| Confirmation | Report the complete manifest and obtain the confirmation below. |
| 6 | [Publish the accepted manifest](references/final-release.md). |
| 7 | [Reconcile advisories, milestones, installation references, website and release branch](references/post-release-updates.md). |
## Human confirmation
After all acceptance passes, report the target, source SHA, source CI run, Linux and completion run URLs, final manifest SHA-256, accepted Console/CLI identities, compatibility/performance results and backlog report. Explicitly ask whether to publish this exact manifest and end the turn without dispatching publication.
Only a new affirmative reply to that exact report authorizes Phase 6. An earlier general release request, silence or automatic continuation does not count. Reuse confirmation for retries of that same manifest; do not ask again merely because a destination failed. A changed source, input, manifest or acceptance result invalidates it and requires renewed acceptance/confirmation.
## Recovery and reporting
Resume failed infrastructure jobs without rebuilding successful units. Fix product defects on main, backport when needed, and start a new candidate. After a final tag exists, retry the same bytes or use a new patch version; never move the tag. Do not conflate implemented, CI-passed, field-accepted, approved, published and post-release-complete states. Report incomplete work and deviations with evidence; a tag or green build alone is not a completed release.
@@ -1,58 +0,0 @@
# Console Release Gate
Read during Phase 0, before changing RustFS version files or tags. Follow the parent skill's release scope and authorization rules.
### Console release gate
Complete this gate before changing any RustFS version file or creating any RustFS tag. Candidate creation resolves `repos/rustfs/console/releases/latest` once and freezes the exact asset for every build, so a successful Console build alone is insufficient. After freezing, later Console releases do not silently change or invalidate the accepted inputs.
1. Read the latest published Console tag and compare it with Console `main`:
```bash
CONSOLE_REPO="rustfs/console"
CONSOLE_LATEST=$(gh api "repos/${CONSOLE_REPO}/releases/latest" --jq .tag_name)
gh api "repos/${CONSOLE_REPO}/compare/${CONSOLE_LATEST}...main" \
--jq '{status, ahead_by, behind_by, commits: [.commits[] | {sha, message: .commit.message}]}'
```
- `ahead_by == 0`: no merged Console change is waiting for release. Still verify the current latest asset using step 4, then continue to Phase 1.
- `ahead_by > 0` and `behind_by == 0`: publish Console before continuing. Report the merged commits and select the next unused `vX.Y.Z` tag. Default to the next patch version when the changes are fixes or backward-compatible UI work; stop for confirmation if a minor/major bump is plausible.
- Any diverged history or `behind_by > 0`: stop and resolve the Console release baseline explicitly. Do not guess a range or publish RustFS.
2. Clone/fetch `rustfs/console` into a scratch directory and record its exact `main` commit. Before creating a tag, check for a `v*` tag or Release workflow already associated with that hash. If one is in progress, wait for it instead of creating another version:
```bash
CONSOLE_SCRATCH=$(mktemp -d)
gh repo clone "$CONSOLE_REPO" "$CONSOLE_SCRATCH/console"
git -C "$CONSOLE_SCRATCH/console" fetch origin main --tags
CONSOLE_HASH=$(git -C "$CONSOLE_SCRATCH/console" rev-parse origin/main)
git -C "$CONSOLE_SCRATCH/console" tag --points-at "$CONSOLE_HASH" 'v*'
gh run list -R "$CONSOLE_REPO" --workflow release.yml --commit "$CONSOLE_HASH" --limit 5
```
If no release exists or is running for `CONSOLE_HASH`, create the selected annotated tag at that exact hash and push it:
```bash
git -C "$CONSOLE_SCRATCH/console" tag -a "<console-tag>" -m "Release <console-tag>" "$CONSOLE_HASH"
git -C "$CONSOLE_SCRATCH/console" push origin "<console-tag>"
```
Console tags include the `v` prefix. Pushing the tag triggers the Console repository's [release workflow](https://github.com/rustfs/console/blob/main/.github/workflows/release.yml) (`🚀 Release`). Remove the task-owned `CONSOLE_SCRATCH` after the gate completes.
3. Find the exact tag run and wait for completion:
```bash
gh run list -R "$CONSOLE_REPO" --workflow release.yml --branch "<console-tag>" --limit 1
gh run watch -R "$CONSOLE_REPO" "<console-run-id>" --exit-status
```
4. Block until the published Release is non-draft, the latest endpoint returns the expected tag, and `rustfs-console-<console-tag>.zip` is uploaded, non-empty, and carries a `sha256:` digest:
```bash
gh release view -R "$CONSOLE_REPO" "<console-tag>" --json isDraft,isPrerelease,assets,url
test "$(gh api "repos/${CONSOLE_REPO}/releases/latest" --jq .tag_name)" = "<console-tag>"
test "$(gh api "repos/${CONSOLE_REPO}/releases/tags/<console-tag>" \
--jq '[.assets[] | select(.name == "rustfs-console-<console-tag>.zip" and .state == "uploaded" and .size > 0 and (.digest | startswith("sha256:")))] | length')" -eq 1
```
Treat a missing/mismatched asset, digest, latest tag, or failed/cancelled workflow as BLOCKED. Do not start Phase 1 until the Console gate passes. Record `CONSOLE_TAG`, `CONSOLE_HASH`, the asset URL/SHA256, Console run URL, and Release URL for comparison with the inputs actually consumed by the RustFS builds and the final report.
@@ -1,13 +0,0 @@
# Final release publication
Read only after fresh confirmation of the complete candidate report. The [parent confirmation gate](../SKILL.md#human-confirmation) applies.
Dispatch `release-publish.yml` on main with the successful completion run ID and approved `release-manifest.json` SHA-256. Do not create the tag manually. The workflow revalidates source, bytes and trusted certification, then creates the annotated final tag at `CANDIDATE_SHA`. A recipe change fails closed and requires a new candidate; unrelated main changes do not require a rebuild.
Require both versioned publication and channel publication jobs to succeed. Verify the final tag's commit and manifest identity; all six platform ZIPs and compatibility aliases; DEB/RPM, checksums, SBOM, provenance and evidence bundle; both multi-architecture image variants in Docker Hub, GHCR and Quay; and the accepted Helm chart in the chart repository. Download a host-compatible final artifact and confirm its digest is exactly the accepted candidate digest and its version/source output agrees.
For stable releases verify GitHub Latest, R2 download aliases and `latest.json` unless a newer stable release already owns those channels. Prereleases must remain GitHub prereleases and leave stable pointers unchanged. Verify generated release notes compare against the frozen previous stable release, excluding internal previews.
On infrastructure failure retry the same workflow/seal; matching destinations are reused. Never clobber an asset, force a tag, rebuild the image or silently accept a changed digest. A source fix after the final tag requires a new patch version. Cross-registry writes are not atomic: report the unfinished destination rather than claiming publication complete.
New candidates create no preview Releases to clean up. For a legacy preview release, preserve its tag and remove its Release only after final delivery and notes are verified. Do not pass `--cleanup-tag`. Complete the [post-release follow-up](post-release-updates.md) separately.
@@ -1,57 +0,0 @@
# Post-release Security Advisory, Milestone, Installation, and Website Updates
Read only after the final non-preview tag has passed Phase 6. Tag creation or a green binary build alone is insufficient: the release assets, source archive, container images, Helm package, and applicable latest channels must be available. Preview tags never enter this phase.
## Security Advisories
Reconcile advisories after every completed final release, including Console-only releases. Read-only preparation may start earlier; do not claim an unpublished artifact is an available fix.
1. Fetch every page of the repository's live Security Advisories, including triage, draft, published, and closed states. A permission error or partial response blocks reconciliation; it does not mean there are no advisories. Keep unpublished advisory details in private task artifacts, never in a public PR, release note, or issue.
2. For each potentially relevant advisory, map its fix to the exact published tag and verify the regression evidence. Main-branch reachability, a merged private fix, or a green build alone does not prove the shipped version is fixed. Check cherry-picked equivalents and reverted fixes explicitly. For embedded Console or dependency fixes, verify the shipped component version.
3. Determine the first available fixed release and affected version ranges from evidence. Preserve an earlier correct patched version; do not relabel all fixes as the current release. An unchanged server baseline such as a Console-only release requires checking the earlier server release as well. Leave unresolved reports and fixes absent from the tag pending; do not close them merely because a release shipped.
4. Prepare any necessary corrections to package identity, affected ranges, patched versions, fix/release links, and advisory text. Preserve unrelated fields, credit, CVE identifiers, and disclosure coordination. Apply updates only within existing user authorization; read back the result to verify it. Publishing a draft, requesting a CVE, or closing a report is a separate action: require explicit authorization for the concrete advisories and respect any embargo. Release-tag approval alone does not authorize those actions. Prepare the exact proposed changes before asking for missing authorization.
5. Record each relevant advisory's URL, current state, fix evidence, first fixed version, action taken or proposed, and remaining blocker in private task artifacts. Report advisory follow-up separately from artifact publication. A successful release with unresolved advisory work is not complete follow-up; retain that work for the release monitor or a concrete user decision.
## Version milestones
1. Query all pages of both open and closed milestones in the RustFS release repository, matching titles exactly to version strings without a `v` prefix. A failed or incomplete query is not evidence that a milestone is absent; resolve the lookup before making changes.
2. Close the milestone matching the published `<target>` if it is open; an already closed milestone needs no change. If it is missing, report the gap and continue checking the next version. Do not create a substitute for the released milestone or close/reassign its issues or PRs.
3. Use an explicitly planned next version when provided; otherwise, for a stable `X.Y.Z`, use `X.Y.(Z+1)`. For example, after `1.0.1` is published, close milestone `1.0.1` and ensure milestone `1.0.2` exists. For an alpha/beta/rc target with a trailing numeric counter, increment that counter within the same channel unless a different next version was specified. If the next version cannot be determined, ask for that version and report this follow-up as blocked. Creating a planning milestone does not confirm the target of a future release or bypass the semver gate.
4. Reuse an existing next-version milestone, whether open or closed; do not reopen it or create a duplicate. Only when the complete query confirms it is absent, create an open milestone with the exact next-version title. Do not invent a due date or description. After an uncertain creation result, query again before retrying.
5. Read back both milestones and report their titles, URLs, and actual states, including whether the next milestone was reused or created. A failed close, create, or verification leaves milestone maintenance incomplete; retry only the unfinished operation after refreshing live state. Continue independent installation and website work without moving or republishing the release tag.
## Installation references
1. Re-read the published deliverable state before edits or a retry. If a newer release has superseded this target, do not downgrade installation references or the website announcement. Report the superseding release and leave those defaults intact.
2. Invoke `rustfs-release-version-bump` with stage `post-release`, the exact target, and the authorized delivery scope. Prepare the follow-up on current main; do not edit or move either validated release tag. Cargo versions may already be preparing the next target and must remain untouched.
3. Verify the exact Docker image manifest, Release download links, RPM source archive, and Helm repository entry before publishing the installation PR. The chart is packaged and tested before sealing; publication uploads those same bytes only after versioned Docker images are available.
4. Report the installation PR and its merge state separately from the completed release. Reuse existing merge authorization and repository checks; do not infer new merge authority from this reference.
## rustfs.com top banner
Use the existing announcement configuration from [website PR #112](https://github.com/rustfs/rustfs.com/pull/112). Do not add a second banner or redesign the header.
1. In an isolated checkout of `rustfs/rustfs.com`, read its current `AGENTS.md`, PR template if present, `data/announcement.ts`, and the consuming header component. Search for an existing announcement PR for this target before creating another.
2. Update `homeAnnouncement` in `data/announcement.ts` after the release and installation availability checks pass. Keep `enabled: true`, `badge: 'New'`, and the existing component. For a stable release, use:
```ts
export const homeAnnouncement = {
enabled: true,
badge: 'New',
message: 'RustFS <target> is now available! Please',
linkText: 'upgrade and try',
href: '/download',
} as const
```
Replace `<target>` with the exact published version. Reserve GA wording for the actual first stable release. For a final alpha/beta/rc tag, explicitly call it a prerelease; use its GitHub Release URL if `/download` does not expose that target. Never announce a `-preview.N` build here.
3. Use the site's current package manager and required checks. For this config-only change, type-check and lint the changed file, build the site, and inspect the generated homepage for the version, visible link text, and destination. Confirm `/download` actually offers the announced version before using that link.
4. Commit, push, and create or update the website PR when included in the authorized release scope. Include the published RustFS Release URL and checks run. Keep merge and production-deployment authority separate; reuse permission already given, but this reference grants none by itself.
5. After an authorized merge/deployment, verify the live `https://rustfs.com/` banner and its destination. If only the PR is ready, report that state and URL; do not report the banner as deployed. A website failure does not invalidate or move the published release tag: retry this phase after resolving the failure.
## Reconcile main and remove the completed release branch
1. Compare the exact published tag with current main. Audit every release-side commit since their merge base, including inherited backports. An ahead count is a history difference, not proof of missing changes. Use ancestry and patch equivalence, then inspect current code and regression evidence for squashed, adapted, superseded, or reverted changes. Record each commit as present, superseded, release-only, or missing; a positive `git cherry` result alone does not prove a missing fix.
2. Bring missing shared changes to main through a reviewed PR and the required checks before closing the follow-up. Preserve main's later changes; decide version bumps and pinned Console configuration separately. If a history merge is needed, use a merge commit and verify the release commit is an ancestor of main; squash does not establish that ancestry. Do not merge main into the published release branch or move its tags. Report unresolved changes instead of claiming reconciliation is complete.
3. After publication and authorized follow-ups are complete, delete the remote `release/<target>` branch. Immediately before deletion, verify its HEAD still equals the published tag's commit and no open PR targets or originates from it. Preserve a branch with newer work or open PRs and report the blocker. Use a deletion with an expected-old-SHA check to avoid removing a concurrently advanced branch.
4. Verify the remote branch is absent and the final tag and any legacy preview tags still resolve to their recorded commits. Remove a local release branch only if it is task-owned, unchecked out, and has no unpreserved work. Record cleanup separately from artifact publication; an explicitly authorized deletion before reconciliation does not make unresolved main changes complete.
@@ -1,68 +0,0 @@
# Candidate Artifact Acceptance
Read after Phase 3 succeeds. Complete every check below before the parent skill's manual confirmation gate. These checks cover the downloaded artifact, embedded Console, and the rc release frozen in the candidate inputs.
## Phase 4 — Run the artifact locally, verify the console
Use a unique task-owned scratch directory. Select the versioned archive matching the acceptance host, and verify it against the sealed manifest before extraction. Before desktop completion, use the Linux acceptance environment rather than rebuilding for the local desktop. Do not substitute a different build when a matching artifact is unavailable; report that acceptance gap. The shell example below assumes the verified executable is at `RUSTFS_BINARY` (`rustfs.exe` on Windows), `SCRATCH` is the task directory, and `S3_PORT` and `CONSOLE_PORT` are distinct available ports. Use equivalent native commands when the shell differs.
Bind both listeners to loopback. Choose task-only credentials, set loopback proxy exclusions for the server/client processes, and record the server PID. A port probe does not reserve a port; retry startup with fresh ports on a bind conflict rather than connecting to an existing service.
```bash
"$RUSTFS_BINARY" --version
mkdir -p "$SCRATCH/data"
S3_ENDPOINT="http://127.0.0.1:${S3_PORT}"
CONSOLE_ENDPOINT="http://127.0.0.1:${CONSOLE_PORT}"
RUSTFS_ACCESS_KEY="$PREVIEW_ACCESS_KEY" RUSTFS_SECRET_KEY="$PREVIEW_SECRET_KEY" \
"$RUSTFS_BINARY" server \
--address "127.0.0.1:${S3_PORT}" \
--console-enable \
--console-address "127.0.0.1:${CONSOLE_PORT}" \
"$SCRATCH/data" >"$SCRATCH/server.log" 2>&1 &
PREVIEW_SERVER_PID=$!
```
Checks (all must pass):
- `"$RUSTFS_BINARY" --version` reports the exact final `<target>` and full `CANDIDATE_SHA`. A preview suffix, another version or another source is a failure.
- Wait with a bounded readiness deadline while checking the recorded server process remains alive; `curl --noproxy '*' -fsS "$S3_ENDPOINT/health/ready"` must return ready from this process's listener.
- Startup log shows the embedded console being served (this was the regression that `fix(release): require embedded console assets` guards).
- Open `CONSOLE_ENDPOINT` in the browser and log in with the task credentials. The dashboard must render without JS console errors; create a bucket, upload a file, download it back (byte-identical), and delete the object and bucket. Keep the server running for Phase 5.
## Phase 5 — Validate with the pinned rc client
`rc` is the RustFS CLI client from <https://github.com/rustfs/cli>.
- Read the CLI release and asset digests from `inputs.json`; verify the local client's version or download a host-compatible asset from that exact release into the task directory. If the host is a desktop, resolve its asset within the same pinned CLI release and record the digest; do not independently follow `latest`. Set `RC_BIN` to that executable and record its version/digest. Avoid changing the user's installed client for this check.
- Inspect the client's current help for an isolated configuration option and use it when supported. Otherwise choose a unique `PREVIEW_ALIAS` and confirm it is absent before creating it; never overwrite an existing alias. Use task-owned bucket/user names and input/output files. The names below are safe only inside the newly started isolated server.
- Point the client at `S3_ENDPOINT` with loopback proxy exclusions and run the command matrix, recording PASS/FAIL per command:
```bash
"$RC_BIN" alias set "$PREVIEW_ALIAS" "$S3_ENDPOINT" "$PREVIEW_ACCESS_KEY" "$PREVIEW_SECRET_KEY"
"$RC_BIN" ls "$PREVIEW_ALIAS/"
"$RC_BIN" mb "$PREVIEW_ALIAS/rel-check"
"$RC_BIN" cp <local-file> "$PREVIEW_ALIAS/rel-check/"
"$RC_BIN" stat "$PREVIEW_ALIAS/rel-check/<file>"
"$RC_BIN" cat "$PREVIEW_ALIAS/rel-check/<file>" # matches source
"$RC_BIN" cp "$PREVIEW_ALIAS/rel-check/<file>" "$SCRATCH/out"
cmp <local-file> "$SCRATCH/out"
"$RC_BIN" cp -r <local-dir>/ "$PREVIEW_ALIAS/rel-check/dir/"
"$RC_BIN" find "$PREVIEW_ALIAS/rel-check" --name "*"
"$RC_BIN" share download "$PREVIEW_ALIAS/rel-check/<file>" --expire 1h
# Fetch the presigned URL with curl --noproxy '*' and compare its body.
"$RC_BIN" rm "$PREVIEW_ALIAS/rel-check/<file>"
"$RC_BIN" rm -r --force "$PREVIEW_ALIAS/rel-check/dir"
"$RC_BIN" rb "$PREVIEW_ALIAS/rel-check"
"$RC_BIN" admin user list "$PREVIEW_ALIAS/"
"$RC_BIN" admin user add "$PREVIEW_ALIAS/" relcheckuser relchecksecret12
"$RC_BIN" admin user remove "$PREVIEW_ALIAS/" relcheckuser
"$RC_BIN" alias remove "$PREVIEW_ALIAS"
```
- Any FAIL blocks acceptance. On success or failure, stop only the recorded task process and remove only task-created aliases/data after retaining the needed failure evidence. Do not erase an existing client configuration or unrelated server data.
## Field acceptance and desktop completion
Before accepting the Linux manifest, record Scanner/Heal, multi-pool, memory, major features, high-concurrency 503, performance and previous-version integrity/upgrade results in backlog. Preserve the release-specific agreed limits and baseline; a PASS label without artifact identities or test results does not satisfy acceptance.
Dispatch `release-complete.yml` only after that report passes, using the exact Linux manifest digest and backlog URL. Require native macOS/Windows checks, package extraction, image startup/scans, chart tests and complete certification to pass. Complete any desktop Console/client checks applicable to the release against these same archives. Record the completion run and final manifest SHA-256 before entering the parent confirmation gate. Partial Linux acceptance does not authorize final publication.
@@ -1,9 +0,0 @@
# Linux candidate verification
Read for Phase 3 after the source passed exact-SHA CI. The [parent invariants](../SKILL.md) apply; commands and recovery live in the [pipeline runbook](../../../../docs/operations/release-pipeline.md).
Dispatch `release-candidate.yml` on main with the confirmed final version and `CANDIDATE_SHA`. Record the run ID, frozen `inputs.json`, every Linux archive digest and `linux-manifest.json` SHA-256. Require the four Linux builds, native startup/S3 checks, previous-release upgrade matrix and Linux seal to succeed. Missing, skipped, cancelled or failed required work is incomplete acceptance.
Verify the frozen Console identity equals the one accepted in Phase 0. Every matrix job consumes that same digest and exact toolchain. The binary must report the final product version and source commit; its candidate ID is separate. No public tag, Release, registry tag, R2 release path, Helm repository or latest pointer should change during this stage.
A green Linux run is ready for field acceptance, not publication. The partial certification must have `publish_allowed: false`. Retain the exact Linux manifest for the next stage; a new source/build-input candidate must not inherit the old approval. A newer unrelated main commit does not invalidate this evidence.
@@ -1,28 +0,0 @@
# Release Candidate Preparation
Read for Phases 1–2. The [parent release invariants](../SKILL.md) and Console gate apply.
## Candidate Resume Preflight
- `git status --short` clean; `git fetch origin main --tags`.
- `gh auth status` works; confirm you can view `gh release list -L 3`.
- Confirm the exact final target version with the user if not explicit.
- Check whether `release/<target>` already exists before creating it; a resumed release reuses the existing branch and its history. For an in-flight release that already has a preview tag but no branch, anchor the new branch to that tag's commit only after verifying its target versions and prior acceptance evidence.
## Phase 1 — Source version bump to the final target (once)
- If `release/<target>` exists, verify its Cargo.toml and workspace members in Cargo.lock read `<target>` and skip the source bump, even if main has advanced. If the branch versions differ, stop and resolve the mismatch without downgrading main.
- If no branch exists but an unfinished preview tag does, verify its commit has `<target>` in both version files and skip the bump; Phase 2 anchors the branch to that commit. Stop if those files disagree. Otherwise, if main already has `<target>` in both files, skip the bump. Installation references intentionally retain the previous published deliverable during preview; if they were advanced to an unavailable target, restore that baseline before choosing the next candidate.
- Only when neither the release branch, an unfinished preview tag, nor current main already has `<target>`: if main is at a later version, select and verify a historical commit containing `<target>` rather than downgrading main; block if none exists. Otherwise invoke the `rustfs-release-version-bump` skill with stage `prepare`, the final `<target>` (NOT a preview version), and full GitHub flow (commit/push/PR).
- If a bump PR was needed, get it merged into main and record its merge commit. For a new release branch when main already has `<target>`, identify the main commit that introduced those versions. Phase 2 chooses a release candidate containing that commit; an existing release branch keeps its own history and a newer main HEAD is not required.
Do not wait for the moving latest main HEAD to become green. The release-branch CI in Phase 2 validates the chosen commit before a candidate build is dispatched.
## Phase 2 — Pin and validate the release source
- Use `release/<target>`. On the first attempt, create it from a selected main commit containing the version bump and intended changes; it need not be the latest main HEAD. On a restart, retain the existing branch and backport only reviewed fixes from main. Verify the branch's Cargo versions still equal `<target>`. A legacy preview can anchor source selection, but its tag and old artifacts do not constitute a candidate in the new protocol; freeze and verify a new candidate after the release recipe is aligned.
- The current `ci.yml` runs automatically on main pushes, not release-branch pushes. Dispatch it explicitly with `gh workflow run ci.yml --ref "release/<target>"`. Locate that dispatch run, verify its `head_sha` equals the release branch's remote HEAD, and require the full expected CI job set to pass, including `Quick Checks`, `Test and Lint`, and `End-to-End Tests (full merge gate)`. Treat a missing or skipped required lane as incomplete. A successful PR check or CI run for a different SHA does not qualify. If the branch advances or a run is cancelled, validate the new SHA again. Do not dispatch a candidate with failing or incomplete CI.
- Prefer protecting `release/*` against force pushes and deletion and requiring reviewed backport PRs. Check live rulesets rather than assuming the main ruleset covers release branches; report a missing release-branch rule as a gap.
- Set `CANDIDATE_SHA` to the CI-validated release-branch commit. Record the branch, exact commit and CI run URL. The new candidate protocol starts from a commit whose release recipe matches the reviewed main dispatcher; backport the recipe before freezing an older release branch if needed.
- Dispatch the Linux candidate as described in [candidate verification](preview-release.md). Do not create a preview or final tag here. The final executable already contains `<target>`; publication later promotes those same bytes.
- After a source or build-input change, run the affected source checks and create a new candidate. Reuse the frozen candidate after an infrastructure failure. Do not restart acceptance for unrelated main advances.
@@ -1,24 +0,0 @@
---
name: rustfs-release-version-bump
description: Prepare an exact RustFS Cargo version or align installation references after publication. Use for requested version bumps or the release workflow.
---
# Release Version Files
Require an exact target; ask if missing or ambiguous. Reject targets containing `-preview`: those identify candidates, not product versions or new public tags. Infer `post-release` only from an explicit request or parent workflow; otherwise use `prepare`. Delivery defaults to local edit/verify; commit, push, PR, and merge follow existing authorization and [Git rules](../../references/pull-requests.md).
| Stage | Files and constraints |
|---|---|
| `prepare` | Only `Cargo.toml` and `Cargo.lock`: workspace package, internal dependency, and workspace member versions become `<target>`. External dependencies stay unchanged. |
| `post-release` | Only `README.md`, `README_ZH.md`, `flake.nix`, `helm/rustfs/Chart.yaml`, and `rustfs.spec`. Cargo may already target the next release; leave it untouched. |
Before installation edits, verify the exact non-preview Release, downloadable assets/source archive, pullable `rustfs/rustfs:<target>` image manifest, and published Helm chart. Block on missing artifacts. Do not downgrade defaults if a newer deliverable superseded the target. Candidate preparation retains the previous published installation references; follow-up commits never replace validated tags or sealed artifacts.
Packaging policy:
- Docker image tags have no `v` prefix. Derive chart/app versions with `scripts/helm_chart_version.sh <target>`; both equal the final product SemVer, including prerelease suffixes. Helm CI uses the frozen candidate version before sealing, so no early installation chart bump is required. Previously published chart versions remain unchanged.
- RPM `Version` is numeric; `Release` is the prerelease suffix or `1` for stable. Expanded `Source0` and the unpack directory must resolve to the exact target including its suffix. Add the changelog entry using current git identity, date, and target; never copy an example identity/date.
- Changing these mappings requires explicit confirmation.
Verification follows root tiers: `prepare` validates Cargo metadata with the updated lockfile and checks every workspace/internal version with no unrelated lockfile updates. `post-release` renders Helm with the verified image, runs `scripts/test_helm_chart_version.sh` for chart version edits, and checks README, Nix, and RPM references. Run `git diff --check`; report unresolved required checks as `BLOCKED` without unrelated fixes.
For authorized commits, keep stages separate: `chore(release): prepare <target>` or `chore(release): align installation references for <target>`. Report target/stage, changed files, verification, material uncertainty, and actual commit/push/PR state.
-37
View File
@@ -1,37 +0,0 @@
---
name: tier-debug
description: Debug ILM tiering / lifecycle transition issues — NoSuchVersion on tier GET, restore failures, xl.meta inspection, remote-tier versionId tracing. Use when investigating tiered/transitioned objects, warm backends, or transition metadata.
---
# Tier / ILM Debugging
Playbook: [docs/operations/tier-ilm-debugging.md](../../../docs/operations/tier-ilm-debugging.md).
Read the section matching the symptom: metadata/`xl.meta`, runtime versionId,
manual jobs, or retained-record recovery. Read the local-first expiry invariant
before changing cleanup ordering. Before a reconcile/disposition action, read
its entire procedure and retain its exact-evidence and confirmation gates.
Quick moves:
```bash
# Inspect transition metadata on disk (one xl.meta per erasure shard disk)
cargo run -p rustfs-filemeta --example dump_fileinfo -- "/path/to/{bucket}/{object}/xl.meta"
# Trace what versionId is sent to the remote tier
RUST_LOG=rustfs_ecstore::bucket::lifecycle=debug ./target/debug/rustfs …
```
Interpretation:
- `transition_ver_id: <none>` → correct for an unversioned tier bucket; no
`versionId` must be sent on tier GET/DELETE.
- `transition_ver_id: 00000000-…` (nil) → corrupt legacy write-back; readers
must filter it out, never send it.
- Empty-string `transitioned-versionID` metadata under both
`x-rustfs-internal-*` and `x-minio-internal-*` keys → object went to an
unversioned tier bucket.
Code entry points: `crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs`
(ILM actions), `crates/ecstore/src/services/tier/` (warm backends),
`crates/filemeta/src/filemeta/version.rs` (metadata read/write + regression
tests).
-44
View File
@@ -1,44 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# RustFS Cargo configuration
# NOTE: `--cfg tokio_unstable` is deliberately NOT set here.
#
# It used to be a global `[build] rustflags` entry so that the (default-off)
# `dial9` telemetry feature could compile. That made every build depend on
# Tokio's unstable, non-semver API, and it broke silently whenever a caller
# exported their own RUSTFLAGS — an environment RUSTFLAGS replaces the value
# from this file rather than appending to it.
#
# The flag is now scoped to telemetry builds, which opt in explicitly:
#
# make build-profiling
# RUSTFLAGS="--cfg tokio_unstable" cargo build --features dial9
#
# `crates/obs/build.rs` fails the compile if the `dial9` feature is on without
# the flag, so the two can no longer drift apart unnoticed.
#
# For CPU profiling, add `-C force-frame-pointers=yes` to that RUSTFLAGS value.
# Exact internode HTTP/2 PING ACK timeout accounting is also deliberately
# default-off. Hyper exposes the exact timeout only through its unstable tracing
# event, so diagnostic builds must opt in to both the feature and cfg:
#
# RUSTFLAGS="--cfg hyper_unstable_tracing" \
# cargo build -p rustfs --features diagnostic-internode-http2-ping
#
# `crates/protos/build.rs` and `crates/obs/build.rs` fail fast if the feature and
# cfg drift apart. Do not move this cfg into global rustflags: doing so would
# recompile every dependency under an unstable, non-semver build contract.
-1
View File
@@ -1 +0,0 @@
../.agents/skills
-16
View File
@@ -1,16 +0,0 @@
{
"schema_version": 1,
"rollout": "enforced",
"core_packages": ["rustfs", "rustfs-ecstore", "rustfs-filemeta", "rustfs-iam", "rustfs-policy"],
"connect_packages": [
"rustfs-credentials", "rustfs-config", "rustfs-common", "rustfs-crypto",
"rustfs-iam", "rustfs-policy", "rustfs-kms", "rustfs-license", "rustfs-security-governance",
"rustfs-utils", "rustfs-tls-runtime", "rustfs-trusted-proxies", "rustfs-filemeta",
"rustfs-lock", "rustfs-obs", "rustfs-signer", "rustfs-io-metrics", "rustfs-s3-ops"
],
"full_paths": [
"Cargo.toml", "Cargo.lock", "*/Cargo.toml", "rust-toolchain*", "rustfmt.toml",
"clippy.toml", ".cargo/*", ".config/*", ".github/*", "Makefile",
"scripts/*", "build.rs", "*.proto"
]
}
-10
View File
@@ -1,10 +0,0 @@
# Rulings for the gateway conformance lane (.github/workflows/gateway-acceptance.yml).
#
# One row per case that fails on the GATEWAY stack for a reviewed reason: `case`; `ledger`, the
# rustfs/backlog#2684 entry that accepts the difference (R-0xx or K-n); a one-line `reason`; and
# `failure`, a prefix of the failure the ruling accepts, as the runner's report.json renders it
# (`[expect/status] at /expect/status: exchange #1, attempt 1: expected status 400, observed 403`).
# scripts/check_conformance_rulings.py fails the lane on a failed case without a row, on a ruled
# case whose failures do not start with its `failure`, and on a row whose case no longer fails or
# no longer exists. The legacy stack is a reference run and takes no rulings: the `rustfs` profile
# describes the gateway edge, not the legacy one.
-20
View File
@@ -1,20 +0,0 @@
# Report-only calibration baseline from https://github.com/rustfs/rustfs/actions/runs/29394996173.
# Update counts only with a linked coverage run and a reviewed explanation.
phase = "report-only"
allowed_drop_percentage_points = 1.0
[crates."crates/iam"]
covered = 5149
count = 8131
[crates."crates/kms"]
covered = 2950
count = 4200
[crates."crates/policy"]
covered = 4636
count = 5464
[crates."crates/crypto"]
covered = 469
count = 494
-2
View File
@@ -1,2 +0,0 @@
sha256-linux=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
sha256-darwin=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
-2
View File
@@ -1,2 +0,0 @@
sha256-darwin=24a8cfaa1b9125bd69d7d4a7ff752a8ab8ed350c761d90ae8472ef52393e2686
sha256-linux=014257197fbbfbb48cab2864da9e10d96ee877d478fa29d119f8186532ea293d
-2
View File
@@ -1,2 +0,0 @@
sha256-darwin=e84e04208f19dc0596b10d4d1b799ce43d6cc431415902f9c5458aa8232bc2d8
sha256-linux=1f9c7add72c44d1c12ef4ac050bb7917431be657c0c90b29f70c3df56f8bbae3
-1
View File
@@ -1 +0,0 @@
sha256=87c05c46d611ea7ed3feb5f7276bda8e5a0f70d72165d305d73a457907e7ba79
-2
View File
@@ -1,2 +0,0 @@
sha256-darwin=dcc076f8a36320d2fe36b4d2f26e2d54c3b9bc2ff7e14c6e90241f4fd8cf4a33
sha256-linux=229ebc0cbc9b35863600ae60c6fbb548516efe2a2f41f4b5db503b3283c2f64f
-1
View File
@@ -1 +0,0 @@
sha256=4eac269f9a59b2f9be4e7a87ea6aa38454f5ebe10ede35c9b8d4218137d2cd35
-1
View File
@@ -1 +0,0 @@
sha256=fd5683273bba1eae3b4402a4e854a5d168ace845bd0e5fc801f8e336485a4394
-387
View File
@@ -1,387 +0,0 @@
{
"lane": "ci/test-and-lint",
"tests": [
{
"invariant": "write-quorum",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::inline_put_commit_path_tests::inline_put_direct_commit_accepts_exact_quorum_and_rejects_quorum_minus_one"
},
{
"invariant": "metadata-rollback",
"suite": "rustfs-ecstore",
"name": "set_disk::core::io_primitives::tests::write_unique_file_info_reverts_metadata_when_write_quorum_fails"
},
{
"invariant": "stale-writer",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::put_object_no_lock_aborts_after_outer_namespace_lock_loss"
},
{
"invariant": "range-body",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::transition_upload_integrity_tests::transitioned_compressed_object_range_get_returns_plaintext_slice"
},
{
"invariant": "multipart-cancellation",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::multipart::tests::cancelled_complete_keeps_upload_lock_through_tail_cleanup"
},
{
"invariant": "list-uncommitted-version",
"suite": "rustfs-filemeta",
"name": "metacache::tests::resolve_merge_keeps_the_newest_version_at_listing_quorum"
},
{
"invariant": "minio-object-fixture",
"suite": "rustfs-filemeta",
"name": "filemeta::test::parses_real_minio_object_xlmeta"
},
{
"invariant": "corrupt-part-arrays",
"suite": "rustfs-filemeta",
"name": "filemeta::test::crc_valid_but_part_arrays_corrupt_into_fileinfo_errors_not_panics"
},
{
"invariant": "odm-source-contract-s3",
"suite": "rustfs",
"name": "on_demand_migration::source_client::tests::s3_backend_satisfies_the_shared_backend_contract"
},
{
"invariant": "odm-source-contract-azure",
"suite": "rustfs",
"name": "on_demand_migration::azure::tests::azure_backend_satisfies_the_shared_backend_contract"
},
{
"invariant": "odm-source-contract-gcs",
"suite": "rustfs",
"name": "on_demand_migration::gcs::tests::gcs_native_backend_satisfies_the_shared_backend_contract"
},
{
"invariant": "rollback-inspection",
"suite": "rustfs-ecstore",
"name": "set_disk::core::io_primitives::tests::rename_rollback_incomplete_preserves_overwrite_data_dirs_and_staging"
},
{
"invariant": "rollback-closed-inspection-channel",
"suite": "rustfs-ecstore",
"name": "set_disk::core::io_primitives::tests::rename_rollback_closed_heal_receiver_preserves_recovery_material"
},
{
"invariant": "rollback-cancellation",
"suite": "rustfs-ecstore",
"name": "set_disk::core::io_primitives::tests::rename_rollback_incomplete_cancelled_task_is_not_success"
},
{
"invariant": "rollback-completion-equivalence",
"suite": "rustfs-ecstore",
"name": "set_disk::core::io_primitives::tests::rename_rollback_incomplete_matches_early_ack_and_full_wait_after_reopen"
},
{
"invariant": "rollback-physical-owner",
"suite": "rustfs-ecstore",
"name": "set_disk::core::io_primitives::tests::rename_rollback_children_keep_namespace_ownership_after_coordinator_panic"
},
{
"invariant": "late-ack-retains-commit",
"suite": "rustfs-ecstore",
"name": "set_disk::core::io_primitives::tests::rename_data_early_ack_post_mutation_tail_error_never_rolls_back_commit"
},
{
"invariant": "tail-drained-cas",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::tail_drained_put_waits_for_tail_and_allows_immediate_cas"
},
{
"invariant": "seeded-put-tail-pressure",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::seeded_put_completion_pressure_drains_every_real_tail"
},
{
"invariant": "borrowed-lock-tail",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::no_lock_put_waits_for_rename_tail_under_outer_guard"
},
{
"invariant": "cancelled-writer-owner",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::tail_drained_put_owned_commit_survives_waiter_cancellation"
},
{
"invariant": "get-body-reconstruction",
"suite": "rustfs-ecstore",
"name": "ecstore_validation_blackbox::blackbox_get_restores_body_after_one_shard_file_is_removed"
},
{
"invariant": "range-reconstruction",
"suite": "rustfs-ecstore",
"name": "ecstore_validation_blackbox::blackbox_range_read_restores_exact_slice_with_one_offline_disk"
},
{
"invariant": "multipart-abort-linearization",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::multipart::tests::abort_and_complete_linearize_for_plain_sse_and_legacy_layouts"
},
{
"invariant": "multipart-write-quorum",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::multipart::tests::abort_enforces_delete_write_quorum_boundary"
},
{
"invariant": "multipart-physical-owner",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::multipart::tests::complete_multipart_timeout_keeps_namespace_owner_until_physical_publication"
},
{
"invariant": "restore-abort-cleanup",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::transition_commit_failure_tests::multipart_restore_aborts_every_post_create_failure"
},
{
"invariant": "legacy-metadata",
"suite": "rustfs-filemeta",
"name": "filemeta::test::test_issue_2288_legacy_xlmeta_compatibility"
},
{
"invariant": "legacy-nil-version",
"suite": "rustfs-filemeta",
"name": "filemeta::test::test_into_fileinfo_reads_legacy_nil_uuid_inline_key"
},
{
"invariant": "commit-phase-schedule",
"suite": "rustfs-ecstore",
"name": "set_disk::core::io_primitives::tests::commit_fault_schedule_keeps_multiple_disk_phases_independent"
},
{
"invariant": "commit-phase-ownership",
"suite": "rustfs-ecstore",
"name": "set_disk::core::io_primitives::tests::commit_fault_schedule_rejects_duplicate_checkpoint_without_disarming_owner"
},
{
"invariant": "metadata-observation-all-four-slot-states-and-arrival-orders-preserve-reduction",
"suite": "rustfs-ecstore",
"name": "set_disk::core::metadata_quorum::tests::metadata_observation_all_four_slot_states_and_arrival_orders_preserve_reduction"
},
{
"invariant": "metadata-observation-mixed-current-identity-arrivals-preserve-selection",
"suite": "rustfs-ecstore",
"name": "set_disk::core::metadata_quorum::tests::metadata_observation_mixed_current_identity_arrivals_preserve_selection"
},
{
"invariant": "metadata-observation-pending-newer-version-and-same-time-different-directory-force-full-wait",
"suite": "rustfs-ecstore",
"name": "set_disk::core::metadata_quorum::tests::metadata_observation_pending_newer_version_and_same_time_different_directory_force_full_wait"
},
{
"invariant": "metadata-observation-duplicate-shard-cannot-supply-a-read-reserve",
"suite": "rustfs-ecstore",
"name": "set_disk::core::metadata_quorum::tests::metadata_observation_duplicate_shard_cannot_supply_a_read_reserve"
},
{
"invariant": "metadata-observation-null-versions-markers-and-invalid-success-keep-their-meaning",
"suite": "rustfs-ecstore",
"name": "set_disk::core::metadata_quorum::tests::metadata_observation_null_versions_markers_and_invalid_success_keep_their_meaning"
},
{
"invariant": "metadata-cache-real-get-and-range-preserve-body-across-read-modes",
"suite": "rustfs-ecstore",
"name": "set_disk::read::metadata_cache_tests::metadata_cache_real_get_and_range_preserve_body_across_read_modes"
},
{
"invariant": "late-metadata-refresh-rejects-changed-generation-and-mapped-shards",
"suite": "rustfs-ecstore",
"name": "set_disk::read::metadata_cache_tests::late_metadata_refresh_rejects_changed_generation_and_mapped_shards"
},
{
"invariant": "late-full-get-rejects-changed-object-generation",
"suite": "rustfs-ecstore",
"name": "set_disk::prepared_get_object_metadata_tests::non_inline_two_phase_get_rejects_generation_change_before_late_refresh"
},
{
"invariant": "list-objects-shared-corpus-static-namespace",
"suite": "rustfs-ecstore",
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_static_namespace"
},
{
"invariant": "list-objects-shared-corpus-cross-subdirectory-limit",
"suite": "rustfs-ecstore",
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_cross_subdirectory_limit"
},
{
"invariant": "list-objects-shared-corpus-max-and-max-plus-one",
"suite": "rustfs-ecstore",
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_max_and_max_plus_one"
},
{
"invariant": "list-objects-shared-corpus-fixed-seed-marker-and-prefix",
"suite": "rustfs-ecstore",
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_fixed_seed_marker_and_prefix"
},
{
"invariant": "list-objects-shared-corpus-offline-corrupt-and-minority-stale",
"suite": "rustfs-ecstore",
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_offline_corrupt_and_minority_stale"
},
{
"invariant": "list-objects-shared-corpus-versions-delete-markers-and-free-versions",
"suite": "rustfs-ecstore",
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_versions_delete_markers_and_free_versions"
},
{
"invariant": "list-objects-shared-corpus-orphan-cleanup-requires-authoritative-evidence",
"suite": "rustfs-ecstore",
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_orphan_listing_preserves_inputs"
},
{
"invariant": "list-objects-shared-corpus-consumer-drop-joins-real-disk-producer",
"suite": "rustfs-ecstore",
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_consumer_drop_joins_real_disk_producer"
},
{
"invariant": "list-objects-shared-corpus-cancelled-quorum-walk-releases-senders-and-disk-lock",
"suite": "rustfs-ecstore",
"name": "store::list_objects::differential_tests::list_objects_shared_corpus_cancelled_quorum_walk_releases_senders_and_disk_lock"
},
{
"invariant": "borrowed-context-boundary",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::borrowed_write_context_rejects_wrong_namespace_and_cache_flag"
},
{
"invariant": "borrowed-put-cancellation",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::put_object_tmp_cleanup_tests::borrowed_tail_drained_put_keeps_owner_after_caller_cancel"
},
{
"invariant": "borrowed-mpu-cancellation",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::multipart::tests::borrowed_complete_keeps_namespace_owner_after_caller_cancel"
},
{
"invariant": "pool-cas-owner",
"suite": "rustfs-ecstore",
"name": "core::pools::pools_tests::pool_meta_cas_context_retains_actual_owner_for_exact_target"
},
{
"invariant": "pool-cas-released-owner",
"suite": "rustfs-ecstore",
"name": "core::pools::pools_tests::pool_meta_cas_rejects_released_owner"
},
{
"invariant": "copy-tail-drained",
"suite": "rustfs-ecstore",
"name": "store::object::tests::borrowed_copy_tail_drained_waits_for_rename_tail"
},
{
"invariant": "catalog-borrowed-publication",
"suite": "rustfs",
"name": "admin::handlers::table_catalog::tests::commit_backend_borrowed_put_preserves_owner_authorization_and_precondition"
},
{
"invariant": "directory-marker-quorum",
"suite": "rustfs-filemeta",
"name": "metacache::tests::resolve_directory_marker_and_prefix_preserve_quorum_in_both_orders"
},
{
"invariant": "logical-prefix-limit",
"suite": "rustfs-ecstore",
"name": "store::list_objects::test::list_path_gather_results_counts_common_prefixes_before_page_limit"
},
{
"invariant": "quota-same-name-owner-and-disk-authority",
"suite": "rustfs-ecstore",
"name": "bucket::metadata_sys::tests::quota_snapshot_reuses_same_name_owner_and_rereads_disk_authority"
},
{
"invariant": "quota-snapshot-scope-and-owner",
"suite": "rustfs-ecstore",
"name": "bucket::metadata_sys::tests::quota_snapshot_rejects_wrong_scope_incarnation_and_transaction_guard"
},
{
"invariant": "quota-snapshot-lifecycle-and-incarnation",
"suite": "rustfs-ecstore",
"name": "bucket::metadata_sys::tests::quota_snapshot_rejects_lifecycle_loss_during_read_and_changed_disk_incarnation"
},
{
"invariant": "marker-purge-receipt-reopen",
"suite": "rustfs-ecstore",
"name": "set_disk::ops::object::marker_purge_receipt_tests::quorum_marker_purge_receipt_survives_reopen_with_stale_member"
},
{
"invariant": "external-minio-shards",
"suite": "rustfs-ecstore::pinned_erasure_fixtures_test",
"name": "pinned_minio_shards_decode_and_reconstruct_plaintext"
},
{
"invariant": "external-legacy-shards",
"suite": "rustfs-ecstore::pinned_erasure_fixtures_test",
"name": "pinned_legacy_shards_decode_and_reconstruct_plaintext"
},
{
"invariant": "external-shard-bitrot",
"suite": "rustfs-ecstore::pinned_erasure_fixtures_test",
"name": "pinned_erasure_shards_reject_bitrot_corruption"
},
{
"invariant": "external-shard-quorum-codec",
"suite": "rustfs-ecstore::pinned_erasure_fixtures_test",
"name": "pinned_erasure_shards_reject_insufficient_quorum_and_wrong_codec"
}
],
"fixtures": [
{
"path": "crates/filemeta/tests/fixtures/minio/object_large_bin.xlmeta.hex",
"sha256": "e8093767806d701e639b48d023190e858fbc4cde69bcfd83c22af8cba8452ce5",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/filemeta/tests/fixtures/minio/object_small_txt.xlmeta.hex",
"sha256": "2a415ad3a3be5a9440035d4026ff880e0e8c1ec1701be9f4e077734e8dce03da",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/filemeta/tests/fixtures/minio/object_versioned_txt.xlmeta.hex",
"sha256": "7f21f50c326dd8b0228deb6dbdb7052b3d0a3f8ee6c85d43486f0e6bb7a97261",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/ecstore/tests/fixtures/minio/bucket_metadata.blob.hex",
"sha256": "f2b6e260aff106adf6039feb1c645686e84e75404ff725491fb18668be5db203",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/ecstore/tests/fixtures/minio/bucket_metadata_full.xlmeta.hex",
"sha256": "3b6de589519c08a1614c8bd409bb8199c17d42043861b07bce513075e6fbfc12",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/minio/README.md"
},
{
"path": "crates/filemeta/tests/fixtures/issue_2288_legacy_xlmeta.hex",
"sha256": "b3e8d32f9950a38b5dc71f679519cc830a2b2a807e8b0b39f78840b238d8eacf",
"source": "Legacy xl.meta captured for rustfs/backlog#2288; filemeta::test::test_issue_2288_legacy_xlmeta_compatibility"
},
{
"path": "crates/filemeta/tests/fixtures/issue_2265_legacy_meta_v2_object.hex",
"sha256": "89dd7ec92d16e54b5d31e0c49c9edffff8a7aa58890aaeff59bc2ff84da0a62f",
"source": "Legacy metadata v2 object captured for rustfs/backlog#2265; crates/filemeta/src/test_data.rs"
},
{
"path": "crates/ecstore/tests/fixtures/list_namespace_keys.json",
"sha256": "5afddb60565deba0938e617470255563f0e15a15fab699645c59bb7ccc84898e",
"source": "Shared static LIST namespace corpus; ECStore differential tests and ODM list_through_static_namespace_boundary_matrix"
},
{
"path": "crates/ecstore/tests/fixtures/erasure-shards/minio/manifest.json",
"sha256": "c398f8dcb76903f6cf8eee07931c76592895ced201ff9d59ed4ff9b0ead2b13c",
"source": "MinIO RELEASE.2025-07-23T15-54-02Z; crates/ecstore/tests/fixtures/erasure-shards/README.md"
},
{
"path": "crates/ecstore/tests/fixtures/erasure-shards/legacy/manifest.json",
"sha256": "dce397d5b2f42748eb5091b4355bb9d133a50cbba003626fafac28095dbe540b",
"source": "RustFS 1.0.0-alpha.37; crates/ecstore/tests/fixtures/erasure-shards/README.md"
},
{
"path": "crates/ecstore/tests/fixtures/erasure-shards/plaintext.bin",
"sha256": "b5a83332327961a31b65a8731c64559fd04ff8ac9f6124022201dbbdba5ffa01",
"source": "Independent fixed SHA256 counter-stream oracle; crates/ecstore/tests/fixtures/erasure-shards/README.md"
}
]
}
-1
View File
@@ -1 +0,0 @@
27e9584a2d776db9416d0edfd040d524d48b5d4d
-64
View File
@@ -1,64 +0,0 @@
## —— Development/Source builds using direct buildx commands ---------------------------------------
.PHONY: docker-dev
docker-dev: ## Build dev multi-arch image (cannot load locally)
@echo "🏗️ Building multi-architecture development Docker images with buildx..."
@echo "💡 This builds from source code and is intended for local development and testing"
@echo "⚠️ Multi-arch images cannot be loaded locally, use docker-dev-push to push to registry"
$(DOCKER_CLI) buildx build \
--platform linux/amd64,linux/arm64 \
--file $(DOCKERFILE_SOURCE) \
--tag rustfs:source-latest \
--tag rustfs:dev-latest \
.
.PHONY: docker-dev-local
docker-dev-local: ## Build dev single-arch image (local load)
@echo "🏗️ Building single-architecture development Docker image for local use..."
@echo "💡 This builds from source code for the current platform and loads locally"
$(DOCKER_CLI) buildx build \
--file $(DOCKERFILE_SOURCE) \
--tag rustfs:source-latest \
--tag rustfs:dev-latest \
--load \
.
.PHONY: docker-dev-push
docker-dev-push: ## Build and push multi-arch development image # e.g (make docker-dev-push REGISTRY=xxx)
@if [ -z "$(REGISTRY)" ]; then \
echo "❌ Error: Please specify registry, example: make docker-dev-push REGISTRY=ghcr.io/username"; \
exit 1; \
fi
@echo "🚀 Building and pushing multi-architecture development Docker images..."
@echo "💡 Pushing to registry: $(REGISTRY)"
$(DOCKER_CLI) buildx build \
--platform linux/amd64,linux/arm64 \
--file $(DOCKERFILE_SOURCE) \
--tag $(REGISTRY)/rustfs:source-latest \
--tag $(REGISTRY)/rustfs:dev-latest \
--push \
.
.PHONY: dev-env-start
dev-env-start: ## Start development container environment
@echo "🚀 Starting development environment..."
$(DOCKER_CLI) buildx build \
--file $(DOCKERFILE_SOURCE) \
--tag rustfs:dev \
--load \
.
$(DOCKER_CLI) stop $(CONTAINER_NAME) 2>/dev/null || true
$(DOCKER_CLI) rm $(CONTAINER_NAME) 2>/dev/null || true
$(DOCKER_CLI) run -d --name $(CONTAINER_NAME) \
-p 9010:9010 -p 9000:9000 \
-v $(shell pwd):/workspace \
-it rustfs:dev
.PHONY: dev-env-stop
dev-env-stop: ## Stop development container environment
@echo "🛑 Stopping development environment..."
$(DOCKER_CLI) stop $(CONTAINER_NAME) 2>/dev/null || true
$(DOCKER_CLI) rm $(CONTAINER_NAME) 2>/dev/null || true
.PHONY: dev-env-restart
dev-env-restart: dev-env-stop dev-env-start ## Restart development container environment
@@ -1,41 +0,0 @@
## —— Production builds using docker buildx (for CI/CD and production) -----------------------------
.PHONY: docker-buildx
docker-buildx: ## Build production multi-arch image (no push)
@echo "🏗️ Building multi-architecture production Docker images with buildx..."
./docker-buildx.sh
.PHONY: docker-buildx-push
docker-buildx-push: ## Build and push production multi-arch image
@echo "🚀 Building and pushing multi-architecture production Docker images with buildx..."
./docker-buildx.sh --push
.PHONY: docker-buildx-version
docker-buildx-version: ## Build and version production multi-arch image # e.g (make docker-buildx-version VERSION=v1.0.0)
@if [ -z "$(VERSION)" ]; then \
echo "❌ Error: Please specify version, example: make docker-buildx-version VERSION=v1.0.0"; \
exit 1; \
fi
@echo "🏗️ Building multi-architecture production Docker images (version: $(VERSION))..."
./docker-buildx.sh --release $(VERSION)
.PHONY: docker-buildx-push-version
docker-buildx-push-version: ## Build and version and push production multi-arch image # e.g (make docker-buildx-push-version VERSION=v1.0.0)
@if [ -z "$(VERSION)" ]; then \
echo "❌ Error: Please specify version, example: make docker-buildx-push-version VERSION=v1.0.0"; \
exit 1; \
fi
@echo "🚀 Building and pushing multi-architecture production Docker images (version: $(VERSION))..."
./docker-buildx.sh --release $(VERSION) --push
.PHONY: docker-buildx-production-local
docker-buildx-production-local: ## Build production single-arch image locally
@echo "🏗️ Building single-architecture production Docker image locally..."
@echo "💡 Alternative to docker-buildx.sh for local testing"
$(DOCKER_CLI) buildx build \
--file $(DOCKERFILE_PRODUCTION) \
--tag rustfs:production-latest \
--tag rustfs:latest \
--load \
--build-arg RELEASE=latest \
.
-16
View File
@@ -1,16 +0,0 @@
## —— Single Architecture Docker Builds (Traditional) ----------------------------------------------
.PHONY: docker-build-production
docker-build-production: ## Build single-arch production image
@echo "🏗️ Building single-architecture production Docker image..."
@echo "💡 Consider using 'make docker-buildx-production-local' for multi-arch support"
$(DOCKER_CLI) build -f $(DOCKERFILE_PRODUCTION) -t rustfs:latest .
.PHONY: docker-build-source
docker-build-source: ## Build single-arch source image
@echo "🏗️ Building single-architecture source Docker image..."
@echo "💡 Consider using 'make docker-dev-local' for multi-arch support"
DOCKER_BUILDKIT=1 $(DOCKER_CLI) build \
--build-arg BUILDKIT_INLINE_CACHE=1 \
-f $(DOCKERFILE_SOURCE) -t rustfs:source .
-22
View File
@@ -1,22 +0,0 @@
## —— Docker-based build (alternative approach) ----------------------------------------------------
# Usage: make BUILD_OS=ubuntu22.04 build-docker
# Output: target/ubuntu22.04/release/rustfs
.PHONY: build-docker
build-docker: SOURCE_BUILD_IMAGE_NAME = rustfs-$(BUILD_OS):v1
build-docker: SOURCE_BUILD_CONTAINER_NAME = rustfs-$(BUILD_OS)-build
build-docker: BUILD_CMD = /root/.cargo/bin/cargo build --release --bin rustfs --target-dir /root/s3-rustfs/target/$(BUILD_OS)
build-docker: ## Build using Docker container # e.g (make build-docker BUILD_OS=ubuntu22.04)
@echo "🐳 Building RustFS using Docker ($(BUILD_OS))..."
$(DOCKER_CLI) buildx build -t $(SOURCE_BUILD_IMAGE_NAME) -f $(DOCKERFILE_SOURCE) --load .
$(DOCKER_CLI) run --rm --name $(SOURCE_BUILD_CONTAINER_NAME) -v $(shell pwd):/root/s3-rustfs -it $(SOURCE_BUILD_IMAGE_NAME) $(BUILD_CMD)
.PHONY: docker-inspect-multiarch
docker-inspect-multiarch: ## Check image architecture support
@if [ -z "$(IMAGE)" ]; then \
echo "❌ Error: Please specify image, example: make docker-inspect-multiarch IMAGE=rustfs/rustfs:latest"; \
exit 1; \
fi
@echo "🔍 Inspecting multi-architecture image: $(IMAGE)"
docker buildx imagetools inspect $(IMAGE)
-74
View File
@@ -1,74 +0,0 @@
## —— Local Native Build using build-rustfs.sh script (Recommended) --------------------------------
.PHONY: build
build: ## Build RustFS binary (includes console by default)
@echo "🔨 Building RustFS using build-rustfs.sh script..."
./build-rustfs.sh
.PHONY: build-dev
build-dev: ## Build RustFS in Development mode
@echo "🔨 Building RustFS in development mode..."
./build-rustfs.sh --dev
.PHONY: build-musl
build-musl: ## Build x86_64 musl version
@echo "🔨 Building rustfs for x86_64-unknown-linux-musl..."
@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
./build-rustfs.sh --platform x86_64-unknown-linux-musl
.PHONY: build-gnu
build-gnu: ## Build x86_64 GNU version
@echo "🔨 Building rustfs for x86_64-unknown-linux-gnu..."
@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
./build-rustfs.sh --platform x86_64-unknown-linux-gnu
.PHONY: build-musl-arm64
build-musl-arm64: ## Build aarch64 musl version
@echo "🔨 Building rustfs for aarch64-unknown-linux-musl..."
@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
./build-rustfs.sh --platform aarch64-unknown-linux-musl
.PHONY: build-gnu-arm64
build-gnu-arm64: ## Build aarch64 GNU version
@echo "🔨 Building rustfs for aarch64-unknown-linux-gnu..."
@echo "💡 On macOS/Windows, use 'make build-docker' or 'make docker-dev' instead"
./build-rustfs.sh --platform aarch64-unknown-linux-gnu
## —— Profiling build (dial9 Tokio runtime telemetry) ------------------------------------------
# dial9 hooks Tokio's unstable runtime instrumentation, so it needs
# `--cfg tokio_unstable`. That flag is deliberately absent from
# .cargo/config.toml: it is not free, and release binaries do not carry it.
# Setting RUSTFLAGS here replaces (never appends to) the config-file value, and
# crates/obs/build.rs fails the build if the feature and the flag disagree.
#
# There are no task-dump or S3-upload features — see the notes in
# crates/obs/Cargo.toml for why.
DIAL9_FEATURES ?= dial9
DIAL9_RUSTFLAGS ?= --cfg tokio_unstable
.PHONY: build-profiling
build-profiling: ## Build RustFS with dial9 Tokio runtime telemetry (diagnostic builds only)
@echo "🔬 Building RustFS with dial9 telemetry (features: $(DIAL9_FEATURES))..."
@echo "⚠️ Diagnostic build: telemetry writes trace segments to disk continuously."
RUSTFLAGS="$(DIAL9_RUSTFLAGS)" cargo build --release --bin rustfs --features $(DIAL9_FEATURES)
.PHONY: build-cross-all
build-cross-all: core-deps ## Build binaries for all architectures
@echo "🔧 Building all target architectures..."
@echo "💡 On macOS/Windows, use 'make docker-dev' for reliable multi-arch builds"
@echo "🔨 Generating protobuf code..."
cargo run --bin gproto || true
@echo "🔨 Building rustfs for x86_64-unknown-linux-musl..."
./build-rustfs.sh --platform x86_64-unknown-linux-musl
@echo "🔨 Building rustfs for x86_64-unknown-linux-gnu..."
./build-rustfs.sh --platform x86_64-unknown-linux-gnu
@echo "🔨 Building rustfs for aarch64-unknown-linux-musl..."
./build-rustfs.sh --platform aarch64-unknown-linux-musl
@echo "🔨 Building rustfs for aarch64-unknown-linux-gnu..."
./build-rustfs.sh --platform aarch64-unknown-linux-gnu
-26
View File
@@ -1,26 +0,0 @@
## —— Check and Inform Dependencies ----------------------------------------------------------------
# Fatal check
# Checks all required dependencies and exits with error if not found
# (e.g., cargo, rustfmt)
check-%:
@command -v $* >/dev/null 2>&1 || { \
echo >&2 "❌ '$*' is not installed."; \
exit 1; \
}
# Warning-only check
# Checks for optional dependencies and issues a warning if not found
warn-%:
@command -v $* >/dev/null 2>&1 || { \
echo >&2 "⚠️ '$*' is not installed."; \
}
# For checking dependencies use check-<dep-name> or warn-<dep-name>
#
# NOTE: cargo-nextest is a HARD dependency of `make test`, gated inside the
# test recipe itself (with a RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1 escape hatch)
# rather than a warn-only prerequisite here — see .config/make/tests.mak.
.PHONY: core-deps fmt-deps
core-deps: check-cargo ## Check core dependencies
fmt-deps: check-rustfmt ## Check lint and formatting dependencies
-26
View File
@@ -1,26 +0,0 @@
## —— Coverage --------------------------------------------------------------------------------------
# Local equivalent of the weekly coverage workflow (.github/workflows/coverage.yml,
# backlog#1153 infra-5): same measurement scope (--workspace --exclude e2e_test,
# nextest `ci` profile) and the same per-crate table. Slow — the instrumented
# build cannot reuse your normal target cache and then runs the whole suite.
# Doctests are not measured (needs nightly). Outputs land in target/llvm-cov/.
.PHONY: coverage
coverage: core-deps ## Workspace line coverage (cargo-llvm-cov + nextest; slow, writes target/llvm-cov/)
@if ! command -v cargo-llvm-cov >/dev/null 2>&1; then \
echo >&2 "❌ cargo-llvm-cov is required for 'make coverage' but was not found."; \
echo >&2 " Install it with:"; \
echo >&2 " cargo install cargo-llvm-cov --locked"; \
echo >&2 " rustup component add llvm-tools-preview"; \
exit 1; \
fi
@if ! command -v cargo-nextest >/dev/null 2>&1; then \
echo >&2 "❌ cargo-nextest is required for 'make coverage' (see 'make test')."; \
echo >&2 " Install it with: cargo install cargo-nextest --locked"; \
exit 1; \
fi
NEXTEST_PROFILE=ci cargo llvm-cov nextest --workspace --exclude e2e_test --no-report
@mkdir -p target/llvm-cov
cargo llvm-cov report --lcov --output-path target/llvm-cov/lcov.info
cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
$(RUSTFS_PYTHON_BIN) scripts/coverage_per_crate.py target/llvm-cov/coverage.json
-6
View File
@@ -1,6 +0,0 @@
## —— Deploy using dev_deploy.sh script ------------------------------------------------------------
.PHONY: deploy-dev
deploy-dev: build-musl ## Deploy to dev server
@echo "🚀 Deploying to dev server: $${IP}"
./scripts/dev_deploy.sh $${IP}
-38
View File
@@ -1,38 +0,0 @@
## —— Help, Help Build and Help Docker -------------------------------------------------------------
.PHONY: help
help: ## Shows This Help Menu
echo -e "$$HEADER"
grep -E '(^[a-zA-Z0-9_-]+:.*?## .*$$)|(^## )' $(MAKEFILE_LIST) | sed 's/^[^:]*://g' | awk 'BEGIN {FS = ":.*?## | #"} /^## / {printf "\n${green}%s${reset}\n", $$0; next} {printf "${cyan}%-30s${reset} ${white}%s${reset} ${green}%s${reset}\n", $$1, $$2, $$3}'
.PHONY: help-build
help-build: ## Shows RustFS build help
@echo ""
@echo "💡 build-rustfs.sh script provides more options, smart detection and binary verification"
@echo ""
@echo "🔧 Direct usage of build-rustfs.sh script:"
@echo ""
@echo " ./build-rustfs.sh --help # View script help"
@echo " ./build-rustfs.sh --no-console # Build without console resources"
@echo " ./build-rustfs.sh --force-console-update # Force update console resources"
@echo " ./build-rustfs.sh --dev # Development mode build"
@echo " ./build-rustfs.sh --sign # Sign binary files"
@echo " ./build-rustfs.sh --platform x86_64-unknown-linux-gnu # Specify target platform"
@echo " ./build-rustfs.sh --skip-verification # Skip binary verification"
@echo ""
.PHONY: help-docker
help-docker: ## Shows docker environment and suggestion help
@echo ""
@echo "📋 Environment Variables:"
@echo " REGISTRY Image registry address (required for push)"
@echo " DOCKERHUB_USERNAME Docker Hub username"
@echo " DOCKERHUB_TOKEN Docker Hub access token"
@echo " GITHUB_TOKEN GitHub access token"
@echo ""
@echo "💡 Suggestions:"
@echo " Production use: Use docker-buildx* commands (based on precompiled binaries)"
@echo " Local development: Use docker-dev* commands (build from source)"
@echo " Development environment: Use dev-env-* commands to manage dev containers"
@echo ""
-123
View File
@@ -1,123 +0,0 @@
## —— Code quality and Formatting ------------------------------------------------------------------
.NOTPARALLEL: fix
.PHONY: fmt
fmt: core-deps fmt-deps ## Format code
@echo "🔧 Formatting code..."
cargo fmt --all
.PHONY: fmt-check
fmt-check: core-deps fmt-deps ## Check code formatting
@echo "📝 Checking code formatting..."
cargo fmt --all --check
.PHONY: clippy-check
clippy-check: core-deps ## Run clippy checks
@echo "🔍 Running clippy checks..."
cargo clippy --all-targets -- -D warnings
.PHONY: clippy-fix
clippy-fix: core-deps ## Apply clippy fixes
@echo "🔧 Applying clippy fixes..."
cargo clippy --fix --allow-dirty
.PHONY: fix
fix: fmt clippy-fix ## Format code and apply clippy fixes
.PHONY: quick-check
quick-check: core-deps ## Run fast workspace compilation check
@echo "🔨 Running fast compilation check..."
cargo check --workspace --exclude e2e_test
.PHONY: unsafe-code-check
unsafe-code-check: ## Check unsafe_code allowances have SAFETY comments
@echo "🔒 Checking unsafe_code allowances..."
./scripts/check_unsafe_code_allowances.sh
.PHONY: architecture-migration-check
architecture-migration-check: ## Check architecture migration guardrails
@echo "🏗️ Checking architecture migration guardrails..."
./scripts/check_architecture_migration_rules.sh
.PHONY: logging-guardrails-check
logging-guardrails-check: ## Check logging guardrails for redaction and noise regressions
@echo "🪵 Checking logging guardrails..."
./scripts/check_logging_guardrails.sh
.PHONY: error-other-ratchet-check
error-other-ratchet-check: ## Check the ecstore ::other(format!) quorum-bucketing ratchet stays shrink-only
@echo "🪣 Checking error other(format!) ratchet..."
./scripts/check_error_other_format_ratchet.sh
.PHONY: tokio-io-uring-check
tokio-io-uring-check: ## Check tokio io-uring runtime feature stays removed
@echo "🚫 Checking tokio io-uring feature guard..."
./scripts/check_no_tokio_io_uring.sh
.PHONY: extension-schema-check
extension-schema-check: ## Check extension-schema stays a lightweight contract crate
@echo "🧩 Checking extension schema boundaries..."
./scripts/check_extension_schema_boundaries.sh
.PHONY: body-cache-whitelist-check
body-cache-whitelist-check: ## Check the body-cache eligibility gate stays a fail-closed allow-list
@echo "🧱 Checking body-cache whitelist guard..."
./scripts/check_body_cache_whitelist.sh
.PHONY: s3s-footprint-check
s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
@echo "📦 Checking s3s footprint ratchet..."
./scripts/check_s3s_footprint.sh
.PHONY: release-feature-boundary-check
release-feature-boundary-check: ## Check no release-reachable feature enables a debug-only one
@echo "🚧 Checking release feature boundaries..."
$(RUSTFS_PYTHON_BIN) ./scripts/check_connect_release_boundary.py --features-only
.PHONY: test-count-ratchet-check
test-count-ratchet-check: ## Check no workspace crate fell below its test floor or rose above its #[ignore] ceiling in the committed baseline
@echo "🧪 Checking per-crate test-count ratchet..."
./scripts/check_test_count_ratchet.sh
.PHONY: test-seams-check
test-seams-check: ## Check tests outside an operation's owner call its use-case only through rustfs/src/app/test_seams
@echo "🧪 Checking cross-owner test seams..."
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_seams.py
.PHONY: test-retirements-check
test-retirements-check: ## Check every test retired by a migration slice names a replacement that exists
@echo "🧪 Checking test retirement tables..."
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_retirements.py
.PHONY: fips-wording-check
fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
@echo "📣 Checking cryptographic capability wording guard..."
./scripts/check_fips_wording.sh
.PHONY: embedded-secrets-check
embedded-secrets-check: ## Check no private key material or credential literal is committed
@echo "🔑 Checking embedded secret material guard..."
./scripts/check_embedded_secrets.sh
.PHONY: offline-enrollment-e2e-check
offline-enrollment-e2e-check: core-deps ## Build and exercise the dedicated offline enrollment E2E root
@echo "🔐 Checking the offline enrollment E2E root boundary..."
./scripts/check_offline_enrollment_e2e.sh
.PHONY: test-wiring-check
test-wiring-check: ## Check tests stay registered and selected by their intended runners
@echo "🧪 Checking test wiring..."
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py
$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --check-workflow
$(RUSTFS_PYTHON_BIN) ./scripts/validation_registry.py check
.PHONY: log-analyzer-rules-check
log-analyzer-rules-check: core-deps ## Check log-analyzer rule anchors still exist verbatim in source
@echo "🩺 Checking log-analyzer rule anchors..."
./scripts/check_log_analyzer_rules.sh
.PHONY: compilation-check
compilation-check: core-deps ## Run compilation check
@echo "🔨 Running compilation check..."
cargo check --all-targets
-32
View File
@@ -1,32 +0,0 @@
## —— Pre Commit Checks ----------------------------------------------------------------------------
.NOTPARALLEL: pre-commit pre-pr dev-check
.PHONY: setup-hooks
setup-hooks: ## Install the configured pre-commit hooks
@echo "🔧 Setting up git hooks..."
pre-commit validate-config
pre-commit install
@echo "✅ Git hooks setup complete!"
.PHONY: doc-paths-check
doc-paths-check: ## Check that instruction/architecture docs reference existing file paths
@echo "📄 Checking doc path references..."
./scripts/check_doc_paths.sh
.PHONY: planning-docs-check
planning-docs-check: ## Check that no planning-type documents are committed
@echo "📄 Checking for committed planning docs..."
./scripts/check_no_planning_docs.sh
.PHONY: pre-commit
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check release-feature-boundary-check test-count-ratchet-check test-seams-check test-retirements-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
@echo "✅ All pre-commit checks passed!"
.PHONY: pre-pr
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check release-feature-boundary-check test-count-ratchet-check test-seams-check test-retirements-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
@echo "✅ All pre-PR checks passed!"
.PHONY: dev-check
dev-check: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast local development checks
@echo "✅ Fast development checks passed!"
-161
View File
@@ -1,161 +0,0 @@
## —— Tests and e2e test ---------------------------------------------------------------------------
TEST_THREADS ?= 1
# cargo-nextest is a HARD dependency of `make test`.
#
# nextest changes test semantics vs plain `cargo test`: it runs every test in
# its own process (so serial_test's #[serial] mutex does not serialize across
# tests) and it is the only runner that honours .config/nextest.toml
# [test-groups] (e.g. the ecstore-serial-flaky serialization guard). CI runs
# nextest (.github/actions/setup/action.yml installs it), so a silent fallback
# to `cargo test` would run with different serialization behaviour than CI and
# mask (or invent) flakes.
#
# Installing cargo-nextest:
# cargo install cargo-nextest --locked # from source
# # or a prebuilt binary (faster) via the taiki-e installer / get.nexte.st:
# # https://nexte.st/docs/installation/
#
# Escape hatch: set RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1 to run the plain
# `cargo test` fallback anyway. Results are NOT authoritative — semantics
# differ from CI and .config/nextest.toml test-groups will NOT apply.
.PHONY: script-tests
.PHONY: release-package-tests
release-package-tests: ## Run release package integration on Linux with fpm, dpkg and rpm installed
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_packages.py
.PHONY: release-image-tests
release-image-tests: ## Run OCI integration with Docker, Go and skopeo; set RELEASE_IMAGE_VARIANT and RELEASE_IMAGE_OUTPUT
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_images.py
script-tests: ## Run shell script tests
@echo "Running script tests..."
trap 'status=$$?; printf "ERROR: script-tests failed (exit %s): %s\n" "$$status" "$$BASH_COMMAND" >&2; exit "$$status"' ERR
./scripts/test_build_rustfs_options.sh
./scripts/test_docker_runtime_timezone.sh
./scripts/test_entrypoint_credentials.sh
./scripts/test_entrypoint_directories.sh
./scripts/test_internode_grpc_ab_bench.sh
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_grpc_window.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_rpc_timeout_boundary.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_msgpack_peer_canary.py
./scripts/test_internode_msgpack_peer_canary_runner.sh
./scripts/test_object_batch_bench_enhanced.sh
$(RUSTFS_PYTHON_BIN) ./scripts/test_inline_docker_lab.py
./scripts/test_package_service_scripts.sh
./scripts/test_hotpath_warp_ab_gate.sh
./scripts/test_hotpath_warp_abba.sh
./scripts/test_scanner_validation_harness.sh
./scripts/test_scanner_heal_checkpoint_crash_evidence.sh
./scripts/test_scanner_heal_authority_evidence.sh
./scripts/test_scanner_heal_scoped_ack_evidence.sh
./scripts/test_scanner_heal_legacy_rollback_evidence.sh
./scripts/test_scanner_heal_g14_multiset_evidence.sh
./scripts/test_scanner_heal_scheduler_pressure_evidence.sh
./scripts/test_scanner_heal_status_outcome_evidence.sh
./scripts/test_scanner_heal_maintenance_evidence.sh
./scripts/test_scanner_heal_w13_mrf_evidence.sh
./scripts/test_scanner_heal_w16_recovery_evidence.sh
./scripts/test_exact_1mib_handoff_abba.sh
./scripts/test_pinned_paired_abba_bench.sh
./scripts/test_manual_transition_runbooks.sh
./scripts/test_fuzz_runner.sh
./scripts/test_python_bin.sh
$(RUSTFS_PYTHON_BIN) ./scripts/test_check_doc_paths.py
./scripts/check_embedded_secrets.sh --self-test
./scripts/check_s3s_footprint.sh --self-test
./scripts/test_check_test_count_ratchet.sh
./scripts/test_guard_source_set.sh
./scripts/test_split_guard_rules.sh
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/split_bucket_usecase.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/split_ecfs_legacy_edge.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/split_lifecycle_transition_api_test.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_seams.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_retirements.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_download_release_asset.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_reference_binary.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/release -p "test_*.py"
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_glibc.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_upgrade_workflow.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_migration_gate_evidence.py
$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_connect_release_boundary.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_artifact.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_merge_group_proof.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_merge_queue.py
$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/ci -p 'test_*.py'
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_functional_chain_health.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_repro_rejoin_readiness_docker.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_validation_registry.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_dual_stack.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_mint_stack_diff.py
$(RUSTFS_PYTHON_BIN) ./scripts/check_conformance_rulings.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/compat_matrix_report.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_api_coverage.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_reference_parity.py
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
./scripts/validate_object_data_cache_cold_stampede.sh --self-test
./scripts/run_scanner_heal_evidence_case.sh --self-test
.PHONY: test
test: core-deps script-tests ## Run all tests (needs cargo-nextest; RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1 to override)
@echo "🧪 Running tests..."
@if command -v cargo-nextest >/dev/null 2>&1; then \
cargo nextest run --all --exclude e2e_test; \
elif [ "$${RUSTFS_ALLOW_CARGO_TEST_FALLBACK:-0}" = "1" ]; then \
echo >&2 "⚠️ ============================================================================"; \
echo >&2 "⚠️ cargo-nextest NOT found — running the 'cargo test' fallback (opt-in)."; \
echo >&2 "⚠️ TEST SEMANTICS DIFFER FROM CI; results are NOT authoritative:"; \
echo >&2 "⚠️ * nextest runs each test in its own process; 'cargo test' does not,"; \
echo >&2 "⚠️ so serial_test #[serial] serialization behaves differently."; \
echo >&2 "⚠️ * .config/nextest.toml [test-groups] will NOT apply (e.g. the"; \
echo >&2 "⚠️ ecstore-serial-flaky group), so load-sensitive tests may flake here"; \
echo >&2 "⚠️ but pass on CI (or vice versa)."; \
echo >&2 "⚠️ Install cargo-nextest and re-run before trusting these results."; \
echo >&2 "⚠️ ============================================================================"; \
cargo test --workspace --exclude e2e_test -- --nocapture --test-threads="$(TEST_THREADS)"; \
else \
echo >&2 "❌ cargo-nextest is required for 'make test' but was not found."; \
echo >&2 ""; \
echo >&2 " RustFS tests run under cargo-nextest (process-per-test isolation)."; \
echo >&2 " CI runs nextest and .config/nextest.toml [test-groups] only take effect"; \
echo >&2 " under nextest. Plain 'cargo test' has different serialization semantics"; \
echo >&2 " and is NOT a faithful substitute."; \
echo >&2 ""; \
echo >&2 " Install it with either:"; \
echo >&2 " cargo install cargo-nextest --locked"; \
echo >&2 " or a prebuilt binary (faster) — see https://nexte.st/docs/installation/"; \
echo >&2 ""; \
echo >&2 " To run the plain 'cargo test' fallback anyway (results NOT authoritative;"; \
echo >&2 " serialization semantics differ from CI), re-run with:"; \
echo >&2 " RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1 make test"; \
exit 1; \
fi
cargo test --all --doc
.PHONY: e2e-server
e2e-server: ## Run e2e-server tests
sh $(shell pwd)/scripts/run.sh
.PHONY: probe-e2e
probe-e2e: ## Probe e2e tests
sh $(shell pwd)/scripts/probe.sh
-11
View File
@@ -1,11 +0,0 @@
# Committed floor for the number of tests selected by the migration-critical
# CI gate (see scripts/check_migration_gate_count.sh, backlog#1153 infra-12).
#
# The floor equals the exact count of rustfs-ecstore --lib tests, with the
# test-util feature enabled, matching the gate filter (name substrings:
# data_movement, rebalance, decommission, source_cleanup, delete_marker) at
# the time this file was last updated.
# CI fails if the selected count drops below this number, so renames or
# removals that thin the gate must update this file in the same PR.
# Adding tests does not require a bump, but bumping keeps the guard tight.
946
-1034
View File
File diff suppressed because it is too large Load Diff
-20
View File
@@ -1,20 +0,0 @@
# Pinned RustFS release used as the "reference" target of parity lanes
# (.github/workflows/reference-parity.yml). This is the single place that names
# the reference release; .github/actions/reference-binary resolves, downloads and
# verifies it, and fails on a missing pin, a missing asset or a digest mismatch.
#
# Bumping the reference is a reviewed event: every per-test difference between
# the candidate and the reference is measured against this exact build.
#
# Each sha256 is the value published in the release's SHA256SUMS asset, which
# matches the digest GitHub reports for the same asset. The action checks both.
schema = 1
repository = "rustfs/rustfs"
version = "1.0.5"
tag = "1.0.5"
[platforms.linux-x86_64-gnu]
asset = "rustfs-linux-x86_64-gnu-v1.0.5.zip"
sha256 = "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
executable = "rustfs"
-24
View File
@@ -1,24 +0,0 @@
# Targets of the s3-tests reference parity lane (.github/workflows/reference-parity.yml).
#
# Exactly one target uses `binary = "reference"` (the release pinned in
# .config/reference-binary.toml). Every other target uses `binary = "candidate"`
# (the binary built from the branch under test) and is compared per node id
# against the reference. All targets run the same harness, test list and server
# configuration; `env` adds only the target's own RUSTFS_* settings.
#
# Adding a target is one more [[targets]] table, for example:
#
# [[targets]]
# name = "candidate-gateway"
# binary = "candidate"
# env = { RUSTFS_S3_STACK = "gateway" }
schema = 1
[[targets]]
name = "reference"
binary = "reference"
[[targets]]
name = "candidate-legacy"
binary = "candidate"
-12
View File
@@ -1,12 +0,0 @@
# Tolerated per-test differences of the s3-tests reference parity lane.
#
# One entry per line, four fields separated by " | ":
#
# <target> | <pytest node id> | <tracking issue URL> | <reason>
#
# <target> is a candidate name from .config/reference-parity-targets.toml, the
# node id is exactly as the lane prints it (s3tests/functional/test_s3.py::...),
# and the issue URL is https://github.com/rustfs/<repo>/issues/<number>.
#
# Same semantics as scripts/junit_diff.py --allow: an entry whose difference no
# longer occurs is stale and fails the lane, so remove it when it is fixed.
-135
View File
@@ -1,135 +0,0 @@
{
"schema_version": 1,
"plan_id": "rustfs-release-validation",
"description": "Certification for immutable candidates. All shipped platforms require native identity and startup; stable and prerelease require the pinned upgrade matrix. Linux-only acceptance never authorizes publication.",
"release_types": [
"preview",
"prerelease",
"stable"
],
"targets": [
{
"os": "linux",
"arch": "x86_64",
"abi": "musl"
},
{
"os": "linux",
"arch": "aarch64",
"abi": "musl"
},
{
"os": "linux",
"arch": "x86_64",
"abi": "gnu"
},
{
"os": "linux",
"arch": "aarch64",
"abi": "gnu"
},
{
"os": "macos",
"arch": "aarch64",
"abi": "none"
},
{
"os": "windows",
"arch": "x86_64",
"abi": "msvc"
}
],
"targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
"features": [
"pyroscope"
],
"trusted_workflows": [
{
"repository": "rustfs/rustfs",
"workflow_id": "release-candidate.yml"
},
{
"repository": "rustfs/rustfs",
"workflow_id": "release-complete.yml"
}
],
"trusted_workflows_basis": "Receipts are produced by the reviewed main workflows and fetched by exact run ID with GitHub artifact digest verification. Candidate inputs freeze the harness commit separately from product source.",
"lanes": [
{
"lane": "identity",
"harness_repository": "rustfs/rustfs",
"harness_sha": "candidate.workflow_sha",
"harness_basis": "The exact workflow commit is frozen in candidate inputs; no mutable branch is accepted as a harness identity.",
"timeout_seconds": 1800,
"environment_capabilities": [
"process-exec"
],
"cases": [
"artifact-identity",
"startup"
],
"applies_to": {
"release_types": [
"preview",
"prerelease",
"stable"
],
"platforms": "all"
},
"applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
},
{
"lane": "upgrade",
"harness_repository": "rustfs/rustfs",
"harness_sha": "candidate.workflow_sha",
"harness_basis": "The exact workflow commit is frozen in candidate inputs; no mutable branch is accepted as a harness identity.",
"timeout_seconds": 5400,
"environment_capabilities": [
"process-exec",
"previous-release-artifact"
],
"cases": [
"direct-upgrade",
"mixed-version-upgrade",
"bucket-config-upgrade",
"bucket-config-rollback",
"multipart-layout-upgrade",
"multipart-layout-baseline"
],
"cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
"applies_to": {
"release_types": [
"prerelease",
"stable"
],
"platforms": [
{
"os": "linux",
"arch": "x86_64",
"abi": "gnu"
}
]
},
"applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
}
],
"upgrade_sources": {
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
"sources": [
{
"version": "1.0.1",
"role": "N-1",
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
"artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
},
{
"version": "1.0.0-rc.5",
"role": "N-2",
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
"artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
}
],
"rollback_supported": false,
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
}
}
-37
View File
@@ -1,37 +0,0 @@
# Files that may still reference s3s (`s3s::` or `s3s_sigv4`) when Phase 1 of
# the gateway migration ends (rustfs/backlog#2734 task T0.6, rustfs/backlog#2741).
# Consumed only by `scripts/check_s3s_footprint.sh --mode allowlist`.
#
# One entry per line: a repository-relative file path or a git `:(glob)`
# pathspec. `**` matches across directories, `*` stays within one path
# component, and a directory entry matches everything under it. Blank lines
# and lines starting with `#` are ignored; an entry may not contain whitespace,
# so a note belongs on its own comment line.
#
# Every Rust file listed by `git grep -l '\bs3s\(::\|_sigv4\)' -- '*.rs'` that
# no entry matches fails the allowlist mode. Task T1.9 switches CI to that
# mode; until then `--dry-run` prints the files still outside this list, which
# is the Phase 1 tracking list.
#
# Never add crates/** other than crates/e2e_test: the engine crates leave s3s
# entirely, and the script rejects such an entry. Only the s3s-facing edge of
# the rustfs binary and the e2e oracle keep it.
# rustfs binary: the s3s-facing edge that survives Phase 1
rustfs/src/storage/ecfs.rs
rustfs/src/storage/s3_compat/**
rustfs/src/storage/access.rs
rustfs/src/auth.rs
rustfs/src/error.rs
rustfs/src/storage_api.rs
rustfs/src/app/storage_api.rs
rustfs/src/admin/router.rs
rustfs/src/app/metadata_route.rs
rustfs/src/server/http.rs
rustfs/src/server/layer.rs
rustfs/src/server/hybrid.rs
rustfs/src/protocols/client.rs
# e2e oracle: the test target that verifies S3 behaviour against s3s
crates/e2e_test/src/fake_s3_target/**
crates/e2e_test/src/**
-18
View File
@@ -1,18 +0,0 @@
# Baseline for the direct s3s footprint, consumed only by
# scripts/check_s3s_footprint.sh (rustfs/backlog#1677; rustfs/backlog#2734 T0.4;
# comparison rules on rustfs/backlog#2765).
#
# Each line is a ceiling everywhere: a counter above it fails, because new code
# widened the s3s surface; route it through the gateway abstractions instead and
# never raise a value here. Pull requests and merge groups are additionally held
# to their base revision's measurement, so a slice that shrinks the surface need
# not edit this file. A counter below its line passes and the check prints the
# lowered line; the migration coordinator lowers the lines in batches.
#
# files Rust files referencing s3s paths, repo-wide minus crates/e2e_test
# s3_error_lines lines holding s3_error!, same scope, minus those the script
# header's rule clears (rustfs_s3_types's macro); fails closed
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
files=63
s3_error_lines=209
ecstore_files=0
-449
View File
@@ -1,449 +0,0 @@
{
"schema": 2,
"cases": {
"background-target-restart": {
"gate": "G14",
"task": "W21",
"lane": "e2e-nightly",
"suite": "e2e_test",
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_restart",
"oracle": "background-target-restart.json",
"evidence": "process-restart",
"unclean_shutdown_marker": false,
"min_objects": 9,
"max_objects": 65,
"topology": {"nodes": 4, "drives_per_node": 1},
"erasure": {"data_blocks": 2, "parity_blocks": 2},
"scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
},
"background-target-crash": {
"gate": "G14",
"task": "W21",
"lane": "e2e-nightly",
"suite": "e2e_test",
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_crash",
"oracle": "background-target-crash.json",
"evidence": "process-crash-restart",
"unclean_shutdown_marker": true,
"min_objects": 9,
"max_objects": 65,
"topology": {"nodes": 4, "drives_per_node": 1},
"erasure": {"data_blocks": 2, "parity_blocks": 2},
"scope": "Target process killed during partial background rebuild, real unclean-shutdown marker, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
},
"ec84-target-drive-restart": {
"gate": "G14",
"task": "W20/W21",
"lane": "e2e-distributed",
"suite": "e2e_test",
"name": "distributed::heal_test::three_node_four_drive_ec8_4_root_heal_rebuilds_replaced_drive_after_restart",
"oracle": "ec84-target-drive-restart.json",
"evidence": "process-restart",
"unclean_shutdown_marker": false,
"min_objects": 5,
"max_objects": 5,
"topology": {"nodes": 3, "drives_per_node": 4},
"erasure": {"data_blocks": 8, "parity_blocks": 4},
"erasure_set_drive_count": 12,
"scope": "3-node x 4-drive single-set EC8+4, graceful target restart, preformatted replacement drive, exact unversioned S3 bodies and physical target shards; not mixed-version, multi-pool or long-window ABBA."
},
"background-target-restart-ec8-4": {
"gate": "G14",
"task": "W21",
"lane": "e2e-nightly",
"suite": "e2e_test",
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_after_background_target_restart",
"oracle": "background-target-restart-ec8-4.json",
"evidence": "process-restart",
"unclean_shutdown_marker": false,
"min_objects": 9,
"max_objects": 65,
"topology": {"nodes": 3, "drives_per_node": 4},
"erasure": {"data_blocks": 8, "parity_blocks": 4},
"erasure_set_drive_count": 12,
"scope": "Target process restart during partial background rebuild on a single 3x4 EC8+4 set; exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool."
},
"background-target-crash-ec8-4": {
"gate": "G14",
"task": "W21",
"lane": "e2e-nightly",
"suite": "e2e_test",
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_after_background_target_crash",
"oracle": "background-target-crash-ec8-4.json",
"evidence": "process-crash-restart",
"unclean_shutdown_marker": true,
"min_objects": 9,
"max_objects": 65,
"topology": {"nodes": 3, "drives_per_node": 4},
"erasure": {"data_blocks": 8, "parity_blocks": 4},
"erasure_set_drive_count": 12,
"scope": "Target process killed during partial background rebuild on a single 3x4 EC8+4 set; real unclean-shutdown marker, exact unversioned S3 bodies and replacement-drive shards; not power loss, multi-set, or multi-pool."
},
"background-target-restart-ec8-4-multi-set": {
"gate": "G14",
"task": "W21",
"lane": "e2e-nightly",
"suite": "e2e_test",
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_set_after_background_target_restart",
"oracle": "background-target-restart-ec8-4-multi-set.json",
"evidence": "process-restart",
"unclean_shutdown_marker": false,
"min_objects": 9,
"max_objects": 65,
"topology": {"nodes": 3, "drives_per_node": 8},
"erasure": {"data_blocks": 8, "parity_blocks": 4},
"erasure_set_drive_count": 12,
"sets": 2,
"pools": 1,
"scope": "Target process restart during partial background rebuild on a 3x8 EC8+4 layout with two erasure sets in one pool; exact unversioned S3 bodies and replacement-drive shards; not power loss or multi-pool."
},
"background-target-crash-ec8-4-multi-pool": {
"gate": "G14",
"task": "W21",
"lane": "e2e-nightly",
"suite": "e2e_test",
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash",
"oracle": "background-target-crash-ec8-4-multi-pool.json",
"evidence": "process-crash-restart",
"unclean_shutdown_marker": true,
"min_objects": 9,
"max_objects": 65,
"topology": {"nodes": 3, "drives_per_node": 12},
"erasure": {"data_blocks": 8, "parity_blocks": 4},
"erasure_set_drive_count": 12,
"sets": 3,
"pools": 3,
"outage_target_manifest_required": false,
"scope": "Target process crash during partial background rebuild on three single-node EC8+4 pools; exact baseline S3 bodies and replacement-drive shards, with down-window outage PUT refusal recorded and a deferred post-rejoin outage object verified through S3 when the full target pool was offline."
}
},
"release_lanes": {
"single-set-restart": {
"status": "implemented",
"cases": ["background-target-restart", "background-target-crash"],
"covers": ["four-node one-drive topology", "unversioned objects", "target restart/crash"]
},
"authority-coverage": {
"status": "pending",
"gates": ["G01", "G12"],
"requires": ["root authority coverage", "quota authority coverage"]
},
"checkpoint-and-crash": {
"status": "pending",
"gates": ["G02", "G04", "R-E"],
"requires": ["bounded checkpoint progress", "boundary crash matrix", "fixed-budget restart evidence"]
},
"status-and-outcome": {
"status": "pending",
"gates": ["G05", "G06", "R-D"],
"requires": ["per-object outcomes", "legacy status clients", "manager/event/ledger disposition"]
},
"mrf-responsibility": {
"status": "pending",
"gates": ["G07", "G08", "P4"],
"requires": ["durable MRF responsibility", "disk-full and replica-loss matrix", "MRF replay cost"]
},
"mixed-version-rollback": {
"status": "pending",
"gates": ["G03", "G09", "R-L"],
"requires": ["mixed-version peers", "rollback payloads", "crash-safe source retirement"]
},
"scheduler-pressure": {
"status": "pending",
"gates": ["G10", "P1", "P2", "P3"],
"requires": ["bounded scheduling", "foreground latency and throughput", "two-hour pressure evidence"]
},
"maintenance-producers": {
"status": "pending",
"gates": ["G11", "G13"],
"requires": ["complete producer coverage", "quorum-minus-one and remount matrix"]
},
"ec8-4-multiset": {
"status": "pending",
"gates": ["G14"],
"requires": ["3x4 EC8+4 topology", "multi-set coverage", "multi-pool coverage"]
}
},
"release_requirements": [
{
"gate": "G01",
"task": "W02/W04",
"lane": "authority-coverage",
"status": "pending",
"description": "Complete root and quota authority coverage",
"requires": ["root authority evidence", "quota authority evidence"],
"evidence_fields": [
"root_authority_evidence",
"quota_authority_evidence"
]
},
{
"gate": "G02",
"task": "W03",
"lane": "checkpoint-and-crash",
"status": "pending",
"description": "Bounded checkpoint progress and independent version inventory",
"requires": ["bounded checkpoint oracle", "independent version inventory"],
"evidence_fields": [
"bounded_checkpoint_oracle",
"independent_version_inventory"
]
},
{
"gate": "G03",
"task": "W17/W18",
"lane": "mixed-version-rollback",
"status": "pending",
"description": "Exact scoped ACK with durable publication and mixed peers",
"requires": ["durable scoped ACK publication", "mixed-peer evidence"],
"evidence_fields": [
"durable_root_publication_proof",
"scoped_ack_request_identity",
"participating_peer_capability_snapshot",
"mixed_peer_ack_fallback_oracle"
]
},
{
"gate": "G04",
"task": "W03/W15/W16",
"lane": "checkpoint-and-crash",
"status": "pending",
"description": "Crash at every cache, root, floor and intent boundary",
"requires": ["cache boundary crash evidence", "root/floor/intent crash evidence"],
"evidence_fields": [
"cache_boundary_crash_evidence",
"root_floor_intent_crash_evidence"
]
},
{
"gate": "G05",
"task": "W06/W07",
"lane": "status-and-outcome",
"status": "pending",
"description": "Per-object outcomes and bounded terminal retention",
"requires": ["per-object outcome oracle", "terminal retention bounds"],
"evidence_fields": [
"per_object_outcome_oracle",
"terminal_retention_bounds"
]
},
{
"gate": "G06",
"task": "W06/W08/W23",
"lane": "status-and-outcome",
"status": "pending",
"description": "Concurrent status, legacy clients and truncation",
"requires": ["concurrent status evidence", "legacy client compatibility", "truncation behavior"],
"evidence_fields": [
"concurrent_status_evidence",
"legacy_client_compatibility",
"truncation_behavior"
]
},
{
"gate": "G07",
"task": "W12/W13/W14",
"lane": "mrf-responsibility",
"status": "pending",
"description": "Durable MRF responsibility at every commit boundary",
"requires": ["MRF responsibility oracle", "commit-boundary crash matrix"],
"evidence_fields": [
"mrf_responsibility_oracle",
"commit_boundary_crash_matrix"
]
},
{
"gate": "G08",
"task": "W12/W13/W14",
"lane": "mrf-responsibility",
"status": "pending",
"description": "MRF capacity, disk-full and replica-loss matrix",
"requires": ["MRF capacity evidence", "disk-full matrix", "replica-loss matrix"],
"evidence_fields": [
"mrf_capacity_evidence",
"disk_full_matrix",
"replica_loss_matrix"
]
},
{
"gate": "G09",
"task": "W13/W18/W23",
"lane": "mixed-version-rollback",
"status": "pending",
"description": "Actual mixed-version reader/writer and rollback payloads",
"requires": ["mixed-version reader evidence", "mixed-version writer evidence", "rollback payload evidence"],
"evidence_fields": [
"mixed_version_reader_evidence",
"mixed_version_writer_evidence",
"rollback_payload_evidence"
]
},
{
"gate": "G10",
"task": "W05/W09/W10/W11",
"lane": "scheduler-pressure",
"status": "pending",
"description": "Bounded scheduling and pressure recovery",
"requires": ["scheduler bound evidence", "pressure recovery evidence"],
"evidence_fields": [
"scheduler_bound_evidence",
"pressure_recovery_evidence"
]
},
{
"gate": "G11",
"task": "W04/W19/W24",
"lane": "maintenance-producers",
"status": "pending",
"description": "Maintenance and complete producer coverage",
"requires": [
"maintenance producer matrix",
"complete producer inventory",
"segment activation preflight"
],
"evidence_fields": [
"maintenance_producer_matrix",
"complete_producer_inventory",
"segment_activation_preflight"
]
},
{
"gate": "G12",
"task": "W02/W15/W16",
"lane": "authority-coverage",
"status": "pending",
"description": "Both quota paths during reset and settlement",
"requires": ["reset quota-path evidence", "settlement quota-path evidence"],
"evidence_fields": [
"reset_quota_path_evidence",
"settlement_quota_path_evidence"
]
},
{
"gate": "G13",
"task": "W07/W14",
"lane": "maintenance-producers",
"status": "pending",
"description": "Quorum-minus-one, unknown disks, remount, Object Lock, dry-run, grace and commit tail",
"requires": ["quorum-minus-one matrix", "unknown-disk/remount matrix", "Object Lock dry-run grace evidence"],
"evidence_fields": [
"quorum_minus_one_matrix",
"unknown_disk_remount_matrix",
"object_lock_dry_run_grace_evidence"
]
},
{
"gate": "G14",
"task": "W20/W21",
"lane": "ec8-4-multiset",
"status": "pending",
"description": "Same-window field evidence with 3x4 EC8+4 and multi-set/pool coverage",
"requires": [
"same-window field evidence",
"3x4 EC8+4 evidence",
"multi-set evidence",
"multi-pool evidence",
"distributed segment invalidation evidence"
],
"evidence_fields": [
"same_window_field_evidence",
"ec8_4_evidence",
"multi_set_evidence",
"multi_pool_evidence",
"distributed_segment_invalidation_evidence"
]
},
{
"gate": "P1",
"task": "W20",
"lane": "scheduler-pressure",
"status": "pending",
"description": "Measured cold-walk share and foreground latency/throughput",
"requires": ["cold-walk share measurement", "foreground latency/throughput measurement"],
"evidence_fields": [
"cold_walk_share_measurement",
"foreground_latency_throughput_measurement",
"profile_evidence"
]
},
{
"gate": "P2",
"task": "W20/W24",
"lane": "scheduler-pressure",
"status": "pending",
"description": "Measured post-stop convergence and cold segment reuse",
"requires": ["post-stop convergence measurement", "cold segment reuse measurement"],
"evidence_fields": [
"post_stop_convergence_measurement",
"cold_segment_reuse_measurement"
]
},
{
"gate": "P3",
"task": "W20",
"lane": "scheduler-pressure",
"status": "pending",
"description": "Measured two-hour pressure/heal capacity and recovery window",
"requires": ["two-hour pressure measurement", "heal capacity measurement", "recovery-window measurement"],
"evidence_fields": [
"two_hour_pressure_measurement",
"heal_capacity_measurement",
"recovery_window_measurement"
]
},
{
"gate": "P4",
"task": "W20",
"lane": "mrf-responsibility",
"status": "pending",
"description": "Measured MRF scale and replay cost with retained responsibility",
"requires": ["MRF scale measurement", "MRF replay-cost measurement", "retained responsibility evidence", "cleanup/GC soak evidence"],
"evidence_fields": [
"mrf_scale_measurement",
"mrf_replay_cost_measurement",
"retained_responsibility_evidence",
"mrf_cleanup_gc_soak_evidence"
]
},
{
"gate": "R-E",
"task": "W03/W05",
"lane": "checkpoint-and-crash",
"status": "pending",
"description": "Fixed-budget real process restart through enumeration and classification",
"requires": ["fixed-budget restart evidence", "enumeration evidence", "classification evidence"],
"evidence_fields": [
"fixed_budget_restart_evidence",
"enumeration_evidence",
"classification_evidence"
]
},
{
"gate": "R-D",
"task": "W07/W14",
"lane": "status-and-outcome",
"status": "pending",
"description": "Manager-to-event-to-ledger exact disposition, including grace",
"requires": ["manager disposition evidence", "event disposition evidence", "ledger disposition evidence", "grace handling"],
"evidence_fields": [
"manager_disposition_evidence",
"event_disposition_evidence",
"ledger_disposition_evidence",
"grace_handling"
]
},
{
"gate": "R-L",
"task": "W13/W14",
"lane": "mixed-version-rollback",
"status": "pending",
"description": "Legacy source conflicts, migration gaps and crash-safe source retirement",
"requires": ["legacy source-conflict evidence", "migration-gap evidence", "crash-safe source retirement evidence"],
"evidence_fields": [
"legacy_source_conflict_evidence",
"migration_gap_evidence",
"crash_safe_source_retirement_evidence"
]
}
]
}
-12
View File
@@ -1,12 +0,0 @@
# Committed floor for the number of security negative-auth tests selected by the
# e2e-smoke PR profile (see scripts/check_security_smoke_count.sh, backlog#1151
# sec-5).
#
# The floor equals the exact count of e2e_test cases whose name starts with a
# security module prefix (negative_sigv4_test, presigned_negative_test,
# admin_auth_test) that the [profile.e2e-smoke] default-filter in
# .config/nextest.toml selects, at the time this file was last updated. CI fails
# if the selected count drops below this number, so a rename or removal that
# thins the security smoke gate must update this file in the same PR.
# Adding tests does not require a bump, but bumping keeps the guard tight.
28
-67
View File
@@ -1,67 +0,0 @@
# Per-crate test-count baseline consumed by scripts/check_test_count_ratchet.sh
# (rustfs/backlog#2734 section 1 rule 3; T0.3 in rustfs/backlog#2738).
#
# One row per workspace member: <package>, tests, ignored, tab-separated.
# tests counts #[test], #[tokio::test], #[tokio::test(...)] and #[test_case(...)]
# attribute lines under the package directory; ignored counts #[ignore] and
# #[ignore = "..."]. The tests column is a floor and the ignored column is a
# ceiling: CI fails when a crate's tests fall below its row or its ignored count
# rises above it. Extra tests and fewer ignores pass; --update tightens the rows.
# Lowering a tests row or raising an ignored row is a reviewed statement that
# tests were intentionally removed or ignored; justify it in the PR description.
# Regenerate with:
# scripts/check_test_count_ratchet.sh --update
e2e_test 1026 20
rustfs 6449 64
rustfs-audit 70 0
rustfs-checksums 28 1
rustfs-common 36 0
rustfs-concurrency 87 0
rustfs-config 43 0
rustfs-connect-symbol-catalog 6 0
rustfs-credentials 24 0
rustfs-crypto 67 0
rustfs-data-usage 58 0
rustfs-ecstore 7858 3
rustfs-extension-schema 23 0
rustfs-filemeta 361 0
rustfs-gateway-adapter 74 0
rustfs-heal 1128 3
rustfs-heal-contracts 8 0
rustfs-iam 415 0
rustfs-io-core 52 0
rustfs-io-metrics 157 0
rustfs-keystone 31 0
rustfs-kms 716 25
rustfs-license 2 0
rustfs-lifecycle 156 0
rustfs-lock 182 0
rustfs-log-analyzer 91 0
rustfs-madmin 210 0
rustfs-notify 140 0
rustfs-object-capacity 92 0
rustfs-object-data-cache 141 0
rustfs-obs 387 0
rustfs-policy 758 0
rustfs-protocols 770 1
rustfs-protos 75 0
rustfs-replication 209 0
rustfs-rio 225 1
rustfs-rio-v2 34 6
rustfs-s3-client 71 0
rustfs-s3-ops 8 0
rustfs-s3-types 105 0
rustfs-s3select-api 232 0
rustfs-s3select-query 235 2
rustfs-scanner 1392 20
rustfs-scanner-metrics 90 0
rustfs-security-governance 20 0
rustfs-signer 34 0
rustfs-storage-api 34 0
rustfs-targets 612 26
rustfs-test-utils 1 0
rustfs-tls-runtime 44 0
rustfs-trusted-proxies 72 0
rustfs-utils 253 1
rustfs-zip 30 0
zstd 11 0
@@ -1,3 +0,0 @@
# Tests the bucket_access slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
@@ -1,3 +0,0 @@
# Tests the bucket_configs slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-3
View File
@@ -1,3 +0,0 @@
# Tests the bucket_core slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
@@ -1,3 +0,0 @@
# Tests the bucket_replication_lifecycle slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-3
View File
@@ -1,3 +0,0 @@
# Tests the cutover slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
@@ -1,3 +0,0 @@
# Tests the listing_multipart slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
@@ -1,3 +0,0 @@
# Tests the object_commit slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-3
View File
@@ -1,3 +0,0 @@
# Tests the object_meta slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-3
View File
@@ -1,3 +0,0 @@
# Tests the object_misc slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-3
View File
@@ -1,3 +0,0 @@
# Tests the object_read slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
@@ -1,3 +0,0 @@
# Tests the object_write slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-3
View File
@@ -1,3 +0,0 @@
# Tests the t211 slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-3
View File
@@ -1,3 +0,0 @@
# Tests the t212 slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-167
View File
@@ -1,167 +0,0 @@
# RustFS Docker Infrastructure
This directory contains the complete Docker infrastructure for building, deploying, and monitoring RustFS. It provides ready-to-use configurations for development, testing, and production-grade observability.
## 📂 Directory Structure
| Directory | Description | Status |
| :--- | :--- | :--- |
| **[`observability/`](observability/README.md)** | **[RECOMMENDED]** Full-stack observability (Prometheus, Grafana, Tempo, Loki). | ✅ Production-Ready |
| **[`compose/`](compose/README.md)** | Specialized setups (e.g., 4-node distributed cluster testing). | ⚠️ Testing Only |
| **[`mqtt/`](mqtt/README.md)** | EMQX Broker configuration for MQTT integration testing. | 🧪 Development |
| **[`openobserve-otel/`](openobserve-otel/README.md)** | Alternative lightweight observability stack using OpenObserve. | 🔄 Alternative |
---
## 📄 Root Directory Files
The following files in the project root are essential for Docker operations:
### Build Scripts & Dockerfiles
| File | Description | Usage |
| :--- | :--- | :--- |
| **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
### Docker Compose Configurations
| File | Description | Usage |
| :--- | :--- | :--- |
| **`docker-compose.yml`** | **Main Development Setup**<br>Comprehensive setup with profiles for development, observability, and proxying. | `docker compose up -d`<br>`docker compose --profile observability up -d` |
| **`docker-compose-simple.yml`** | **Quick Start Setup**<br>Minimal configuration running a single RustFS instance with 4 volumes. Perfect for first-time users. | `docker compose -f docker-compose-simple.yml up -d` |
---
## 🌟 Observability Stack (Recommended)
Located in: [`.docker/observability/`](observability/README.md)
We provide a comprehensive, industry-standard observability stack designed for deep insights into RustFS performance. This is the recommended setup for both development and production monitoring.
### Components
- **Metrics**: Prometheus (Collection) + Grafana (Visualization)
- **Traces**: Tempo (Storage) + Jaeger (UI)
- **Logs**: Loki
- **Ingestion**: OpenTelemetry Collector
### Key Features
- **Full Persistence**: All metrics, logs, and traces are saved to Docker volumes, ensuring no data loss on restarts.
- **Correlation**: Seamlessly jump between Logs, Traces, and Metrics in Grafana.
- **High Performance**: Optimized configurations for batching, compression, and memory management.
### Quick Start
```bash
cd .docker/observability
docker compose up -d
```
---
## 🧪 Specialized Environments
Located in: [`.docker/compose/`](compose/README.md)
These configurations are tailored for specific testing scenarios that require complex topologies.
### Distributed Cluster (4-Nodes)
Simulates a real-world distributed environment with 4 RustFS nodes running locally.
```bash
docker compose -f .docker/compose/docker-compose.cluster.yaml up -d
```
### Integrated Observability Test
A self-contained environment running 4 RustFS nodes alongside the full observability stack. Useful for end-to-end testing of telemetry.
```bash
docker compose -f .docker/compose/docker-compose.observability.yaml up -d
```
---
## 📡 MQTT Integration
Located in: [`.docker/mqtt/`](mqtt/README.md)
Provides an EMQX broker for testing RustFS MQTT features.
### Quick Start
```bash
cd .docker/mqtt
docker compose up -d
```
- **Dashboard**: [http://localhost:18083](http://localhost:18083) (Default: `admin` / `public`)
- **MQTT Port**: `1883`
---
## 👁️ Alternative: OpenObserve
Located in: [`.docker/openobserve-otel/`](openobserve-otel/README.md)
For users preferring a lightweight, all-in-one solution, we support OpenObserve. It combines logs, metrics, and traces into a single binary and UI.
### Quick Start
```bash
cd .docker/openobserve-otel
docker compose up -d
```
---
## Docker Compose environment values
Use map syntax when quoting environment values in YAML:
```yaml
services:
rustfs:
image: rustfs/rustfs:latest
ports:
- "9000:9000"
- "9001:9001"
volumes:
- ./data:/data
- ./logs:/var/log/rustfs
environment:
RUSTFS_ADDRESS: ":9000"
RUSTFS_CONSOLE_ADDRESS: ":9001"
RUSTFS_CONSOLE_ENABLE: "true"
RUSTFS_OBS_LOGGER_LEVEL: error
RUSTFS_OBS_LOG_DIRECTORY: "/var/log/rustfs/"
RUSTFS_ACCESS_KEY: "${RUSTFS_ACCESS_KEY:?Set RUSTFS_ACCESS_KEY}"
RUSTFS_SECRET_KEY: "${RUSTFS_SECRET_KEY:?Set RUSTFS_SECRET_KEY}"
```
Create the host directories and grant the container user write access before starting this example; official images run as UID/GID `10001:10001`. Set both credential variables to your own values, then run `docker compose config` to inspect the resolved configuration locally before `docker compose up -d`. The resolved output includes credentials, so do not share it publicly.
With list syntax, use `- RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/` or quote the whole item as `- "RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/"`. Writing `- RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"` passes the inner quotes literally, making this a relative path beginning with `"`, which can fail with `mkdir: cannot create directory '"': Permission denied`. This also applies to addresses and credentials: fix every affected list item, rather than only the log directory. The entrypoint preserves configured values, including literal quotes in valid paths and secrets.
---
## Local storage pool expansion
To append a second four-drive pool, set `RUSTFS_VOLUMES: "/data/rustfs0/{0...3} /data/rustfs1/{0...3}"` and keep the original pool's drives, order, and erasure set width. The container prepares each drive directory and passes one argument per pool to RustFS.
If an older image already formatted both ranges together as a single eight-drive pool, list the eight individual paths in their original order to keep that layout when upgrading.
## 🔧 Common Operations
### Cleaning Up
To stop all containers and remove volumes (**WARNING**: deletes all persisted data):
```bash
docker compose down -v
```
### Viewing Logs
To follow logs for a specific service:
```bash
docker compose logs -f [service_name]
```
### Checking Status
To see the status of all running containers:
```bash
docker compose ps
```
-102
View File
@@ -1,102 +0,0 @@
# Specialized Docker Compose Configurations
This directory contains specialized Docker Compose configurations for specific testing scenarios.
## ⚠️ Important Note
**For Observability:**
We **strongly recommend** using the new, fully integrated observability stack located in `../observability/`. It provides a production-ready setup with Prometheus, Grafana, Tempo, Loki, and OpenTelemetry Collector, all with persistent storage and optimized configurations.
The `docker-compose.observability.yaml` in this directory is kept for legacy reference or specific minimal testing needs but is **not** the primary recommended setup.
## 📁 Configuration Files
### Cluster Testing
- **`docker-compose.cluster.yaml`**
- **Purpose**: Simulates a 4-node RustFS distributed cluster.
- **Use Case**: Testing distributed storage logic, consensus, and failover.
- **Nodes**: 4 RustFS instances.
- **Storage**: Uses local HTTP endpoints.
### Legacy / Minimal Observability
- **`docker-compose.observability.yaml`**
- **Purpose**: A minimal observability setup.
- **Status**: **Deprecated**. Please use `../observability/docker-compose.yml` instead.
## 🚀 Usage Examples
### Cluster Testing
To start a 4-node cluster for distributed testing:
```bash
# From project root
docker compose -f .docker/compose/docker-compose.cluster.yaml up -d
```
### Script-Based 4-Node Validation (Recommended)
Use the local validation script when you need local-source image build, failover checks,
and benchmark workflow in one command:
```bash
# Default mode: WAIT_PROBE_MODE=service
# This avoids false negatives where /health/ready remains 503 locally
# while the service path is already available.
./scripts/run_four_node_cluster_failover_bench.sh
```
Strict mode is available when you explicitly want `/health/ready == 200` as the gate:
```bash
WAIT_PROBE_MODE=ready ./scripts/run_four_node_cluster_failover_bench.sh
```
### Profiling + Trace Validation
The profiling-focused 4-node compose keeps profiling enabled and points RustFS
to an OTLP/HTTP collector endpoint:
```bash
docker compose -f .docker/compose/docker-compose.cluster.local-build.profiling-amd64.yml up -d
```
Important behavior notes:
- `RUSTFS_OBS_ENDPOINT` is the OTLP/HTTP base URL. RustFS automatically sends
traces to `/v1/traces`, metrics to `/v1/metrics`, and logs to `/v1/logs`.
- Startup usually produces logs and metrics first. That does not guarantee
visible traces yet.
- Trace data becomes obvious only after real HTTP/S3/gRPC requests hit RustFS.
- `RUSTFS_OBS_LOGGER_LEVEL=info` keeps the top-level request span but filters
many nested `debug` spans. If Tempo/Jaeger looks sparse, retry with
`RUSTFS_OBS_LOGGER_LEVEL=debug` before suspecting the collector.
Minimal trace verification flow:
```bash
# 1. Start the profiling compose with richer span visibility.
RUSTFS_OBS_LOGGER_LEVEL=debug \
docker compose -f .docker/compose/docker-compose.cluster.local-build.profiling-amd64.yml up -d
# 2. Generate real request traffic after startup.
curl -I http://127.0.0.1:9000/health
curl -I http://127.0.0.1:9000/health/ready
# 3. Then inspect Tempo or Jaeger.
# Grafana: http://localhost:3000
# Jaeger: http://localhost:16686
```
If logs and metrics are present but traces are sparse, the most common cause is
"no real request traffic yet" or "`info` level filtered nested spans", not an
OTLP routing failure.
### (Deprecated) Minimal Observability
```bash
# From project root
docker compose -f .docker/compose/docker-compose.observability.yaml up -d
```
@@ -1,240 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Profiling-first 4-node local-build compose.
#
# Goals:
# - force linux/amd64 runtime/build on Apple Silicon hosts;
# - enable RustFS built-in CPU profiling;
# - keep all tuning knobs host-overridable via env.
#
# Observability notes:
# - `RUSTFS_OBS_ENDPOINT` is the OTLP/HTTP base URL. RustFS appends
# `/v1/traces`, `/v1/metrics`, and `/v1/logs` automatically.
# - Logs and metrics usually appear during startup. Traces mainly appear after
# real HTTP/S3/gRPC requests create spans.
# - `RUSTFS_OBS_LOGGER_LEVEL=info` keeps the top-level request trace span but
# filters many `debug`-level nested spans. Use `debug` when validating trace
# richness rather than collector reachability.
services:
node1:
platform: ${RUSTFS_DOCKER_PLATFORM:-linux/amd64}
image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
build:
context: ../..
dockerfile: Dockerfile.source
hostname: node1
environment:
- RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
# `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
# should show richer nested spans during request-path verification.
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
- RUSTFS_OBS_PROFILING_ENDPOINT=${RUSTFS_OBS_PROFILING_ENDPOINT:-http://host.docker.internal:4040}
- RUSTFS_OBS_PROFILING_EXPORT_ENABLED=${RUSTFS_OBS_PROFILING_EXPORT_ENABLED:-true}
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
- RUSTFS_ENABLE_PROFILING=${RUSTFS_ENABLE_PROFILING:-true}
- RUSTFS_PROF_CPU_MODE=${RUSTFS_PROF_CPU_MODE:-continuous}
- RUSTFS_PROF_CPU_FREQ=${RUSTFS_PROF_CPU_FREQ:-99}
- RUSTFS_PROF_OUTPUT_DIR=${RUSTFS_PROF_OUTPUT_DIR:-/tmp/rustfs-profiles}
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=${RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS:-48}
- RUSTFS_OBJECT_IO_BUFFER_SIZE=${RUSTFS_OBJECT_IO_BUFFER_SIZE:-262144}
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=${RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD:-6}
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=${RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD:-12}
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=${RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY:-8}
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=${RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE:-8388608}
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=${RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES:-25165824}
- RUSTFS_RUNTIME_WORKER_THREADS=${RUSTFS_RUNTIME_WORKER_THREADS:-12}
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=${RUSTFS_RUNTIME_MAX_BLOCKING_THREADS:-512}
- RUSTFS_ALLOCATOR_RECLAIM_ENABLED=${RUSTFS_ALLOCATOR_RECLAIM_ENABLED:-false}
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- node1_data_0:/data/rustfs0
- node1_data_1:/data/rustfs1
- node1_data_2:/data/rustfs2
- node1_data_3:/data/rustfs3
ports:
- "9000:9000"
networks:
- rustfs-cluster-net
node2:
platform: ${RUSTFS_DOCKER_PLATFORM:-linux/amd64}
image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
build:
context: ../..
dockerfile: Dockerfile.source
hostname: node2
environment:
- RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
# `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
# should show richer nested spans during request-path verification.
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
- RUSTFS_OBS_PROFILING_ENDPOINT=${RUSTFS_OBS_PROFILING_ENDPOINT:-http://host.docker.internal:4040}
- RUSTFS_OBS_PROFILING_EXPORT_ENABLED=${RUSTFS_OBS_PROFILING_EXPORT_ENABLED:-true}
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
- RUSTFS_ENABLE_PROFILING=${RUSTFS_ENABLE_PROFILING:-true}
- RUSTFS_PROF_CPU_MODE=${RUSTFS_PROF_CPU_MODE:-continuous}
- RUSTFS_PROF_CPU_FREQ=${RUSTFS_PROF_CPU_FREQ:-99}
- RUSTFS_PROF_OUTPUT_DIR=${RUSTFS_PROF_OUTPUT_DIR:-/tmp/rustfs-profiles}
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=${RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS:-48}
- RUSTFS_OBJECT_IO_BUFFER_SIZE=${RUSTFS_OBJECT_IO_BUFFER_SIZE:-262144}
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=${RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD:-6}
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=${RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD:-12}
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=${RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY:-8}
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=${RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE:-8388608}
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=${RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES:-25165824}
- RUSTFS_RUNTIME_WORKER_THREADS=${RUSTFS_RUNTIME_WORKER_THREADS:-12}
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=${RUSTFS_RUNTIME_MAX_BLOCKING_THREADS:-512}
- RUSTFS_ALLOCATOR_RECLAIM_ENABLED=${RUSTFS_ALLOCATOR_RECLAIM_ENABLED:-false}
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- node2_data_0:/data/rustfs0
- node2_data_1:/data/rustfs1
- node2_data_2:/data/rustfs2
- node2_data_3:/data/rustfs3
ports:
- "9001:9000"
networks:
- rustfs-cluster-net
node3:
platform: ${RUSTFS_DOCKER_PLATFORM:-linux/amd64}
image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
build:
context: ../..
dockerfile: Dockerfile.source
hostname: node3
environment:
- RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
# `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
# should show richer nested spans during request-path verification.
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
- RUSTFS_OBS_PROFILING_ENDPOINT=${RUSTFS_OBS_PROFILING_ENDPOINT:-http://host.docker.internal:4040}
- RUSTFS_OBS_PROFILING_EXPORT_ENABLED=${RUSTFS_OBS_PROFILING_EXPORT_ENABLED:-true}
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
- RUSTFS_ENABLE_PROFILING=${RUSTFS_ENABLE_PROFILING:-true}
- RUSTFS_PROF_CPU_MODE=${RUSTFS_PROF_CPU_MODE:-continuous}
- RUSTFS_PROF_CPU_FREQ=${RUSTFS_PROF_CPU_FREQ:-99}
- RUSTFS_PROF_OUTPUT_DIR=${RUSTFS_PROF_OUTPUT_DIR:-/tmp/rustfs-profiles}
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=${RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS:-48}
- RUSTFS_OBJECT_IO_BUFFER_SIZE=${RUSTFS_OBJECT_IO_BUFFER_SIZE:-262144}
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=${RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD:-6}
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=${RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD:-12}
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=${RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY:-8}
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=${RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE:-8388608}
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=${RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES:-25165824}
- RUSTFS_RUNTIME_WORKER_THREADS=${RUSTFS_RUNTIME_WORKER_THREADS:-12}
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=${RUSTFS_RUNTIME_MAX_BLOCKING_THREADS:-512}
- RUSTFS_ALLOCATOR_RECLAIM_ENABLED=${RUSTFS_ALLOCATOR_RECLAIM_ENABLED:-false}
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- node3_data_0:/data/rustfs0
- node3_data_1:/data/rustfs1
- node3_data_2:/data/rustfs2
- node3_data_3:/data/rustfs3
ports:
- "9002:9000"
networks:
- rustfs-cluster-net
node4:
platform: ${RUSTFS_DOCKER_PLATFORM:-linux/amd64}
image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
build:
context: ../..
dockerfile: Dockerfile.source
hostname: node4
environment:
- RUSTFS_VOLUMES=http://node{1...4}:9000/data/rustfs{0...3}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfs-cluster-admin}
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfs-cluster-secret}
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
# `info` is enough for startup logs/metrics. Use `debug` if Tempo/Jaeger
# should show richer nested spans during request-path verification.
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
- RUSTFS_OBS_PROFILING_ENDPOINT=${RUSTFS_OBS_PROFILING_ENDPOINT:-http://host.docker.internal:4040}
- RUSTFS_OBS_PROFILING_EXPORT_ENABLED=${RUSTFS_OBS_PROFILING_EXPORT_ENABLED:-true}
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
- RUSTFS_ENABLE_PROFILING=${RUSTFS_ENABLE_PROFILING:-true}
- RUSTFS_PROF_CPU_MODE=${RUSTFS_PROF_CPU_MODE:-continuous}
- RUSTFS_PROF_CPU_FREQ=${RUSTFS_PROF_CPU_FREQ:-99}
- RUSTFS_PROF_OUTPUT_DIR=${RUSTFS_PROF_OUTPUT_DIR:-/tmp/rustfs-profiles}
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=${RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS:-48}
- RUSTFS_OBJECT_IO_BUFFER_SIZE=${RUSTFS_OBJECT_IO_BUFFER_SIZE:-262144}
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=${RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD:-6}
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=${RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD:-12}
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=${RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY:-8}
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=${RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE:-8388608}
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=${RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES:-25165824}
- RUSTFS_RUNTIME_WORKER_THREADS=${RUSTFS_RUNTIME_WORKER_THREADS:-12}
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=${RUSTFS_RUNTIME_MAX_BLOCKING_THREADS:-512}
- RUSTFS_ALLOCATOR_RECLAIM_ENABLED=${RUSTFS_ALLOCATOR_RECLAIM_ENABLED:-false}
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- node4_data_0:/data/rustfs0
- node4_data_1:/data/rustfs1
- node4_data_2:/data/rustfs2
- node4_data_3:/data/rustfs3
ports:
- "9003:9000"
networks:
- rustfs-cluster-net
volumes:
node1_data_0:
node1_data_1:
node1_data_2:
node1_data_3:
node2_data_0:
node2_data_1:
node2_data_2:
node2_data_3:
node3_data_0:
node3_data_1:
node3_data_2:
node3_data_3:
node4_data_0:
node4_data_1:
node4_data_2:
node4_data_3:
networks:
rustfs-cluster-net:
driver: bridge
@@ -1,248 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
services:
node1:
image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
build:
context: ../..
dockerfile: Dockerfile.source
hostname: node1
environment:
- RUSTFS_VOLUMES=${RUSTFS_VOLUMES:-http://node{1...4}:9000/data/rustfs{0...3}}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
- RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
- RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
- RUSTFS_ISSUE3031_DIAG_ENABLE=${RUSTFS_ISSUE3031_DIAG_ENABLE:-false}
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
- RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE=${RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE:-1048576}
- RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE=${RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE:-2097152}
- RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE=${RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE:-104857600}
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
- RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD=${RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD:-3}
- RUSTFS_INTERNODE_IDEMPOTENT_READ_RETRIES=${RUSTFS_INTERNODE_IDEMPOTENT_READ_RETRIES:-1}
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- node1_data_0:/data/rustfs0
- node1_data_1:/data/rustfs1
- node1_data_2:/data/rustfs2
- node1_data_3:/data/rustfs3
ports:
- "${RUSTFS_NODE1_PORT:-9000}:9000"
networks:
- rustfs-cluster-net
node2:
image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
build:
context: ../..
dockerfile: Dockerfile.source
hostname: node2
environment:
- RUSTFS_VOLUMES=${RUSTFS_VOLUMES:-http://node{1...4}:9000/data/rustfs{0...3}}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
- RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
- RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
- RUSTFS_ISSUE3031_DIAG_ENABLE=${RUSTFS_ISSUE3031_DIAG_ENABLE:-false}
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
- RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE=${RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE:-1048576}
- RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE=${RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE:-2097152}
- RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE=${RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE:-104857600}
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
- RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD=${RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD:-3}
- RUSTFS_INTERNODE_IDEMPOTENT_READ_RETRIES=${RUSTFS_INTERNODE_IDEMPOTENT_READ_RETRIES:-1}
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- node2_data_0:/data/rustfs0
- node2_data_1:/data/rustfs1
- node2_data_2:/data/rustfs2
- node2_data_3:/data/rustfs3
ports:
- "${RUSTFS_NODE2_PORT:-9001}:9000"
networks:
- rustfs-cluster-net
node3:
image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
build:
context: ../..
dockerfile: Dockerfile.source
hostname: node3
environment:
- RUSTFS_VOLUMES=${RUSTFS_VOLUMES:-http://node{1...4}:9000/data/rustfs{0...3}}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
- RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
- RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
- RUSTFS_ISSUE3031_DIAG_ENABLE=${RUSTFS_ISSUE3031_DIAG_ENABLE:-false}
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
- RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE=${RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE:-1048576}
- RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE=${RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE:-2097152}
- RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE=${RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE:-104857600}
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
- RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD=${RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD:-3}
- RUSTFS_INTERNODE_IDEMPOTENT_READ_RETRIES=${RUSTFS_INTERNODE_IDEMPOTENT_READ_RETRIES:-1}
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- node3_data_0:/data/rustfs0
- node3_data_1:/data/rustfs1
- node3_data_2:/data/rustfs2
- node3_data_3:/data/rustfs3
ports:
- "${RUSTFS_NODE3_PORT:-9002}:9000"
networks:
- rustfs-cluster-net
node4:
image: ${RUSTFS_IMAGE:-rustfs/rustfs:local-4node}
build:
context: ../..
dockerfile: Dockerfile.source
hostname: node4
environment:
- RUSTFS_VOLUMES=${RUSTFS_VOLUMES:-http://node{1...4}:9000/data/rustfs{0...3}}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
- RUSTFS_OBS_ENDPOINT=${RUSTFS_OBS_ENDPOINT:-http://host.docker.internal:4318}
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
- RUSTFS_OBS_LOGGER_LEVEL=${RUSTFS_OBS_LOGGER_LEVEL:-info}
- RUSTFS_OBS_USE_STDOUT=${RUSTFS_OBS_USE_STDOUT:-false}
- RUSTFS_OBS_LOG_STDOUT_ENABLED=${RUSTFS_OBS_LOG_STDOUT_ENABLED:-false}
- RUSTFS_ISSUE3031_DIAG_ENABLE=${RUSTFS_ISSUE3031_DIAG_ENABLE:-false}
- RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT:-5}
- RUSTFS_LOCK_ACQUIRE_TIMEOUT=${RUSTFS_LOCK_ACQUIRE_TIMEOUT:-5}
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=${RUSTFS_UNSAFE_BYPASS_DISK_CHECK:-true}
- RUSTFS_DURABILITY_MODE=${RUSTFS_DURABILITY_MODE:-strict}
- RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE=${RUSTFS_OBJECT_DIRECT_IO_WRITE_ENABLE:-false}
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_ENABLE:-false}
- RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS=${RUSTFS_EXPERIMENTAL_FILE_FDATASYNC_GROUP_COMMIT_WAIT_MICROS:-0}
# Internode gRPC optimization knobs (grpc-optimization P0-P3). Defaults match the binary
# defaults, so leaving these unset is a no-op; the A/B driver exports them to toggle a stage.
- RUSTFS_INTERNODE_RPC_TCP_NODELAY=${RUSTFS_INTERNODE_RPC_TCP_NODELAY:-true}
- RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE=${RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE:-1048576}
- RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE=${RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE:-2097152}
- RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE=${RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE:-104857600}
- RUSTFS_INTERNODE_CHANNEL_ISOLATION=${RUSTFS_INTERNODE_CHANNEL_ISOLATION:-false}
- RUSTFS_INTERNODE_BULK_CHANNELS=${RUSTFS_INTERNODE_BULK_CHANNELS:-2}
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY:-false}
- RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=${RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED:-false}
- RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED=${RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED:-false}
- RUSTFS_INTERNODE_PREWARM=${RUSTFS_INTERNODE_PREWARM:-false}
- RUSTFS_INTERNODE_OFFLINE_BYPASS=${RUSTFS_INTERNODE_OFFLINE_BYPASS:-false}
- RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS=${RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS:-5}
- RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD=${RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD:-3}
- RUSTFS_INTERNODE_IDEMPOTENT_READ_RETRIES=${RUSTFS_INTERNODE_IDEMPOTENT_READ_RETRIES:-1}
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- node4_data_0:/data/rustfs0
- node4_data_1:/data/rustfs1
- node4_data_2:/data/rustfs2
- node4_data_3:/data/rustfs3
ports:
- "${RUSTFS_NODE4_PORT:-9003}:9000"
networks:
- rustfs-cluster-net
volumes:
node1_data_0:
node1_data_1:
node1_data_2:
node1_data_3:
node2_data_0:
node2_data_1:
node2_data_2:
node2_data_3:
node3_data_0:
node3_data_1:
node3_data_2:
node3_data_3:
node4_data_0:
node4_data_1:
node4_data_2:
node4_data_3:
networks:
rustfs-cluster-net:
driver: bridge
@@ -1,56 +0,0 @@
services:
node1:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=56
- RUSTFS_OBJECT_IO_BUFFER_SIZE=131072
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=10
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=20
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=10
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824
- RUSTFS_RUNTIME_WORKER_THREADS=16
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=768
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=420
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=50
node2:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=56
- RUSTFS_OBJECT_IO_BUFFER_SIZE=131072
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=10
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=20
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=10
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824
- RUSTFS_RUNTIME_WORKER_THREADS=16
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=768
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=420
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=50
node3:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=56
- RUSTFS_OBJECT_IO_BUFFER_SIZE=131072
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=10
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=20
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=10
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824
- RUSTFS_RUNTIME_WORKER_THREADS=16
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=768
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=420
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=50
node4:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=56
- RUSTFS_OBJECT_IO_BUFFER_SIZE=131072
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=10
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=20
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=10
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824
- RUSTFS_RUNTIME_WORKER_THREADS=16
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=768
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=420
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=50
@@ -1,56 +0,0 @@
services:
node1:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=96
- RUSTFS_OBJECT_IO_BUFFER_SIZE=524288
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=16
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=32
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=24
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=33554432
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=50331648
- RUSTFS_RUNTIME_WORKER_THREADS=24
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1536
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=600
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=80
node2:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=96
- RUSTFS_OBJECT_IO_BUFFER_SIZE=524288
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=16
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=32
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=24
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=33554432
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=50331648
- RUSTFS_RUNTIME_WORKER_THREADS=24
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1536
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=600
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=80
node3:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=96
- RUSTFS_OBJECT_IO_BUFFER_SIZE=524288
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=16
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=32
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=24
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=33554432
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=50331648
- RUSTFS_RUNTIME_WORKER_THREADS=24
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1536
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=600
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=80
node4:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=96
- RUSTFS_OBJECT_IO_BUFFER_SIZE=524288
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=16
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=32
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=24
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=33554432
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=50331648
- RUSTFS_RUNTIME_WORKER_THREADS=24
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1536
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=600
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=80
@@ -1,56 +0,0 @@
services:
node1:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=64
- RUSTFS_OBJECT_IO_BUFFER_SIZE=262144
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=12
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=24
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=16
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=16777216
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=33554432
- RUSTFS_RUNTIME_WORKER_THREADS=20
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1024
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=60
node2:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=64
- RUSTFS_OBJECT_IO_BUFFER_SIZE=262144
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=12
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=24
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=16
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=16777216
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=33554432
- RUSTFS_RUNTIME_WORKER_THREADS=20
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1024
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=60
node3:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=64
- RUSTFS_OBJECT_IO_BUFFER_SIZE=262144
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=12
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=24
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=16
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=16777216
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=33554432
- RUSTFS_RUNTIME_WORKER_THREADS=20
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1024
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=60
node4:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=64
- RUSTFS_OBJECT_IO_BUFFER_SIZE=262144
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=12
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=24
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=16
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=16777216
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=33554432
- RUSTFS_RUNTIME_WORKER_THREADS=20
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1024
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=60
@@ -1,56 +0,0 @@
services:
node1:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=48
- RUSTFS_OBJECT_IO_BUFFER_SIZE=262144
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=8
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=16
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=12
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824
- RUSTFS_RUNTIME_WORKER_THREADS=12
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=512
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=40
node2:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=48
- RUSTFS_OBJECT_IO_BUFFER_SIZE=262144
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=8
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=16
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=12
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824
- RUSTFS_RUNTIME_WORKER_THREADS=12
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=512
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=40
node3:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=48
- RUSTFS_OBJECT_IO_BUFFER_SIZE=262144
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=8
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=16
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=12
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824
- RUSTFS_RUNTIME_WORKER_THREADS=12
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=512
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=40
node4:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=48
- RUSTFS_OBJECT_IO_BUFFER_SIZE=262144
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=8
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=16
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=12
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=25165824
- RUSTFS_RUNTIME_WORKER_THREADS=12
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=512
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=40
@@ -1,56 +0,0 @@
services:
node1:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=32
- RUSTFS_OBJECT_IO_BUFFER_SIZE=262144
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=6
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=12
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=8
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=16777216
- RUSTFS_RUNTIME_WORKER_THREADS=6
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=256
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=30
node2:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=32
- RUSTFS_OBJECT_IO_BUFFER_SIZE=262144
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=6
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=12
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=8
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=16777216
- RUSTFS_RUNTIME_WORKER_THREADS=6
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=256
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=30
node3:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=32
- RUSTFS_OBJECT_IO_BUFFER_SIZE=262144
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=6
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=12
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=8
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=16777216
- RUSTFS_RUNTIME_WORKER_THREADS=6
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=256
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=30
node4:
environment:
- RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=32
- RUSTFS_OBJECT_IO_BUFFER_SIZE=262144
- RUSTFS_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD=6
- RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=12
- RUSTFS_OBJECT_IO_RANDOM_READAHEAD_DISABLE_CONCURRENCY=8
- RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608
- RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES=16777216
- RUSTFS_RUNTIME_WORKER_THREADS=6
- RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=256
- RUSTFS_CAPACITY_SCHEDULED_INTERVAL=300
- RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD=30
@@ -1,82 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
services:
node0:
image: rustfs/rustfs:latest # Replace with your image name and label
container_name: node0
hostname: node0
environment:
- RUSTFS_VOLUMES=http://node{0...3}:9000/data/rustfs{0...3}
- RUSTFS_ADDRESS=0.0.0.0:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
platform: linux/amd64
ports:
- "9000:9000" # Map port 9001 of the host to port 9000 of the container
volumes:
- ../../target/x86_64-unknown-linux-gnu/release/rustfs:/app/rustfs
command: "/app/rustfs"
node1:
image: rustfs/rustfs:latest
container_name: node1
hostname: node1
environment:
- RUSTFS_VOLUMES=http://node{0...3}:9000/data/rustfs{0...3}
- RUSTFS_ADDRESS=0.0.0.0:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
platform: linux/amd64
ports:
- "9001:9000" # Map port 9002 of the host to port 9000 of the container
volumes:
- ../../target/x86_64-unknown-linux-gnu/release/rustfs:/app/rustfs
command: "/app/rustfs"
node2:
image: rustfs/rustfs:latest
container_name: node2
hostname: node2
environment:
- RUSTFS_VOLUMES=http://node{0...3}:9000/data/rustfs{0...3}
- RUSTFS_ADDRESS=0.0.0.0:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
platform: linux/amd64
ports:
- "9002:9000" # Map port 9003 of the host to port 9000 of the container
volumes:
- ../../target/x86_64-unknown-linux-gnu/release/rustfs:/app/rustfs
command: "/app/rustfs"
node3:
image: rustfs/rustfs:latest
container_name: node3
hostname: node3
environment:
- RUSTFS_VOLUMES=http://node{0...3}:9000/data/rustfs{0...3}
- RUSTFS_ADDRESS=0.0.0.0:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY:-rustfsadmin-local}
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY:-rustfssecret-local}
platform: linux/amd64
ports:
- "9003:9000" # Map port 9004 of the host to port 9000 of the container
volumes:
- ../../target/x86_64-unknown-linux-gnu/release/rustfs:/app/rustfs
command: "/app/rustfs"
@@ -1,286 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
services:
# --- Observability Stack ---
tempo-init:
image: busybox:latest
command: [ "sh", "-c", "chown -R 10001:10001 /var/tempo" ]
volumes:
- tempo-data:/var/tempo
user: root
networks:
- rustfs-network
restart: "no"
tempo:
image: grafana/tempo:2.10.5
user: "10001"
command: [ "-config.file=/etc/tempo.yaml" ]
volumes:
- ../../.docker/observability/tempo.yaml:/etc/tempo.yaml:ro
- tempo-data:/var/tempo
ports:
- "3200:3200" # tempo
- "4317" # otlp grpc
- "4318" # otlp http
- "7946" # memberlist
restart: unless-stopped
networks:
- rustfs-network
depends_on:
tempo-init:
condition: service_completed_successfully
healthcheck:
test: [ "CMD", "/tempo", "-version" ]
interval: 10s
timeout: 5s
retries: 3
start_period: 15s
otel-collector:
image: otel/opentelemetry-collector-contrib:latest
environment:
- TZ=Asia/Shanghai
volumes:
- ../../.docker/observability/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
ports:
- "1888:1888" # pprof
- "8888:8888" # Prometheus metrics for Collector
- "8889:8889" # Prometheus metrics for application indicators
- "13133:13133" # health check
- "4317:4317" # OTLP gRPC
- "4318:4318" # OTLP HTTP
- "55679:55679" # zpages
networks:
- rustfs-network
depends_on:
- tempo
- jaeger
- prometheus
- loki
restart: unless-stopped
healthcheck:
test: [ "CMD", "/otelcol-contrib", "--version" ]
interval: 10s
timeout: 5s
retries: 3
jaeger:
image: jaegertracing/jaeger:latest
environment:
- TZ=Asia/Shanghai
- SPAN_STORAGE_TYPE=badger
- BADGER_EPHEMERAL=false
- BADGER_DIRECTORY_VALUE=/badger/data
- BADGER_DIRECTORY_KEY=/badger/key
- COLLECTOR_OTLP_ENABLED=true
volumes:
- ../../.docker/observability/jaeger.yaml:/etc/jaeger/config.yml:ro
- jaeger-data:/badger
ports:
- "16686:16686" # Web UI
- "14269:14269" # Admin/Metrics
- "4317" # otlp grpc
- "4318" # otlp http
command: [ "--config", "/etc/jaeger/config.yml" ]
networks:
- rustfs-network
restart: unless-stopped
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:14269" ]
interval: 10s
timeout: 5s
retries: 3
start_period: 15s
prometheus:
image: prom/prometheus:latest
environment:
- TZ=Asia/Shanghai
volumes:
- ../../.docker/observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ../../.docker/observability/prometheus-rules:/etc/prometheus/rules:ro
- prometheus-data:/prometheus
ports:
- "9090:9090"
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--web.enable-otlp-receiver'
- '--web.enable-remote-write-receiver'
- '--enable-feature=promql-experimental-functions'
- '--storage.tsdb.path=/prometheus'
- '--storage.tsdb.retention.time=30d'
networks:
- rustfs-network
restart: unless-stopped
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy" ]
interval: 10s
timeout: 5s
retries: 3
loki:
image: grafana/loki:latest
environment:
- TZ=Asia/Shanghai
volumes:
- ../../.docker/observability/loki.yaml:/etc/loki/loki.yaml:ro
- loki-data:/loki
ports:
- "3100:3100"
command: -config.file=/etc/loki/loki.yaml
networks:
- rustfs-network
restart: unless-stopped
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ]
interval: 15s
timeout: 10s
retries: 5
start_period: 60s
pyroscope:
image: grafana/pyroscope:latest
ports:
- "4040:4040"
command:
- -self-profiling.disable-push=true
networks:
- rustfs-network
restart: unless-stopped
grafana:
image: grafana/grafana:latest
ports:
- "3000:3000" # Web UI
environment:
- GF_SECURITY_ADMIN_PASSWORD=admin
- GF_SECURITY_ADMIN_USER=admin
- TZ=Asia/Shanghai
- GF_INSTALL_PLUGINS=grafana-pyroscope-datasource
- GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/home.json
networks:
- rustfs-network
volumes:
- ../../.docker/observability/grafana/provisioning:/etc/grafana/provisioning:ro
- ../../.docker/observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana-data:/var/lib/grafana
depends_on:
- prometheus
- tempo
- loki
restart: unless-stopped
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health" ]
interval: 10s
timeout: 5s
retries: 3
# --- RustFS Cluster ---
node1:
build:
context: ../..
dockerfile: Dockerfile.source
container_name: node1
environment:
- RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
- RUSTFS_OBS_LOGGER_LEVEL=debug
platform: linux/amd64
ports:
- "9001:9000"
networks:
- rustfs-network
depends_on:
- otel-collector
node2:
build:
context: ../..
dockerfile: Dockerfile.source
container_name: node2
environment:
- RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
- RUSTFS_OBS_LOGGER_LEVEL=debug
platform: linux/amd64
ports:
- "9002:9000"
networks:
- rustfs-network
depends_on:
- otel-collector
node3:
build:
context: ../..
dockerfile: Dockerfile.source
container_name: node3
environment:
- RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
- RUSTFS_OBS_LOGGER_LEVEL=debug
platform: linux/amd64
ports:
- "9003:9000"
networks:
- rustfs-network
depends_on:
- otel-collector
node4:
build:
context: ../..
dockerfile: Dockerfile.source
container_name: node4
environment:
- RUSTFS_VOLUMES=http://node{1...4}:9000/root/data/target/volume/test{1...4}
- RUSTFS_ADDRESS=:9000
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
- RUSTFS_OBS_LOGGER_LEVEL=debug
platform: linux/amd64
ports:
- "9004:9000"
networks:
- rustfs-network
depends_on:
- otel-collector
volumes:
prometheus-data:
tempo-data:
loki-data:
jaeger-data:
grafana-data:
networks:
rustfs-network:
driver: bridge
name: "network_rustfs_config"
driver_opts:
com.docker.network.enable_ipv6: "true"
-58
View File
@@ -1,58 +0,0 @@
# syntax=docker/dockerfile:1.6
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Server-only Linux validation image. The build context supplies a Git bundle.
FROM rust:1.100.0-trixie AS build-tools
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates clang lld libssl-dev pkg-config protobuf-compiler python3 python3-boto3 \
&& rm -rf /var/lib/apt/lists/*
ENV RUSTUP_TOOLCHAIN=1.100.0 \
CARGO_BUILD_JOBS=4 \
CARGO_NET_GIT_FETCH_WITH_CLI=true \
CARGO_TARGET_DIR=/work/rustfs/target \
RUSTFLAGS="-C link-arg=-fuse-ld=lld"
FROM build-tools AS build
ARG SOURCE_REVISION
COPY repository.bundle /tmp/repository.bundle
COPY inspect_geometry.rs /tmp/inspect_geometry.rs
RUN git clone /tmp/repository.bundle /work/rustfs \
&& git -C /work/rustfs checkout --detach "$SOURCE_REVISION" \
&& test "$(git -C /work/rustfs rev-parse HEAD)" = "$SOURCE_REVISION"
WORKDIR /work/rustfs
RUN --mount=type=cache,id=rustfs-inline-registry,target=/usr/local/cargo/registry \
--mount=type=cache,id=rustfs-inline-git,target=/usr/local/cargo/git \
--mount=type=cache,id=rustfs-inline-linux-target,target=/work/rustfs/target \
cargo build --locked --release --bin rustfs \
&& install -Dm755 target/release/rustfs /out/rustfs
# Build the diagnostic tool after preserving the clean-source server binary.
RUN cp /tmp/inspect_geometry.rs crates/filemeta/examples/inline_lab_inspect.rs
RUN --mount=type=cache,id=rustfs-inline-registry,target=/usr/local/cargo/registry \
--mount=type=cache,id=rustfs-inline-git,target=/usr/local/cargo/git \
--mount=type=cache,id=rustfs-inline-linux-target,target=/work/rustfs/target \
cargo build --locked --release -p rustfs-filemeta --example inline_lab_inspect \
&& install -Dm755 target/release/examples/inline_lab_inspect /out/inline_lab_inspect
FROM debian:trixie-slim AS runtime
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates curl libssl3t64 \
&& rm -rf /var/lib/apt/lists/*
ARG SOURCE_REVISION
LABEL org.opencontainers.image.revision="$SOURCE_REVISION" \
rustfs.validation="inline"
COPY --from=build /out/rustfs /usr/local/bin/rustfs
COPY --from=build /out/inline_lab_inspect /usr/local/bin/inline_lab_inspect
ENTRYPOINT ["/usr/local/bin/rustfs"]
-5
View File
@@ -1,5 +0,0 @@
FROM debian:trixie-slim
COPY mc /usr/bin/mc
RUN chmod 0755 /usr/bin/mc && /usr/bin/mc --version
LABEL rustfs.validation="inline"
ENTRYPOINT ["/usr/bin/mc"]
-133
View File
@@ -1,133 +0,0 @@
# Inline geometry Linux lab
**Use this when** validating inline placement, block geometry, Range reads and reader rollback in a task-owned local Docker cluster.
The adapter reuses the repository's four-node compose topology with 16 logical Docker volumes and EC12+4. It binds one S3 port to loopback, keeps strict durability and checks running binary hashes and persisted object geometry. Docker volumes share the host storage substrate. Performance observations describe this local virtualized lab.
## Build a frozen server image
Use the current stable toolchain pinned in the lab Dockerfile. Supply a Git bundle and the metadata inspector source as the build context. The server is compiled from a clean checkout before the diagnostic example is added.
```bash
set -euo pipefail
LAB_BUILD_DIR="$(mktemp -d)"
LAB_REVISION="$(git rev-parse HEAD)"
git bundle create "$LAB_BUILD_DIR/repository.bundle" HEAD
cp crates/filemeta/examples/inline_lab_inspect.rs "$LAB_BUILD_DIR/inspect_geometry.rs"
docker --context "$LAB_CONTEXT" build --target build-tools \
-t rustfs-inline-build-tools:1.100 \
-f .docker/inline-lab/Dockerfile .docker/inline-lab
docker --context "$LAB_CONTEXT" build \
--build-arg SOURCE_REVISION="$LAB_REVISION" \
-t rustfs-inline-validation:local \
-f .docker/inline-lab/Dockerfile "$LAB_BUILD_DIR"
LAB_EXPORT_ID="$(docker --context "$LAB_CONTEXT" create \
--label rustfs.validation=inline rustfs-inline-validation:local)"
docker --context "$LAB_CONTEXT" cp \
"$LAB_EXPORT_ID:/usr/local/bin/rustfs" "$LAB_BUILD_DIR/rustfs-linux"
docker --context "$LAB_CONTEXT" rm "$LAB_EXPORT_ID"
```
## Configure and probe
Select a new compose project and a private state directory. The adapter generates test credentials there. Existing resources require matching ownership labels. Configure checks the image's source revision and pins immutable image IDs. Each operation takes an exclusive state lock. Deployment waits for `/health/ready` on every node, covering storage, IAM, lock quorum and peer dependencies.
```bash
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" configure \
--context "$LAB_CONTEXT" --project "$LAB_PROJECT" \
--image rustfs-inline-validation:local --revision "$LAB_REVISION" \
--binary "$LAB_BUILD_DIR/rustfs-linux" --port 19080
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" deploy \
--bucket inline-geometry-probe --shard-bytes 0
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" probe \
--bucket inline-geometry-probe
```
The probe verifies complete bytes, HEAD sizes, Range contents, EC12+4 and the actual persisted block. It records inline markers for 4 KiB, 1 MiB, 1 MiB + 1 byte, 1.25 MiB, 1,572,856 bytes, 1,572,857 bytes and 1.5 MiB objects in `probe.json`. The server-only image includes the existing filemeta decoder through a small geometry inspector. Console assets and the administration CLI are outside this lab image.
## ABBA deployment hook
Use `inline_docker_lab.py ... deploy` in the existing `run_hotpath_warp_abba.sh` external deploy hook. Export `RUSTFS_ERASURE_SHARD_BYTES` according to `HOTPATH_ABBA_PHASE`. The adapter uses the supplied leg, phase, bucket, binary and evidence-file context, resets only owned lab volumes, checks running configuration and binary hashes, creates the fresh bucket and writes the runner's deployment evidence.
Use the same frozen binary for both sides of a geometry experiment. Fix the inline budget and encoding mode, then compare zero with one candidate shard value. Include the CPU, memory and image identities from every node in each result. The state `config.json` supplies the generated test credentials for the load driver. Set `DOCKER_CONTEXT` to the selected context when the driver captures Docker resource snapshots.
For content compatibility across an image or geometry change, configure the next selected image and artifact using the same state, then deploy with `--keep-data` and an existing bucket. This retains owned volumes. Compare existing objects and version IDs with the manifest prepared before the change.
## Cleanup
```bash
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" down --purge
```
Cleanup checks ownership for containers, volumes and networks. A configuration cannot adopt another project's resources. Raw measurements, state, Git bundles and build outputs stay in task artifact directories.
Unit checks run with `scripts/python_bin.sh scripts/test_inline_docker_lab.py` and are included in `make script-tests`.
## Linux content-probe client
The PUT after-probe requires a working `mc`. For a macOS client affected by cgo initialization failures, compile the same frozen public mc module for Linux with CGO disabled, build it with `Dockerfile.mc`, and use `scripts/inline_docker_mc.sh` as the process-local `mc` command. Record the module version, checksum and resulting image ID with the run.
The adapter rewrites only the lab loopback alias endpoint to the private `node1` endpoint. Alias configuration stays in the probe's own `MC_CONFIG_DIR`, and `cat` streams raw bytes. The existing HEAD, GET and hash checks continue to execute.
```bash
export RUSTFS_INLINE_LAB_STATE="$LAB_STATE"
export RUSTFS_INLINE_MC_IMAGE="$LAB_MC_IMAGE"
mkdir -p "$LAB_TOOLS"
ln -s "$PWD/scripts/inline_docker_mc.sh" "$LAB_TOOLS/mc"
export PATH="$LAB_TOOLS:$PATH"
```
## Compare transport and placement independently
`deploy --rpc-mode json-compat` explicitly clears both MessagePack-only flags on every node. `deploy --rpc-mode msgpack-only` enables both flags for the selected homogeneous lab fleet. Deployment evidence records the mode and checks both running flags. The default is `json-compat`. The ABBA hook may select the same mode through `RUSTFS_INLINE_LAB_RPC_MODE`.
First compare the base and bounded-multi-block binaries at `--shard-bytes 0` in JSON compatibility mode. Then repeat with MessagePack-only on both sides. A same-binary transport ABBA isolates the encoding effect. Keep the 128 KiB budget, CPU sets, strict durability, load and object set fixed. Check actual markers and block sizes at the boundary sizes before timing. Retain baseline-drift and candidate gates, error logs and raw request distributions.
For a rolling downgrade, redeploy the current binary with `--keep-data --rpc-mode json-compat` before introducing an older node. Check old and new reads, version IDs and batch deletes against the persisted manifest. A running MessagePack-only flag pair establishes the lab configuration. Compatibility with a particular old reader requires the explicit reader test.
## Preserve a compatibility manifest
Run the seed once in a fresh lab bucket. It records full byte oracles and the observed version inventory, including any additional version matching a known body, and leaves a 5 MiB multipart part ready for the next reader. Each bucket has its own private `compat-<bucket>.json` manifest. Keep the same state and volumes across image changes. A fresh deployment or cleanup that resets volumes moves existing compatibility manifests into `compat-history`. Retained-data deployments preserve the active manifests.
```bash
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-seed --bucket inline-compat
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check --bucket inline-compat
# Configure the next frozen image and binary, then retain the data.
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" deploy \
--bucket inline-compat --shard-bytes 0 --rpc-mode json-compat --keep-data
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check --bucket inline-compat
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-finish-mpu --bucket inline-compat
```
The check compares every persisted version ID, HEAD length, complete SHA256, cross-block Range and per-version geometry from `xl.meta`. The seed includes zero-length objects, null versions, the 192 KiB versioned admission boundary and the accumulated 128 KiB shard boundary. Seed a second bucket on the new writer before the downgrade, then check both manifests using the selected old reader. Complete an old upload on the new binary and a new upload on the old binary. After returning to the current fleet, `compat-delete --bucket <bucket>` deletes the recorded versions in one S3 batch and checks the exact remaining inventory and bytes.
Successful multipart-completion and batch-delete receipts are saved before subsequent content and geometry checks. If either check fails, rerun the same command with the same state directory to resume verification from the saved completion state. After a seed check fails with a saved manifest, use `compat-check` to resume verification.
For mixed-version testing, retain JSON compatibility mode throughout the rolling replacement. Verify each node's immutable image and binary, then perform manifest checks and new writes through both coordinator versions using `--coordinator node1` and `--coordinator node4`. The manifest records the actual writer coordinator revision from the running image. Stop one owned node to check read availability at EC12+4, restore it and check the manifests again. Record the exact oldest tested source revision, toolchain and build profile with the evidence. The version boundary is established by these executed cells.
An owned lab can replace just its fourth node through a compose image override. Select the frozen old image first and verify its revision label. Keep `compose.json` in JSON compatibility mode for this cell.
```bash
set -euo pipefail
OLD_IMAGE_ID="$(docker --context "$LAB_CONTEXT" image inspect "$OLD_IMAGE" --format '{{.Id}}')"
test "$(docker --context "$LAB_CONTEXT" image inspect "$OLD_IMAGE_ID" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" = "$OLD_REVISION"
INLINE_OLD_IMAGE_ID="$OLD_IMAGE_ID" scripts/python_bin.sh - <<'PYTHON' > "$LAB_STATE/mixed-image.json"
import json
import os
import sys
override = {"services": {"node4": {"image": os.environ["INLINE_OLD_IMAGE_ID"]}}}
json.dump(override, sys.stdout)
PYTHON
docker --context "$LAB_CONTEXT" compose --project-name "$LAB_PROJECT" \
--file "$LAB_STATE/compose.json" --file "$LAB_STATE/mixed-image.json" \
up --detach --no-deps --no-build --pull never node4
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check \
--bucket inline-compat --coordinator node4
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-seed \
--bucket inline-mixed-writer --coordinator node4
scripts/python_bin.sh scripts/inline_docker_lab.py --state-dir "$LAB_STATE" compat-check \
--bucket inline-mixed-writer --coordinator node1
```
Inspect all four running image IDs and binary hashes before recording the mixed-fleet result. Redeploy the configured current image with `--keep-data --rpc-mode json-compat` to restore a homogeneous fleet. The same manifest checks apply after stopping and restarting one owned node through the base compose file.
-30
View File
@@ -1,30 +0,0 @@
# MQTT Broker (EMQX)
This directory contains the configuration for running an EMQX MQTT broker, which can be used for testing RustFS's MQTT integration.
## 🚀 Quick Start
To start the EMQX broker:
```bash
docker compose up -d
```
## 📊 Access
- **Dashboard**: [http://localhost:18083](http://localhost:18083)
- **Default Credentials**: `admin` / `public`
- **MQTT Port**: `1883`
- **WebSocket Port**: `8083`
## 🛠️ Configuration
The `docker-compose.yml` file sets up a single-node EMQX instance.
- **Persistence**: Data is not persisted by default (for testing).
- **Network**: Uses the default bridge network.
## 📝 Notes
- This setup is intended for development and testing purposes.
- For production deployments, please refer to the official [EMQX Documentation](https://www.emqx.io/docs/en/latest/).
-37
View File
@@ -1,37 +0,0 @@
# 节点配置
node.name = "emqx@127.0.0.1"
node.cookie = "aBcDeFgHiJkLmNoPqRsTuVwXyZ012345"
node.data_dir = "/opt/emqx/data"
# 日志配置
log.console = {level = info, enable = true}
log.file = {path = "/opt/emqx/log/emqx.log", enable = true, level = info}
# MQTT TCP 监听器
listeners.tcp.default = {bind = "0.0.0.0:1883", max_connections = 1000000, enable = true}
# MQTT SSL 监听器
listeners.ssl.default = {bind = "0.0.0.0:8883", enable = false}
# MQTT WebSocket 监听器
listeners.ws.default = {bind = "0.0.0.0:8083", enable = true}
# MQTT WebSocket SSL 监听器
listeners.wss.default = {bind = "0.0.0.0:8084", enable = false}
# 管理控制台
dashboard.listeners.http = {bind = "0.0.0.0:18083", enable = true}
# HTTP API
management.listeners.http = {bind = "0.0.0.0:8081", enable = true}
# 认证配置
authentication = [
{enable = true, mechanism = password_based, backend = built_in_database, user_id_type = username}
]
# 授权配置
authorization.sources = [{type = built_in_database, enable = true}]
# 持久化消息存储
message.storage.backend = built_in_database
-9
View File
@@ -1,9 +0,0 @@
-name emqx@127.0.0.1
-setcookie aBcDeFgHiJkLmNoPqRsTuVwXyZ012345
+P 2097152
+t 1048576
+zdbbl 32768
-kernel inet_dist_listen_min 6000
-kernel inet_dist_listen_max 6100
-smp enable
-mnesia dir "/opt/emqx/data/mnesia"
-74
View File
@@ -1,74 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
services:
emqx:
image: emqx/emqx:latest
container_name: emqx
restart: unless-stopped
environment:
- EMQX_NODE__NAME=emqx@127.0.0.1
- EMQX_NODE__COOKIE=aBcDeFgHiJkLmNoPqRsTuVwXyZ012345
- EMQX_NODE__DATA_DIR=/opt/emqx/data
- EMQX_LOG__CONSOLE__LEVEL=info
- EMQX_LOG__CONSOLE__ENABLE=true
- EMQX_LOG__FILE__PATH=/opt/emqx/log/emqx.log
- EMQX_LOG__FILE__LEVEL=info
- EMQX_LOG__FILE__ENABLE=true
- EMQX_LISTENERS__TCP__DEFAULT__BIND=0.0.0.0:1883
- EMQX_LISTENERS__TCP__DEFAULT__MAX_CONNECTIONS=1000000
- EMQX_LISTENERS__TCP__DEFAULT__ENABLE=true
- EMQX_LISTENERS__SSL__DEFAULT__BIND=0.0.0.0:8883
- EMQX_LISTENERS__SSL__DEFAULT__ENABLE=false
- EMQX_LISTENERS__WS__DEFAULT__BIND=0.0.0.0:8083
- EMQX_LISTENERS__WS__DEFAULT__ENABLE=true
- EMQX_LISTENERS__WSS__DEFAULT__BIND=0.0.0.0:8084
- EMQX_LISTENERS__WSS__DEFAULT__ENABLE=false
- EMQX_DASHBOARD__LISTENERS__HTTP__BIND=0.0.0.0:18083
- EMQX_DASHBOARD__LISTENERS__HTTP__ENABLE=true
- EMQX_MANAGEMENT__LISTENERS__HTTP__BIND=0.0.0.0:8081
- EMQX_MANAGEMENT__LISTENERS__HTTP__ENABLE=true
- EMQX_AUTHENTICATION__1__ENABLE=true
- EMQX_AUTHENTICATION__1__MECHANISM=password_based
- EMQX_AUTHENTICATION__1__BACKEND=built_in_database
- EMQX_AUTHENTICATION__1__USER_ID_TYPE=username
- EMQX_AUTHORIZATION__SOURCES__1__TYPE=built_in_database
- EMQX_AUTHORIZATION__SOURCES__1__ENABLE=true
ports:
- "1883:1883" # MQTT TCP
- "8883:8883" # MQTT SSL
- "8083:8083" # MQTT WebSocket
- "8084:8084" # MQTT WebSocket SSL
- "18083:18083" # Web 管理控制台
- "8081:8081" # HTTP API
volumes:
- ./data:/opt/emqx/data
- ./log:/opt/emqx/log
- ./config:/opt/emqx/etc
networks:
- mqtt-net
healthcheck:
test: [ "CMD", "/opt/emqx/bin/emqx_ctl", "status" ]
interval: 30s
timeout: 10s
retries: 3
logging:
driver: "json-file"
options:
max-size: "100m"
max-file: "3"
networks:
mqtt-net:
driver: bridge
-29
View File
@@ -1,29 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
services:
emqx:
image: emqx/emqx:latest
container_name: emqx
ports:
- "1883:1883"
- "8083:8083"
- "8084:8084"
- "8883:8883"
- "18083:18083"
restart: unless-stopped
networks:
default:
driver: bridge
-93
View File
@@ -1,93 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
worker_processes auto;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
error_log /var/log/nginx/error.log warn;
sendfile on;
keepalive_timeout 65;
# RustFS Server Block
server {
listen 80;
server_name localhost;
# Redirect HTTP to HTTPS (optional, uncomment if SSL is configured)
# return 301 https://$host$request_uri;
location / {
proxy_pass http://rustfs:9000;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# S3 specific headers
proxy_set_header X-Amz-Date $http_x_amz_date;
proxy_set_header Authorization $http_authorization;
# Disable buffering for large uploads
proxy_request_buffering off;
client_max_body_size 0;
}
location /rustfs/console {
proxy_pass http://rustfs:9001;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
}
# SSL Configuration (Example)
# server {
# listen 443 ssl;
# server_name localhost;
#
# ssl_certificate /etc/nginx/ssl/server.crt;
# ssl_certificate_key /etc/nginx/ssl/server.key;
#
# # Restrict to modern TLS versions and ciphers. Operators copying this
# # example must keep at least these directives — without them, nginx
# # may negotiate older protocol versions that have known weaknesses.
# ssl_protocols TLSv1.2 TLSv1.3;
# ssl_prefer_server_ciphers on;
# ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305';
# ssl_session_timeout 1d;
# ssl_session_cache shared:SSL:10m;
# ssl_session_tickets off;
# # add_header Strict-Transport-Security "max-age=63072000" always;
#
# location / {
# proxy_pass http://rustfs:9000;
# ...
# }
# }
}
View File
-5
View File
@@ -1,5 +0,0 @@
jaeger-data/*
loki-data/*
prometheus-data/*
tempo-data/*
grafana-data/*
-225
View File
@@ -1,225 +0,0 @@
# RustFS Observability Stack
This directory contains the comprehensive observability stack for RustFS, designed to provide deep insights into application performance, logs, and traces.
## Components
The stack is composed of the following best-in-class open-source components:
- **Prometheus** (v2.53.1): The industry standard for metric collection and alerting.
- **Grafana** (v11.1.0): The leading platform for observability visualization.
- **Loki** (v3.1.0): A horizontally-scalable, highly-available, multi-tenant log aggregation system.
- **Tempo** (v2.5.0): A high-volume, minimal dependency distributed tracing backend.
- **Jaeger** (v1.59.0): Distributed tracing system (configured as a secondary UI/storage).
- **OpenTelemetry Collector** (v0.104.0): A vendor-agnostic implementation for receiving, processing, and exporting telemetry data.
By default, this stack uses Tempo in single-binary mode and does not require Kafka/Redpanda.
If you want the Kafka-backed HA Tempo path, use `docker-compose-example-for-rustfs.yml` together with `docker-compose-tempo-ha-override.yml`.
## Architecture
1. **Telemetry Collection**: Applications send OTLP (OpenTelemetry Protocol) data (Metrics, Logs, Traces) to the **OpenTelemetry Collector**.
2. **Processing & Exporting**: The Collector processes the data (batching, memory limiting) and exports it to the respective backends:
- **Traces** -> **Tempo** (Primary) & **Jaeger** (Secondary/Optional)
- **Metrics** -> **Prometheus** (via scraping the Collector's exporter)
- **Logs** -> **Loki**
3. **Visualization**: **Grafana** connects to all backends (Prometheus, Tempo, Loki, Jaeger) to provide a unified dashboard experience.
## Features
- **Full Persistence**: All data (Metrics, Logs, Traces) is persisted to Docker volumes, ensuring no data loss on restart.
- **Correlation**: Seamless navigation between Metrics, Logs, and Traces in Grafana.
- Jump from a Metric spike to relevant Traces.
- Jump from a Trace to relevant Logs.
- **High Performance**: Optimized configurations for batching, compression, and memory management.
- **Standardized Protocols**: Built entirely on OpenTelemetry standards.
## GET Performance Optimization Dashboards
Three pre-built Grafana dashboards are included for monitoring RustFS GET performance optimization rollout:
### Available Dashboards
| Dashboard | File | Description |
|-----------|------|-------------|
| **GET Rollout Health** | `grafana-get-rollout-health.json` | Monitors optimization rollout: latency by reader path, early-stop hit rate, codec streaming usage, pipeline failures |
| **GET Data Integrity** | `grafana-get-data-integrity.json` | Monitors data safety: bitrot verify failures, decode errors, short reads, shard read outcomes |
| **GET Resource Impact** | `grafana-get-resource-impact.json` | Monitors resource usage: concurrent requests, IO queue utilization, disk permit wait, RSS trend |
| **Object Data Cache** | `grafana-object-data-cache.json` | Monitors the GET body cache (`rustfs_object_data_cache_*`): hit ratio, lookup/plan/fill outcomes, fill duration quantiles, hit vs fill throughput, entries/weighted bytes, inflight fills, memory-pressure skips, invalidations, and size-class breakdowns |
### Storage Metrics
Storage panels require `prometheus-rules/rustfs-storage.yml` and the cluster resource attribute. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for ownership, observer selection, freshness, and rolling upgrades.
### Prometheus Alert Rules
The file `prometheus-rules/rustfs-get-optimization-alerts.yaml` contains pre-configured alerting rules:
| Alert | Severity | Condition |
|-------|----------|-----------|
| `GetP99Regression` | Critical | GET p99 latency > 2x baseline for 10m |
| `PipelineFailureSpike` | Critical | Pipeline failure rate > 5x baseline for 5m |
| `BitrotMismatchSpike` | Critical | Bitrot mismatch rate > 3x baseline for 5m |
| `EarlyStopInsufficientQuorum` | Warning | Early-stop insufficient quorum rate > 0.1/s for 5m |
| `CodecStreamingFallbackSpike` | Warning | Codec streaming fallback > 10x baseline for 10m |
| `IoQueueSaturation` | Warning | IO queue utilization > 90% for 5m |
The file `prometheus-rules/rustfs-kms-alerts.yml` contains alerting rules for the KMS backend operation metrics. Thresholds are conservative defaults pending staging baseline calibration; response procedures live in `docs/operations/kms-observability-runbook.md`, and the matching dashboard is `deploy/observability/grafana/rustfs-kms-observability.json`.
| Alert | Severity | Condition |
|-------|----------|-----------|
| `KmsBackendFatalErrors` | Critical | Fatal (non-retryable) attempt failures > 0 for 5m |
| `KmsBackendHighErrorRate` | Critical | Non-success operation ratio > 5% for 10m (with traffic guard) |
| `KmsBackendP99LatencyHigh` | Warning | Operation p99 duration (incl. retries) > 2s for 10m |
| `KmsBackendAttemptFailureSpike` | Warning | Attempt failure rate > 0.5/s for 10m |
| `KmsBackendRetryBudgetExhausted` | Warning | budget_exhausted / deadline_exceeded outcomes > 0.05/s for 10m |
### Enabling Alert Rules
Add the alert rules file to your Prometheus configuration:
```yaml
# prometheus.yml
rule_files:
- "/etc/prometheus/rules/*.yml"
# Or mount the file in docker-compose.yml:
# volumes:
# - ./prometheus-rules:/etc/prometheus/rules
```
### Dashboard Usage
The dashboards are automatically provisioned when Grafana starts. They use the `${DS_PROMETHEUS}` datasource variable, so you need a Prometheus datasource configured in Grafana.
Key panels to monitor during optimization rollout:
1. **GET Latency by Reader Path** - Compare `codec_streaming` vs `legacy_duplex` latency
2. **Early-Stop Hit Rate** - Verify early-stop is triggering effectively
3. **Pipeline Failure Rate** - Detect any new failure modes introduced by optimizations
4. **Bitrot Verify Failures** - Ensure data integrity is maintained
## Quick Start
### Prerequisites
- Docker
- Docker Compose
### Deploy
Run the following command to start the entire stack:
```bash
docker compose up -d
```
### High Availability Tempo
The default `docker-compose.yml` is the single-node stack.
If you need the Kafka-backed HA Tempo configuration, start it with:
```bash
docker compose -f docker-compose-example-for-rustfs.yml -f docker-compose-tempo-ha-override.yml up -d
```
### Access Dashboards
| Service | URL | Credentials | Description |
| :------------- | :----------------------------------------------- | :---------------- | :----------------------------- |
| **Grafana** | [http://localhost:3000](http://localhost:3000) | `admin` / `admin` | Main visualization hub. |
| **Prometheus** | [http://localhost:9090](http://localhost:9090) | - | Metric queries and status. |
| **Jaeger UI** | [http://localhost:16686](http://localhost:16686) | - | Secondary trace visualization. |
| **Tempo** | [http://localhost:3200](http://localhost:3200) | - | Tempo status/metrics. |
## Configuration
### Data Persistence
Data is stored in the following Docker volumes:
- `prometheus-data`: Prometheus metrics
- `tempo-data`: Tempo traces (WAL and Blocks)
- `loki-data`: Loki logs (Chunks and Rules)
- `jaeger-data`: Jaeger traces (Badger DB)
To clear all data:
```bash
docker compose down -v
```
### Customization
- **Prometheus**: Edit `prometheus.yml` to add scrape targets or alerting rules.
- **Grafana**: Dashboards and datasources are provisioned from the `grafana/` directory.
- **Collector**: Edit `otel-collector-config.yaml` to modify pipelines, processors, or exporters.
### Verifying RustFS Traces
When RustFS points `RUSTFS_OBS_ENDPOINT` at this stack, treat the value as the
OTLP/HTTP base URL, for example:
```bash
export RUSTFS_OBS_ENDPOINT=http://host.docker.internal:4318
```
RustFS automatically expands that base URL to:
- `/v1/traces`
- `/v1/metrics`
- `/v1/logs`
Important behavior notes:
- Logs and metrics usually appear during startup, so seeing those two signals
first is expected.
- The OpenTelemetry bridge sends `tracing` fields as log attributes. Loki stores
those attributes as structured metadata, and the Collector also mirrors the
common troubleshooting fields into the log line so simple line filters can
find them.
- Visible trace data usually requires real HTTP/S3/gRPC request traffic after
startup, because request-path spans are created on demand.
- `RUSTFS_OBS_LOGGER_LEVEL=info` keeps the top-level request span but filters
many nested `debug` spans. If Tempo or Jaeger looks sparse, retry with
`RUSTFS_OBS_LOGGER_LEVEL=debug` before suspecting collector or Tempo issues.
Minimal validation flow:
```bash
# 1. Start this observability stack.
docker compose up -d
# 2. Start RustFS with OTLP/HTTP export and richer span visibility.
export RUSTFS_OBS_ENDPOINT=http://host.docker.internal:4318
export RUSTFS_OBS_LOGGER_LEVEL=debug
# 3. Generate real request traffic.
curl -I http://127.0.0.1:9000/health
curl -I http://127.0.0.1:9000/health/ready
# 4. Inspect Grafana or Jaeger.
# Grafana: http://localhost:3000
# Jaeger: http://localhost:16686
```
For a structured RustFS log such as an inter-node RPC authentication failure,
the Loki line now includes fields such as `event`, `component`, `subsystem`,
`failure_reason`, `rpc_service`, `rpc_method`, and `expected_audience`. Useful
LogQL checks:
```logql
{service_name="RustFS"} |= "RPC signature verification failed"
{service_name="RustFS"} |= "failure_reason="
{service_name="RustFS"} | failure_reason != ""
```
If logs and metrics are present but traces are sparse, the most common cause is
"no real request traffic yet" or "`info` level filtered nested spans", not an
OTLP routing failure.
## Troubleshooting
- **Service Health**: Check the health of services using `docker compose ps`.
- **Logs**: View logs for a specific service using `docker compose logs -f <service_name>`.
- **Otel Collector**: Check `http://localhost:13133` for health status and `http://localhost:1888/debug/pprof/` for profiling.
-211
View File
@@ -1,211 +0,0 @@
# RustFS 可观测性技术栈
本目录包含 RustFS 的全面可观测性技术栈,旨在提供对应用程序性能、日志和追踪的深入洞察。
## 组件
该技术栈由以下一流的开源组件组成:
- **Prometheus** (v2.53.1): 行业标准的指标收集和告警工具。
- **Grafana** (v11.1.0): 领先的可观测性可视化平台。
- **Loki** (v3.1.0): 水平可扩展、高可用、多租户的日志聚合系统。
- **Tempo** (v2.5.0): 高吞吐量、最小依赖的分布式追踪后端。
- **Jaeger** (v1.59.0): 分布式追踪系统(配置为辅助 UI/存储)。
- **OpenTelemetry Collector** (v0.104.0): 接收、处理和导出遥测数据的供应商无关实现。
默认情况下,这套技术栈使用 Tempo 单二进制模式,不依赖 Kafka/Redpanda。
如果需要基于 Kafka 的 HA Tempo 路径,请使用 `docker-compose-example-for-rustfs.yml` 配合 `docker-compose-tempo-ha-override.yml`。
## 架构
1. **遥测收集**: 应用程序将 OTLP (OpenTelemetry Protocol) 数据(指标、日志、追踪)发送到 **OpenTelemetry Collector**。
2. **处理与导出**: Collector 处理数据(批处理、内存限制)并将其导出到相应的后端:
- **追踪** -> **Tempo** (主要) & **Jaeger** (辅助/可选)
- **指标** -> **Prometheus** (通过抓取 Collector 的导出器)
- **日志** -> **Loki**
3. **可视化**: **Grafana** 连接到所有后端(Prometheus, Tempo, Loki, Jaeger),提供统一的仪表盘体验。
## 特性
- **完全持久化**: 所有数据(指标、日志、追踪)都持久化到 Docker 卷,确保重启后无数据丢失。
- **关联性**: 在 Grafana 中实现指标、日志和追踪之间的无缝导航。
- 从指标峰值跳转到相关追踪。
- 从追踪跳转到相关日志。
- **高性能**: 针对批处理、压缩和内存管理进行了优化配置。
- **标准化协议**: 完全基于 OpenTelemetry 标准构建。
## GET 性能优化仪表盘
包含三个预构建的 Grafana 仪表盘,用于监控 RustFS GET 性能优化发布:
### 可用仪表盘
| 仪表盘 | 文件 | 描述 |
|--------|------|------|
| **GET 发布健康度** | `grafana-get-rollout-health.json` | 监控优化发布:按 reader path 的延迟、early-stop 命中率、codec streaming 使用率、pipeline 失败率 |
| **GET 数据完整性** | `grafana-get-data-integrity.json` | 监控数据安全:bitrot 校验失败、decode 错误、short read、shard 读取结果 |
| **GET 资源影响** | `grafana-get-resource-impact.json` | 监控资源使用:并发请求数、IO 队列利用率、disk permit 等待、RSS 趋势 |
| **对象数据缓存** | `grafana-object-data-cache.json` | 监控 GET body 缓存(`rustfs_object_data_cache_*`):命中率、查找/规划/填充结果、填充耗时分位、命中 vs 填充吞吐、条目数/加权字节、在途填充、内存压力拒绝、失效、按尺寸档拆分 |
### Prometheus 告警规则
文件 `prometheus-rules/rustfs-get-optimization-alerts.yaml` 包含预配置的告警规则:
| 告警 | 级别 | 条件 |
|------|------|------|
| `GetP99Regression` | 严重 | GET p99 延迟 > 2x 基线,持续 10 分钟 |
| `PipelineFailureSpike` | 严重 | Pipeline 失败率 > 5x 基线,持续 5 分钟 |
| `BitrotMismatchSpike` | 严重 | Bitrot 不匹配率 > 3x 基线,持续 5 分钟 |
| `EarlyStopInsufficientQuorum` | 警告 | Early-stop quorum 不足率 > 0.1/s,持续 5 分钟 |
| `CodecStreamingFallbackSpike` | 警告 | Codec streaming 回退 > 10x 基线,持续 10 分钟 |
| `IoQueueSaturation` | 警告 | IO 队列利用率 > 90%,持续 5 分钟 |
文件 `prometheus-rules/rustfs-kms-alerts.yml` 包含 KMS 后端操作指标的告警规则。阈值为保守默认值,待 staging 基线校准;响应流程见 `docs/operations/kms-observability-runbook.md`,配套仪表盘为 `deploy/observability/grafana/rustfs-kms-observability.json`。
| 告警 | 级别 | 条件 |
|------|------|------|
| `KmsBackendFatalErrors` | 严重 | fatal(不可重试)尝试失败 > 0,持续 5 分钟 |
| `KmsBackendHighErrorRate` | 严重 | 非 success 操作占比 > 5%,持续 10 分钟(含流量下限保护) |
| `KmsBackendP99LatencyHigh` | 警告 | 操作 p99 耗时(含重试)> 2s,持续 10 分钟 |
| `KmsBackendAttemptFailureSpike` | 警告 | 尝试失败率 > 0.5/s,持续 10 分钟 |
| `KmsBackendRetryBudgetExhausted` | 警告 | budget_exhausted / deadline_exceeded 结果 > 0.05/s,持续 10 分钟 |
### 启用告警规则
在 Prometheus 配置中添加告警规则文件:
```yaml
# prometheus.yml
rule_files:
- "/etc/prometheus/rules/*.yml"
# 或在 docker-compose.yml 中挂载文件:
# volumes:
# - ./prometheus-rules:/etc/prometheus/rules
```
### 仪表盘使用
仪表盘在 Grafana 启动时自动预置。它们使用 `${DS_PROMETHEUS}` 数据源变量,因此需要在 Grafana 中配置 Prometheus 数据源。
优化发布期间需要关注的关键面板:
1. **GET 延迟按 Reader Path** - 对比 `codec_streaming` vs `legacy_duplex` 延迟
2. **Early-Stop 命中率** - 验证 early-stop 是否有效触发
3. **Pipeline 失败率** - 检测优化引入的新故障模式
4. **Bitrot 校验失败** - 确保数据完整性
## 快速开始
### 前置条件
- Docker
- Docker Compose
### 部署
运行以下命令启动整个技术栈:
```bash
docker compose up -d
```
### Tempo 高可用模式
默认的 `docker-compose.yml` 对应单机栈。
如果需要基于 Kafka 的 HA Tempo 配置,请使用:
```bash
docker compose -f docker-compose-example-for-rustfs.yml -f docker-compose-tempo-ha-override.yml up -d
```
### 访问仪表盘
| 服务 | URL | 凭据 | 描述 |
| :--- | :--- | :--- | :--- |
| **Grafana** | [http://localhost:3000](http://localhost:3000) | `admin` / `admin` | 主要可视化中心。 |
| **Prometheus** | [http://localhost:9090](http://localhost:9090) | - | 指标查询和状态。 |
| **Jaeger UI** | [http://localhost:16686](http://localhost:16686) | - | 辅助追踪可视化。 |
| **Tempo** | [http://localhost:3200](http://localhost:3200) | - | Tempo 状态/指标。 |
## 配置
### 数据持久化
数据存储在以下 Docker 卷中:
- `prometheus-data`: Prometheus 指标
- `tempo-data`: Tempo 追踪 (WAL 和 Blocks)
- `loki-data`: Loki 日志 (Chunks 和 Rules)
- `jaeger-data`: Jaeger 追踪 (Badger DB)
要清除所有数据:
```bash
docker compose down -v
```
### 自定义
- **Prometheus**: 编辑 `prometheus.yml` 以添加抓取目标或告警规则。
- **Grafana**: 仪表盘和数据源从 `grafana/` 目录预置。
- **Collector**: 编辑 `otel-collector-config.yaml` 以修改管道、处理器或导出器。
### 验证 RustFS Trace
当 RustFS 将 `RUSTFS_OBS_ENDPOINT` 指向这套技术栈时,应将该值视为
OTLP/HTTP 的基础 URL,例如:
```bash
export RUSTFS_OBS_ENDPOINT=http://host.docker.internal:4318
```
RustFS 会自动在该基础 URL 后补全:
- `/v1/traces`
- `/v1/metrics`
- `/v1/logs`
需要注意:
- 启动阶段通常会先看到日志和指标,因此“先有日志/指标、后有 trace”是正常现象。
- OpenTelemetry bridge 会把 `tracing` 字段作为日志 attributes 发送。Loki 会将这些 attributes 存为 structured metadata,同时 Collector 会把常用排障字段镜像进日志行,方便用简单的行内容过滤直接查到。
- 可见的 trace 数据通常依赖启动后的真实 HTTP/S3/gRPC 请求流量,因为请求路径上的 span 是按需创建的。
- `RUSTFS_OBS_LOGGER_LEVEL=info` 会保留顶层请求 span,但会过滤掉很多 `debug` 级别的嵌套 span。
如果 Tempo 或 Jaeger 中的 trace 看起来很稀疏,建议先改成 `RUSTFS_OBS_LOGGER_LEVEL=debug`,再判断是否是 collector 或 Tempo 问题。
最小验证流程:
```bash
# 1. 启动本目录下的可观测性技术栈。
docker compose up -d
# 2. 以 OTLP/HTTP 导出方式启动 RustFS,并提高 span 可见性。
export RUSTFS_OBS_ENDPOINT=http://host.docker.internal:4318
export RUSTFS_OBS_LOGGER_LEVEL=debug
# 3. 产生真实请求流量。
curl -I http://127.0.0.1:9000/health
curl -I http://127.0.0.1:9000/health/ready
# 4. 到 Grafana 或 Jaeger 中检查。
# Grafana: http://localhost:3000
# Jaeger: http://localhost:16686
```
对于 RustFS 结构化日志,例如节点间 RPC 鉴权失败,Loki 日志行现在会包含 `event`、`component`、`subsystem`、`failure_reason`、`rpc_service`、`rpc_method`、`expected_audience` 等字段。常用 LogQL 检查:
```logql
{service_name="RustFS"} |= "RPC signature verification failed"
{service_name="RustFS"} |= "failure_reason="
{service_name="RustFS"} | failure_reason != ""
```
如果日志和指标已经正常,但 trace 仍然稀疏,最常见的原因通常是
“还没有真实请求流量”或“`info` 级别过滤了嵌套 span”,而不是 OTLP 路由失败。
## 故障排除
- **服务健康**: 使用 `docker compose ps` 检查服务健康状况。
- **日志**: 使用 `docker compose logs -f <service_name>` 查看特定服务的日志。
- **Otel Collector**: 检查 `http://localhost:13133` 获取健康状态,检查 `http://localhost:1888/debug/pprof/` 进行性能分析。
@@ -1,269 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
services:
rustfs:
security_opt:
- "no-new-privileges:true"
image: rustfs/rustfs:latest
container_name: rustfs-server
ports:
- "9000:9000" # S3 API port
- "9001:9001" # Console port
environment:
- RUSTFS_VOLUMES=/data/rustfs
- RUSTFS_ADDRESS=0.0.0.0:9000
- RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001
- RUSTFS_CONSOLE_ENABLE=true
- RUSTFS_CORS_ALLOWED_ORIGINS=*
- RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS=*
- RUSTFS_ACCESS_KEY=rustfsadmin
- RUSTFS_SECRET_KEY=rustfsadmin
- RUSTFS_OBS_LOGGER_LEVEL=info
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
- OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES:-rustfs.cluster.id=rustfs-dev}
- RUSTFS_OBS_PROFILING_ENDPOINT=http://pyroscope:4040
volumes:
- rustfs-data:/data/rustfs
networks:
- otel-network
restart: unless-stopped
healthcheck:
test:
[
"CMD",
"sh",
"-c",
"curl -f http://127.0.0.1:9000/health && curl -f http://127.0.0.1:9001/rustfs/console/health",
]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
depends_on:
otel-collector:
condition: service_started
rustfs-init:
image: alpine
container_name: rustfs-init
volumes:
- rustfs-data:/data
networks:
- otel-network
command: >
sh -c "
chown -R 10001:10001 /data &&
echo 'Volume Permissions fixed' &&
exit 0
"
restart: no
# --- Tracing ---
tempo:
image: grafana/tempo:latest
container_name: tempo
command: [ "-config.file=/etc/tempo.yaml" ]
volumes:
- ./tempo.yaml:/etc/tempo.yaml:ro
- tempo-data:/var/tempo
ports:
- "3200:3200" # tempo
- "4317" # otlp grpc
- "4318" # otlp http
networks:
- otel-network
restart: unless-stopped
healthcheck:
test: [ "CMD", "/tempo", "-version" ]
interval: 10s
timeout: 5s
retries: 3
start_period: 15s
redpanda:
image: redpandadata/redpanda:latest # for tempo ingest
container_name: redpanda
ports:
- "9092:9092"
networks:
- otel-network
restart: unless-stopped
command: >
redpanda start --overprovisioned
--mode=dev-container
--kafka-addr=PLAINTEXT://0.0.0.0:9092
--advertise-kafka-addr=PLAINTEXT://redpanda:9092
jaeger:
image: jaegertracing/jaeger:latest
container_name: jaeger
environment:
- SPAN_STORAGE_TYPE=badger
- BADGER_EPHEMERAL=false
- BADGER_DIRECTORY_VALUE=/badger/data
- BADGER_DIRECTORY_KEY=/badger/key
- COLLECTOR_OTLP_ENABLED=true
volumes:
- ./jaeger.yaml:/etc/jaeger/config.yml
- jaeger-data:/badger
ports:
- "16686:16686" # Web UI
- "14269:14269" # Admin/Metrics
- "4317" # otlp grpc
- "4318" # otlp http
command: [ "--config", "/etc/jaeger/config.yml" ]
networks:
- otel-network
restart: unless-stopped
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:14269" ]
interval: 10s
timeout: 5s
retries: 3
start_period: 15s
# --- Metrics ---
prometheus:
image: prom/prometheus:latest
container_name: prometheus
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
- prometheus-data:/prometheus
ports:
- "9090:9090"
command:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--web.enable-otlp-receiver" # Enable OTLP
- "--web.enable-remote-write-receiver" # Enable remote write
- "--enable-feature=promql-experimental-functions" # Enable info()
- "--storage.tsdb.retention.time=30d"
restart: unless-stopped
networks:
- otel-network
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy" ]
interval: 10s
timeout: 5s
retries: 3
# --- Logging ---
loki:
image: grafana/loki:latest
container_name: loki
volumes:
- ./loki.yaml:/etc/loki/loki.yaml:ro
- loki-data:/loki
ports:
- "3100:3100"
command: -config.file=/etc/loki/loki.yaml
networks:
- otel-network
restart: unless-stopped
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3100/ready" ]
interval: 15s
timeout: 10s
retries: 5
start_period: 60s
# --- Collection ---
otel-collector:
image: otel/opentelemetry-collector-contrib:latest
volumes:
- ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
ports:
- "1888:1888" # pprof
- "8888:8888" # Prometheus metrics for Collector
- "8889:8889" # Prometheus metrics for application indicators
- "13133:13133" # health check
- "4317:4317" # OTLP gRPC
- "4318:4318" # OTLP HTTP
- "55679:55679" # zpages
networks:
- otel-network
restart: unless-stopped
depends_on:
- tempo
- jaeger
- prometheus
- loki
healthcheck:
test: [ "CMD", "/otelcol-contrib", "--version" ]
interval: 10s
timeout: 5s
retries: 3
# --- Profiles ---
pyroscope:
image: grafana/pyroscope:latest
container_name: pyroscope
ports:
- "4040:4040"
command:
- -self-profiling.disable-push=true
networks:
- otel-network
restart: unless-stopped
# --- Visualization ---
grafana:
image: grafana/grafana:latest
container_name: grafana
ports:
- "3000:3000"
environment:
- GF_SECURITY_ADMIN_PASSWORD=admin
- GF_SECURITY_ADMIN_USER=admin
volumes:
- ./grafana/provisioning:/etc/grafana/provisioning:ro
- ./grafana/dashboards:/etc/grafana/dashboards:ro
- grafana-data:/var/lib/grafana
networks:
- otel-network
restart: unless-stopped
depends_on:
- prometheus
- tempo
- loki
healthcheck:
test:
[ "CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health" ]
interval: 10s
timeout: 5s
retries: 3
volumes:
rustfs-data:
tempo-data:
jaeger-data:
prometheus-data:
loki-data:
grafana-data:
networks:
otel-network:
driver: bridge
name: "network_otel"
ipam:
config:
- subnet: 172.28.0.0/16
driver_opts:
com.docker.network.enable_ipv6: "true"
@@ -1,62 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Docker Compose override file for High Availability Tempo setup
#
# Usage:
# docker-compose -f docker-compose-example-for-rustfs.yml \
# -f docker-compose-tempo-ha-override.yml up
services:
# Override Tempo to use high-availability configuration
tempo:
volumes:
- ./tempo-ha.yaml:/etc/tempo.yaml:ro
- tempo-data:/var/tempo
ports:
- "3200:3200" # Tempo HTTP
- "4317:4317" # OTLP gRPC
- "4318:4318" # OTLP HTTP
- "7946:7946" # Memberlist
- "14250:14250" # Jaeger gRPC
- "14268:14268" # Jaeger Thrift HTTP
- "9411:9411" # Zipkin
environment:
- TEMPO_MEMBERLIST_BIND_PORT=7946
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:3200/ready" ]
interval: 10s
timeout: 5s
retries: 5
start_period: 30s
depends_on:
- redpanda
volumes:
tempo-data:
driver: local
driver_opts:
type: tmpfs
device: tmpfs
o: "size=4g" # Allocate 4GB tmpfs for Tempo data (adjust based on your needs)
# Network configuration remains the same
# networks:
# otel-network:
# driver: bridge
# name: "network_otel"
# ipam:
# config:
# - subnet: 172.28.0.0/16
-211
View File
@@ -1,211 +0,0 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
services:
# --- Tracing ---
tempo:
image: grafana/tempo:2.10.5
container_name: tempo
command: [ "-config.file=/etc/tempo.yaml" ]
volumes:
- ./tempo.yaml:/etc/tempo.yaml:ro
- tempo-data:/var/tempo
ports:
- "3200:3200" # tempo
- "4317" # otlp grpc
- "4318" # otlp http
- "7946" # memberlist
networks:
- otel-network
restart: unless-stopped
healthcheck:
test: [ "CMD", "/tempo", "-version" ]
interval: 10s
timeout: 5s
retries: 3
start_period: 15s
vulture:
image: grafana/tempo-vulture:latest
restart: always
command:
[
"-prometheus-listen-address=:8080",
"-tempo-query-url=http://tempo:3200",
"-tempo-push-url=http://tempo:4317",
]
depends_on:
- tempo
jaeger:
image: jaegertracing/jaeger:latest
container_name: jaeger
environment:
- SPAN_STORAGE_TYPE=badger
- BADGER_EPHEMERAL=false
- BADGER_DIRECTORY_VALUE=/badger/data
- BADGER_DIRECTORY_KEY=/badger/key
- COLLECTOR_OTLP_ENABLED=true
volumes:
- ./jaeger.yaml:/etc/jaeger/config.yml
- jaeger-data:/badger
ports:
- "16686:16686" # Web UI
- "18888:8888" # Metrics
- "4317" # otlp grpc
- "4318" # otlp http
command: [ "--config", "/etc/jaeger/config.yml" ]
networks:
- otel-network
restart: unless-stopped
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:8888/metrics" ]
interval: 10s
timeout: 5s
retries: 3
start_period: 15s
# --- Metrics ---
prometheus:
image: prom/prometheus:latest
container_name: prometheus
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./prometheus-rules:/etc/prometheus/rules:ro
- prometheus-data:/prometheus
ports:
- "9090:9090"
command:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--web.enable-otlp-receiver" # Enable OTLP
- "--web.enable-remote-write-receiver" # Enable remote write
- "--enable-feature=promql-experimental-functions" # Enable info()
- "--storage.tsdb.retention.time=30d"
restart: unless-stopped
networks:
- otel-network
healthcheck:
test: [ "CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy" ]
interval: 10s
timeout: 5s
retries: 3
# --- Logging ---
loki:
image: grafana/loki:latest
container_name: loki
volumes:
- ./loki.yaml:/etc/loki/loki.yaml:ro
- loki-data:/loki
ports:
- "3100:3100"
command: -config.file=/etc/loki/loki.yaml
networks:
- otel-network
restart: unless-stopped
healthcheck:
test: [ "CMD", "/usr/bin/loki", "--version" ]
interval: 15s
timeout: 10s
retries: 5
start_period: 60s
# --- Collection ---
otel-collector:
image: otel/opentelemetry-collector-contrib:latest
volumes:
- ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
ports:
- "1888:1888" # pprof
- "8888:8888" # Prometheus metrics for Collector
- "8889:8889" # Prometheus metrics for application indicators
- "13133:13133" # health check
- "4317:4317" # OTLP gRPC
- "4318:4318" # OTLP HTTP
- "55679:55679" # zpages
networks:
- otel-network
restart: unless-stopped
depends_on:
- tempo
- jaeger
- prometheus
- loki
healthcheck:
test: [ "CMD", "/otelcol-contrib", "--version" ]
interval: 10s
timeout: 5s
retries: 3
# --- Profiles ---
pyroscope:
image: grafana/pyroscope:latest
container_name: pyroscope
ports:
- "4040:4040"
command:
- -self-profiling.disable-push=true
networks:
- otel-network
restart: unless-stopped
# --- Visualization ---
grafana:
image: grafana/grafana:latest
container_name: grafana
ports:
- "3000:3000"
environment:
- GF_SECURITY_ADMIN_PASSWORD=admin
- GF_SECURITY_ADMIN_USER=admin
volumes:
- ./grafana/provisioning:/etc/grafana/provisioning:ro
- ./grafana/dashboards:/etc/grafana/dashboards:ro
- grafana-data:/var/lib/grafana
networks:
- otel-network
restart: unless-stopped
depends_on:
- prometheus
- tempo
- loki
healthcheck:
test:
[ "CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health" ]
interval: 10s
timeout: 5s
retries: 3
volumes:
tempo-data:
jaeger-data:
prometheus-data:
loki-data:
grafana-data:
networks:
otel-network:
driver: bridge
name: "network_otel"
ipam:
config:
- subnet: 172.28.0.0/16
driver_opts:
com.docker.network.enable_ipv6: "true"
@@ -1,500 +0,0 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 20,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "line+area"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "transparent",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["mean", "max", "sum"]
}
},
"title": "Bitrot Verify Failures",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_io_get_object_pipeline_failures_total{stage=\"bitrot_verify\"}[$__rate_interval])) by (path, reason)",
"legendFormat": "bitrot_fail {{path}} / {{reason}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_io_get_object_pipeline_failures_total{reason=\"bitrot_verify_failed\"}[$__rate_interval])) by (path, stage)",
"legendFormat": "verify_failed {{path}} / {{stage}}",
"refId": "B"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 2,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["mean", "max"]
}
},
"title": "Bitrot Verify Duration (p50 / p95 / p99)",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.50, sum(rate(rustfs_io_get_object_shard_bitrot_verify_duration_seconds_bucket[$__rate_interval])) by (le))",
"legendFormat": "p50",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, sum(rate(rustfs_io_get_object_shard_bitrot_verify_duration_seconds_bucket[$__rate_interval])) by (le))",
"legendFormat": "p95",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, sum(rate(rustfs_io_get_object_shard_bitrot_verify_duration_seconds_bucket[$__rate_interval])) by (le))",
"legendFormat": "p99",
"refId": "C"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 20,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "line+area"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "transparent",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 3,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["mean", "max", "sum"]
}
},
"title": "Decode Errors",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_io_get_object_pipeline_failures_total{reason=\"decode_error\"}[$__rate_interval])) by (path, stage)",
"legendFormat": "decode_error {{path}} / {{stage}}",
"refId": "A"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 20,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "line+area"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "transparent",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"id": 4,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["mean", "max", "sum"]
}
},
"title": "Short Reads",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_io_get_object_pipeline_failures_total{reason=\"short_read\"}[$__rate_interval])) by (path, stage)",
"legendFormat": "short_read {{path}} / {{stage}}",
"refId": "A"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
},
"id": 5,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["mean", "max"]
}
},
"title": "Shard Read Outcomes by Role",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_io_get_object_shard_read_total[$__rate_interval])) by (path, role, outcome)",
"legendFormat": "{{path}} {{role}}/{{outcome}}",
"refId": "A"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 16
},
"id": 6,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["mean", "max"]
}
},
"title": "Request Result Status Rate",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_io_get_object_request_results_total[$__rate_interval])) by (status)",
"legendFormat": "{{status}}",
"refId": "A"
}
]
}
],
"refresh": "30s",
"schemaVersion": 38,
"style": "dark",
"tags": ["rustfs", "get-optimization", "data-integrity"],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "Prometheus",
"value": "Prometheus"
},
"hide": 0,
"includeAll": false,
"multi": false,
"name": "DS_PROMETHEUS",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "RustFS GET Data Integrity",
"uid": "rustfs-get-data-integrity",
"version": 1
}
@@ -1,716 +0,0 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_strategy_total[$__rate_interval])) or vector(0)",
"legendFormat": "{{strategy}} / {{mode}}",
"refId": "A"
}
],
"title": "GET Reader Setup Strategy Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (path, strategy, size_bucket) (rate(rustfs_io_get_object_reader_setup_strategy_by_size_total[$__rate_interval])) or vector(0)",
"legendFormat": "{{path}} / {{strategy}} / {{size_bucket}}",
"refId": "A"
}
],
"title": "GET Reader Setup Strategy by Size",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "none",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 3,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_scheduled_sum[$__rate_interval])) / clamp_min(sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_scheduled_count[$__rate_interval])), 1) or vector(0)",
"legendFormat": "scheduled {{strategy}} / {{mode}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_attempted_sum[$__rate_interval])) / clamp_min(sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_attempted_count[$__rate_interval])), 1) or vector(0)",
"legendFormat": "attempted {{strategy}} / {{mode}}",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_ready_sum[$__rate_interval])) / clamp_min(sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_ready_count[$__rate_interval])), 1) or vector(0)",
"legendFormat": "ready {{strategy}} / {{mode}}",
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_deferred_sum[$__rate_interval])) / clamp_min(sum by (strategy, mode) (rate(rustfs_io_get_object_reader_setup_deferred_count[$__rate_interval])), 1) or vector(0)",
"legendFormat": "deferred {{strategy}} / {{mode}}",
"refId": "D"
}
],
"title": "GET Reader Setup Fanout Average",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "none",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"id": 4,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (path, strategy, size_bucket) (rate(rustfs_io_get_object_reader_setup_scheduled_by_size_sum[$__rate_interval])) / clamp_min(sum by (path, strategy, size_bucket) (rate(rustfs_io_get_object_reader_setup_scheduled_by_size_count[$__rate_interval])), 1) or vector(0)",
"legendFormat": "scheduled {{path}} / {{strategy}} / {{size_bucket}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (path, strategy, size_bucket) (rate(rustfs_io_get_object_reader_setup_ready_by_size_sum[$__rate_interval])) / clamp_min(sum by (path, strategy, size_bucket) (rate(rustfs_io_get_object_reader_setup_ready_by_size_count[$__rate_interval])), 1) or vector(0)",
"legendFormat": "ready {{path}} / {{strategy}} / {{size_bucket}}",
"refId": "B"
}
],
"title": "GET Reader Setup Fanout by Size",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
},
"id": 5,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, sum by (path, stage, size_bucket, le) (rate(rustfs_io_get_object_stage_duration_seconds_by_size_bucket{stage=~\"reader_setup|reader_setup_schedule|reader_setup_wait_quorum|reader_setup_drop_pending|reader_task_file_open|reader_task_reader_construction|reader_task_bitrot_reader_init\"}[$__rate_interval]))) or vector(0)",
"legendFormat": "p95 {{path}} / {{stage}} / {{size_bucket}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, sum by (path, stage, size_bucket, le) (rate(rustfs_io_get_object_stage_duration_seconds_by_size_bucket{stage=~\"reader_setup|reader_setup_schedule|reader_setup_wait_quorum|reader_setup_drop_pending|reader_task_file_open|reader_task_reader_construction|reader_task_bitrot_reader_init\"}[$__rate_interval]))) or vector(0)",
"legendFormat": "p99 {{path}} / {{stage}} / {{size_bucket}}",
"refId": "B"
}
],
"title": "GET Hot Stage Latency by Size",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 16
},
"id": 6,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, sum by (path, stage, le) (rate(rustfs_io_get_object_stage_duration_seconds_bucket{stage=~\"request_context|first_byte|response_handoff|full_body|body_build|output_poll|output_lock_wait|reader_open_mmap_copy_success|reader_open_mmap_copy_fallback|reader_open_stream|reader_stream_first_read|reader_mmap_blocking_wait|reader_mmap_blocking_task|reader_mmap_file_open|reader_mmap_map|reader_mmap_copy_buffer|reader_mmap_direct_read_copy\"}[$__rate_interval]))) or vector(0)",
"legendFormat": "p95 {{path}} / {{stage}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, sum by (path, stage, le) (rate(rustfs_io_get_object_stage_duration_seconds_bucket{stage=~\"request_context|first_byte|response_handoff|full_body|body_build|output_poll|output_lock_wait|reader_open_mmap_copy_success|reader_open_mmap_copy_fallback|reader_open_stream|reader_stream_first_read|reader_mmap_blocking_wait|reader_mmap_blocking_task|reader_mmap_file_open|reader_mmap_map|reader_mmap_copy_buffer|reader_mmap_direct_read_copy\"}[$__rate_interval]))) or vector(0)",
"legendFormat": "p99 {{path}} / {{stage}}",
"refId": "B"
}
],
"title": "GET Reader, Handler and Body Latency",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 24
},
"id": 7,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (subpath) (rate(rustfs_io_get_object_direct_memory_subpath_total[$__rate_interval])) or vector(0)",
"legendFormat": "{{subpath}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (subpath, size_bucket) (rate(rustfs_io_get_object_direct_memory_subpath_by_size_total[$__rate_interval])) or vector(0)",
"legendFormat": "{{subpath}} / {{size_bucket}}",
"refId": "B"
}
],
"title": "GET Direct-Memory Subpath Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 24
},
"id": 8,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (path, stage, kind) (rate(rustfs_io_get_object_mmap_page_faults_total[$__rate_interval])) or vector(0)",
"legendFormat": "mmap {{path}} / {{stage}} / {{kind}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (path, stage, kind) (rate(rustfs_io_get_object_direct_read_page_faults_total[$__rate_interval])) or vector(0)",
"legendFormat": "direct-read {{path}} / {{stage}} / {{kind}}",
"refId": "B"
}
],
"title": "GET mmap vs Direct-Read Page Fault Rate",
"type": "timeseries"
}
],
"refresh": "30s",
"schemaVersion": 38,
"style": "dark",
"tags": [
"rustfs",
"get",
"performance",
"attribution"
],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "Prometheus",
"value": "Prometheus"
},
"hide": 0,
"includeAll": false,
"multi": false,
"name": "DS_PROMETHEUS",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "",
"title": "RustFS GET Performance Attribution",
"uid": "rustfs-get-performance-attribution",
"version": 1,
"weekStart": ""
}
@@ -1,525 +0,0 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "none",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 20,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["mean", "max"]
}
},
"title": "Concurrent GET Requests",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rustfs_io_get_object_concurrent_requests",
"legendFormat": "concurrent GETs",
"refId": "A"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "percent",
"min": 0,
"max": 100,
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 20,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "line+area"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "transparent",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 2,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["mean", "max"]
}
},
"title": "IO Queue Utilization",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rustfs_io_queue_utilization_percent",
"legendFormat": "queue utilization %",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rustfs_io_queue_permits_in_use",
"legendFormat": "permits in use",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rustfs_io_queue_permits_available",
"legendFormat": "permits available",
"refId": "C"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 3,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["mean", "max"]
}
},
"title": "Disk Permit Wait Duration (p50 / p95 / p99)",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.50, sum(rate(rustfs_io_disk_permit_wait_duration_seconds_bucket[$__rate_interval])) by (le))",
"legendFormat": "p50",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, sum(rate(rustfs_io_disk_permit_wait_duration_seconds_bucket[$__rate_interval])) by (le))",
"legendFormat": "p95",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, sum(rate(rustfs_io_disk_permit_wait_duration_seconds_bucket[$__rate_interval])) by (le))",
"legendFormat": "p99",
"refId": "C"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "bytes",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"id": 4,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["lastNotNull"]
}
},
"title": "RSS Trend (Process Memory)",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "process_resident_memory_bytes{job=~\"rustfs.*\"}",
"legendFormat": "RSS {{instance}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rustfs_process_memory_bytes",
"legendFormat": "RustFS memory {{instance}}",
"refId": "B"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
},
"id": 5,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["mean", "max"]
}
},
"title": "IO Queue Operations by Priority",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_io_queue_operations[$__rate_interval])) by (operation, priority)",
"legendFormat": "{{operation}} / {{priority}}",
"refId": "A"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 16
},
"id": 6,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["sum"]
}
},
"title": "IO Queue Congestion Events",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_io_queue_congestion_total[$__rate_interval]))",
"legendFormat": "congestion events/s",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_io_get_object_timeout_total[$__rate_interval])) by (stage)",
"legendFormat": "timeout {{stage}}",
"refId": "B"
}
]
}
],
"refresh": "30s",
"schemaVersion": 38,
"style": "dark",
"tags": ["rustfs", "get-optimization", "resource-impact"],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "Prometheus",
"value": "Prometheus"
},
"hide": 0,
"includeAll": false,
"multi": false,
"name": "DS_PROMETHEUS",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "RustFS GET Resource Impact",
"uid": "rustfs-get-resource-impact",
"version": 1
}
@@ -1,530 +0,0 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"axisLabel": "",
"axisColorMode": "text",
"scaleDistribution": {
"type": "linear"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["mean", "max"]
}
},
"title": "GET Latency by Reader Path (p50 / p95 / p99)",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.50, sum(rate(rustfs_io_get_object_stage_duration_seconds_bucket{stage=\"total\"}[$__rate_interval])) by (le, path))",
"legendFormat": "p50 {{path}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, sum(rate(rustfs_io_get_object_stage_duration_seconds_bucket{stage=\"total\"}[$__rate_interval])) by (le, path))",
"legendFormat": "p95 {{path}}",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, sum(rate(rustfs_io_get_object_stage_duration_seconds_bucket{stage=\"total\"}[$__rate_interval])) by (le, path))",
"legendFormat": "p99 {{path}}",
"refId": "C"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1,
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.3
},
{
"color": "red",
"value": 0.7
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 2,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["mean", "max"]
}
},
"title": "Early-Stop Hit Rate",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_io_get_object_metadata_early_stop_total{decision=\"hit\"}[$__rate_interval])) by (path) / clamp_min(sum(rate(rustfs_io_get_object_metadata_early_stop_total[$__rate_interval])) by (path), 1)",
"legendFormat": "hit rate {{path}}",
"refId": "A"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1,
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 3,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["mean", "max"]
}
},
"title": "Codec Streaming Usage Rate",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_io_get_object_codec_streaming_decision_total[$__rate_interval])) by (decision) / clamp_min(sum(rate(rustfs_io_get_object_codec_streaming_decision_total[$__rate_interval])), 1)",
"legendFormat": "codec_streaming {{decision}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_io_get_object_stream_strategy_total[$__rate_interval])) by (strategy) / clamp_min(sum(rate(rustfs_io_get_object_stream_strategy_total[$__rate_interval])), 1)",
"legendFormat": "stream_strategy {{strategy}}",
"refId": "B"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "bars",
"fillOpacity": 80,
"lineWidth": 0,
"pointSize": 5,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "normal",
"group": "A"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"id": 4,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["sum"]
}
},
"title": "Pipeline Failure Rate by Stage",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_io_get_object_pipeline_failures_total[$__rate_interval])) by (stage, reason)",
"legendFormat": "{{stage}} / {{reason}}",
"refId": "A"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
},
"id": 5,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["mean", "max"]
}
},
"title": "GET Request Rate by Path",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_io_get_object_reader_path_total[$__rate_interval])) by (path)",
"legendFormat": "{{path}}",
"refId": "A"
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 16
},
"id": 6,
"options": {
"reduceOptions": {
"values": false,
"calcs": ["lastNotNull"],
"fields": ""
},
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": ["mean", "max"]
}
},
"title": "GET Total Latency (p50 / p95 / p99)",
"type": "timeseries",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.50, sum(rate(rustfs_io_get_object_total_duration_seconds_bucket[$__rate_interval])) by (le))",
"legendFormat": "p50",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, sum(rate(rustfs_io_get_object_total_duration_seconds_bucket[$__rate_interval])) by (le))",
"legendFormat": "p95",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, sum(rate(rustfs_io_get_object_total_duration_seconds_bucket[$__rate_interval])) by (le))",
"legendFormat": "p99",
"refId": "C"
}
]
}
],
"refresh": "30s",
"schemaVersion": 38,
"style": "dark",
"tags": ["rustfs", "get-optimization"],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "Prometheus",
"value": "Prometheus"
},
"hide": 0,
"includeAll": false,
"multi": false,
"name": "DS_PROMETHEUS",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "RustFS GET Rollout Health",
"uid": "rustfs-get-rollout-health",
"version": 1
}
File diff suppressed because it is too large Load Diff
@@ -1,534 +0,0 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (path, eager_status, size_bucket, buffer_bucket, large_concurrency_tuning) (rate(rustfs_s3_put_object_diagnostics_total[$__rate_interval])) or vector(0)",
"legendFormat": "{{path}} / {{eager_status}} / {{size_bucket}} / {{buffer_bucket}} / large={{large_concurrency_tuning}}",
"refId": "A"
}
],
"title": "PUT Diagnostics Decision Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (path) (rate(rustfs_s3_put_object_path_total[$__rate_interval])) or vector(0)",
"legendFormat": "{{path}}",
"refId": "A"
}
],
"title": "PUT Path Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ms",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 3,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, sum by (stage, le) (rate(rustfs_s3_put_object_stage_duration_ms_bucket{stage=~\"ingress_prepare|set_disk_writer_setup|set_disk_encode|set_disk_rename|set_disk_old_data_cleanup|multipart_ingress_prepare|multipart_set_disk_writer_setup|multipart_set_disk_encode|multipart_complete_tail\"}[$__rate_interval]))) or vector(0)",
"legendFormat": "p95 {{stage}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, sum by (stage, le) (rate(rustfs_s3_put_object_stage_duration_ms_bucket{stage=~\"ingress_prepare|set_disk_writer_setup|set_disk_encode|set_disk_rename|set_disk_old_data_cleanup|multipart_ingress_prepare|multipart_set_disk_writer_setup|multipart_set_disk_encode|multipart_complete_tail\"}[$__rate_interval]))) or vector(0)",
"legendFormat": "p99 {{stage}}",
"refId": "B"
}
],
"title": "PUT Hot Stage Latency",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "bytes",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"id": 4,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.50, sum by (path, size_bucket, le) (rate(rustfs_s3_put_object_selected_buffer_size_bytes_bucket[$__rate_interval]))) or vector(0)",
"legendFormat": "p50 {{path}} / {{size_bucket}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, sum by (path, size_bucket, le) (rate(rustfs_s3_put_object_selected_buffer_size_bytes_bucket[$__rate_interval]))) or vector(0)",
"legendFormat": "p95 {{path}} / {{size_bucket}}",
"refId": "B"
}
],
"title": "PUT Selected Buffer Size",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1,
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
},
"id": 5,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_s3_put_object_zero_copy_eligible_total[$__rate_interval])) / clamp_min(sum(rate(rustfs_s3_put_object_total[$__rate_interval])), 1) or vector(0)",
"legendFormat": "eligible / total",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_s3_put_object_zero_copy_enabled_total[$__rate_interval])) / clamp_min(sum(rate(rustfs_s3_put_object_total[$__rate_interval])), 1) or vector(0)",
"legendFormat": "enabled / total",
"refId": "B"
}
],
"title": "PUT Zero-Copy Eligibility Ratio",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"pointSize": 5,
"lineWidth": 1,
"showPoints": "auto",
"spanNulls": false,
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 16
},
"id": 6,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(rustfs_s3_put_object_total[$__rate_interval])) or vector(0)",
"legendFormat": "PUT total",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (stage) (rate(rustfs_system_storage_erasure_write_quorum_failures_total[$__rate_interval])) or vector(0)",
"legendFormat": "write quorum failure {{stage}}",
"refId": "B"
}
],
"title": "PUT Throughput and Quorum Failure Signals",
"type": "timeseries"
}
],
"refresh": "30s",
"schemaVersion": 38,
"style": "dark",
"tags": [
"rustfs",
"put",
"performance",
"attribution"
],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "Prometheus",
"value": "Prometheus"
},
"hide": 0,
"includeAll": false,
"multi": false,
"name": "DS_PROMETHEUS",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timezone": "",
"title": "RustFS PUT Performance Attribution",
"uid": "rustfs-put-performance-attribution",
"version": 1,
"weekStart": ""
}
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More