Compare commits

..

89 Commits

Author SHA1 Message Date
Henry Guo 64c0ede026 feat(table-catalog): add product API surface (#3320)
* feat(table-catalog): add product API surface

* fix(table-catalog): harden product API commits

* fix(table-catalog): close product API review gaps

* fix(table-catalog): validate rollback metadata before commit

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-10 07:58:22 +00:00
Henry Guo 66fd55a8e0 feat(scanner): expose pacing pressure status (#3319)
* feat(scanner): expose pacing pressure status

* fix(scanner): preserve merged pause pressure

* fix(scanner): default missing primary pressure

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-10 07:33:41 +00:00
安正超 bb5d9565a6 feat(storage-api): add bucket DTO contract (#3314)
* feat(storage-api): add bucket DTO contract

* ci(build): increase workflow timeout to 90 minutes

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
2026-06-10 07:16:14 +00:00
houseme b9c924a6ed build(ci): update macOS runner and bump regex/uuid (#3318)
* ci(build): use macos-26-intel for x86_64 job

* build(deps): bump regex and uuid

* ci(build): disable cross for macos x86_64 job

---------

Co-authored-by: majinghe <42570491+majinghe@users.noreply.github.com>
2026-06-10 04:42:10 +00:00
houseme 68400933b5 chore(release): prepare 1.0.0-beta.8 (#3317)
* chore(release): prepare 1.0.0-beta.8

* chore(release): align release assets for 1.0.0-beta.8
2026-06-10 04:34:39 +00:00
安正超 0dc99a1abb docs: inventory KMS development defaults (#3304) 2026-06-10 12:46:08 +08:00
Henry Guo d7a7d1fc1e feat(table-catalog): add REST catalog alias (#3316)
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-10 03:55:58 +00:00
Henry Guo 62e9c5b94f feat(scanner): add adaptive pacing controls (#3315)
* feat(scanner): add adaptive pacing controls

* fix(scanner): bound pacing delay status precision

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-10 03:46:02 +00:00
安正超 f80162b5c9 docs: decide config model boundary (#3307) 2026-06-10 10:02:53 +08:00
Henry Guo 2eafd9c602 feat(table-catalog): add credential response boundary (#3305)
* feat(table-catalog): add credential response boundary

* fix(table-catalog): enforce scoped catalog resources

* fix(table-catalog): reduce admin auth scope arguments

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
2026-06-10 01:15:51 +00:00
安正超 a73c90c811 security: redact IAM and target debug secrets (#3306) 2026-06-10 09:03:50 +08:00
安正超 3795d44b86 feat(storage-api): add error code contract (#3313) 2026-06-10 09:03:29 +08:00
安正超 5fef105484 docs: define background controller contract (#3312) 2026-06-10 07:30:21 +08:00
安正超 bc95b9fb9e test: guard table catalog ingress unknown fields (#3308) 2026-06-09 20:06:36 +00:00
安正超 45391ac776 feat(storage-api): add contract crate (#3310) 2026-06-09 20:06:15 +00:00
安正超 f9a5e6d7e6 docs: inventory background services (#3311) 2026-06-10 04:05:33 +08:00
安正超 03eb10b07f feat: add KMS config redaction safeguards (#3303) 2026-06-09 14:08:58 +00:00
Henry Guo 7ec16e197c feat(table-catalog): add metadata maintenance control plane (#3302)
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-09 22:08:28 +08:00
Julien Pervillé 992de65c58 feat(helm): add priorityClassName attribute (#3301)
Co-authored-by: houseme <housemecn@gmail.com>
2026-06-09 22:07:41 +08:00
Maksim Vykhota c4d0d395af Refactor(CLI): avoid unnecessary unreachable! macro in CLI parsing (#3288)
* Refactor(CLI): avoid unnecessary `unreachable!` macro in CLI parsing

* Fix: clippy lint

---------

Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: houseme <housemecn@gmail.com>
2026-06-09 22:01:51 +08:00
Henry Guo e79c793803 feat(heal): add scanner-aware bitrot controls (#3297)
* feat(heal): add scanner-aware bitrot controls

* fix(config): register heal defaults

* test(admin): avoid heal config queue conflict

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
2026-06-09 13:43:18 +00:00
Henry Guo 0cdcd1eb7b feat(table-catalog): internalize catalog backing paths (#3295)
* feat(table-catalog): internalize catalog backing paths

* fix(table-catalog): clean internal catalog on bucket delete

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-09 12:01:25 +00:00
cxymds 55590e38fb fix(config): accept Kafka SASL keys in legacy admin (#3300) 2026-06-09 12:01:15 +00:00
安正超 f5bb034ec8 feat(kms): migrate KMS handlers to dedicated actions (#3298)
feat: migrate KMS handlers to dedicated actions
2026-06-09 12:01:02 +00:00
安正超 4fec606dc4 feat: add KMS action taxonomy (#3294) 2026-06-09 05:06:15 +00:00
Henry Guo ad9bf41fc8 feat(heal): expose scanner heal admission outcomes (#3292)
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: cxymds <Cxymds@qq.com>
2026-06-09 03:39:57 +00:00
Henry Guo 51c26278a4 feat(table-catalog): support standard REST create commit (#3287)
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-09 02:18:33 +00:00
安正超 b9a3a2245a feat: add admin route policy inventory (#3286)
* feat: add admin route policy inventory

* fix: align table route policy actions

* fix: align load table route policy action
2026-06-09 02:17:50 +00:00
houseme aabda41ae5 build: upgrade Rust baseline to 1.96.0 (#3291)
* build: upgrade Rust baseline to 1.96.0

* ci: pin Rust workflows to 1.96.0

* remove

* ci: pin setup action to Rust 1.96.0

* build: pin toolchain to Rust 1.96.0
2026-06-08 20:32:37 +00:00
houseme 9354b939ca refactor(zip): simplify archive extraction path (#3290)
* refactor(zip): simplify archive extraction path

* build(deps): align datafusion and http updates

* fix(test): restore pre-commit compatibility
2026-06-08 20:03:02 +00:00
Henry Guo 6413df4b7f feat(replication): expose scanner repair outcomes (#3278)
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: cxymds <Cxymds@qq.com>
2026-06-08 14:24:27 +00:00
cxymds 452c003341 fix(scanner): ignore missing rebalance metadata (#3282)
* fix(ecstore): harden rebalance data movement

* fix(ecstore): preserve failed rebalance status

* fix(ecstore): avoid rebalance walkdir total timeout

* fix(ecstore): retry rebalance listing timeouts

* fix(ecstore): restrict data movement resume target

* refactor(ecstore): simplify multipart movement target

* fix(ecstore): restore resume target checks

* perf(ecstore): speed up rebalance bucket merges

* fix: keep rebalance listing alive on transient failures

* fix(scanner): ignore missing rebalance metadata

---------

Co-authored-by: houseme <housemecn@gmail.com>
2026-06-08 14:24:16 +00:00
Henry Guo 8c3e52efb8 feat(table-catalog): refine table catalog permissions (#3283)
* feat(table-catalog): refine table catalog permissions

* fix(policy): scope table admin resources

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: cxymds <Cxymds@qq.com>
2026-06-08 14:23:52 +00:00
唐小鸭 f7724d223b feat(rio): rio_v2 is compatible with minio for storing data. (#3115)
* Set up a compatibility layer for replacing old Rio components with new ones.

* fix(rio). compress range

* feat(rio). Add the experimental feature rio_v2 to support minio data at the binary level.

* feat(rio_v2): add sse-c test

* test compression component

* simple fix

* fix minlz encode

* fix metadata

* fix kms key cache error

* Update launch.json

* ci: set nix crate download user agent

* fix: gate obs pyroscope backend

* ignore minio test

* fix encrypt check

* fix

* fix

* fix

* Update object_usecase.rs

* Update ci.yml

* fix

* ci add rio-v2 test

* fix

* ci fix

* fix

* Reconstructed into a more reasonable compatibility mode

* fix

* fix

---------

Signed-off-by: houseme <housemecn@gmail.com>
Signed-off-by: 唐小鸭 <tangtang1251@qq.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
2026-06-08 11:59:14 +00:00
cxymds 9504dff595 fix(ecstore): harden rebalance data movement (#3234)
* fix(ecstore): harden rebalance data movement

* fix(ecstore): preserve failed rebalance status

* fix(ecstore): avoid rebalance walkdir total timeout

* fix(ecstore): retry rebalance listing timeouts

* fix(ecstore): restrict data movement resume target

* refactor(ecstore): simplify multipart movement target

* fix(ecstore): restore resume target checks

* perf(ecstore): speed up rebalance bucket merges

* fix: keep rebalance listing alive on transient failures

---------

Co-authored-by: houseme <housemecn@gmail.com>
2026-06-08 10:40:26 +00:00
Michael 2fc1cf90e0 feat(helm): add option to disable log PVCs and mounts (#3189)
Co-authored-by: cxymds <Cxymds@qq.com>
2026-06-08 10:35:12 +00:00
houseme ed73952cb6 perf(ecstore): improve erasure write diagnostics and single-block performance (#3280)
* docs(object-capacity): add localized crate docs

* fix(ecstore): improve quorum and transport diagnostics

* perf(ecstore): add safe single-block write fast path

* refactor(ecstore): collapse layered small write paths

* chore(docs): keep issue 662 design note tracked

* fix(docs): restore issue 662 design note

* chore(docs): keep issue 662 design local only

* feat(obs): add internode reliability metrics and dashboard

* feat(obs): extend internode diagnostics and service logging

* fix(docs): use AGENTS guide filename

* perf(ecstore): reuse owned buffer in small encode

* fix(ecstore): tighten small write diagnostics

---------

Co-authored-by: cxymds <Cxymds@qq.com>
2026-06-08 09:45:56 +00:00
cxymds ddd35badad fix(ecstore): avoid offline disks on admin timeout (#3263)
* fix(ecstore): avoid offline disks on admin timeout

* fix(ecstore): handle poisoned admin cache locks

* fix(ecstore): recover poisoned admin cache on success

---------

Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
2026-06-08 08:35:36 +00:00
Henry Guo f40dd2f93c fix(scanner): publish partial usage for compacted scans (#3277)
* fix(scanner): publish partial usage for compacted scans

* fix(scanner): publish first partial usage immediately

* fix(scanner): skip startup delay for cold usage cache

* fix(scanner): tighten cold usage publish gate

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
2026-06-08 06:18:35 +00:00
Henry Guo 4751a9f4b9 feat(table-catalog): add recovery diagnostics (#3275)
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
2026-06-08 03:59:13 +00:00
安正超 3d0e6ce0da feat: add security governance policy contracts (#3271)
* feat: add security governance policy contracts

* fix: require signatures for release assets

* docs: update policy verification counts

---------

Co-authored-by: houseme <housemecn@gmail.com>
2026-06-08 02:42:03 +00:00
安正超 78df276d25 fix: return 503 on lock contention instead of 500 (#3274)
* fix(ecstore): skip hidden metadata in walk limit

* fix(ecstore): match walk limit to visible versions

* fix(ecstore): avoid decoding all versions for limit

* fix: return 503 on lock contention instead of 500

When concurrent PUTs to the same key contend for the namespace write
lock, lock timeout and conflict errors were wrapped as
StorageError::other(...) → StorageError::Io(...), which fell through
to S3ErrorCode::InternalError (500) in the error mapping.

Only QuorumNotReached was correctly mapped to ServiceUnavailable (503);
all other lock errors (Timeout, AlreadyLocked, etc.) became 500.

Fix: map_namespace_lock_error now returns StorageError::Lock(err) for
non-quorum lock errors, and StorageError::Lock is mapped to
S3ErrorCode::ServiceUnavailable in the HTTP error conversion.

Affected paths:
- crates/ecstore/src/set_disk/lock.rs
- crates/ecstore/src/store/object.rs
- crates/ecstore/src/store/bucket.rs (make_bucket, delete_bucket)
- rustfs/src/app/object_usecase.rs (copy_object)

Regression test: test_concurrent_put_same_key_never_returns_500

* test: fail on unexpected lock contention errors

---------

Co-authored-by: houseme <housemecn@gmail.com>
2026-06-08 02:20:21 +00:00
houseme c479f3d0cb fix(ecstore): gate rustix fs diagnostics on windows (#3267)
* fix(ecstore): gate rustix fs diagnostics on windows

* fix(ecstore): improve windows disk validation diagnostics

* build(deps): bump workspace dependency versions
2026-06-08 00:05:55 +00:00
Henry Guo c452dd8ad7 feat(table-catalog): add metadata maintenance cleanup (#3266)
* feat(table-catalog): add metadata maintenance dry run

* feat(table-catalog): add metadata-only maintenance delete

* fix(table-catalog): protect recent metadata cleanup candidates

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-08 00:05:41 +00:00
安正超 7a9bf707ee feat: add security governance contract types (#3270) 2026-06-07 23:14:11 +00:00
安正超 c03c0ebc36 test: add admin route matrix guard (#3268) 2026-06-08 05:30:18 +08:00
安正超 dee550a831 ci: add architecture migration rule checks (#3264)
Co-authored-by: houseme <housemecn@gmail.com>
2026-06-07 16:23:38 +00:00
Henry Guo f49df41db9 fix(lifecycle): harden scanner ILM expiry accounting (#3257)
* fix(lifecycle): harden scanner ILM expiry accounting

* fix(scanner): gate ILM action accounting on enqueue

* fix(metrics): avoid scanner source work argument list

* fix(scanner): gate local ILM accounting on enqueue

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
Co-authored-by: houseme <housemecn@gmail.com>
2026-06-07 16:18:34 +00:00
houseme cebea9a238 fix(ecstore): add disk validation diagnostics (#3265)
Log local endpoint disk-topology details during physical disk independence checks.
Include canonical paths, st_dev major:minor values, and resolved device ids in validation output to make Docker and bind-mount startup failures easier to diagnose.

Also assert the new diagnostics are present in the shared-disk regression test.
2026-06-07 15:32:38 +00:00
唐小鸭 74296761fa fix(replication): repair site replication setup (#3252)
* fix: repair site replication setup

* Optimized site status collection

---------

Co-authored-by: cxymds <Cxymds@qq.com>
2026-06-07 13:19:50 +00:00
安正超 6f4d0b54a1 fix(ci): install ripgrep for script checks (#3260) 2026-06-07 22:03:25 +08:00
安正超 4254633c9e docs: update security advisory lessons (#3262) 2026-06-07 22:02:56 +08:00
安正超 e5b2bcc088 docs: complete config helper inventory (#3261) 2026-06-07 22:02:21 +08:00
Henry Guo dd0ef1f766 feat(table-catalog): tighten REST load/register compatibility (#3245)
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: cxymds <Cxymds@qq.com>
2026-06-07 21:54:15 +08:00
安正超 3c71d5ef1c docs: inventory ecstore config consumers (#3259) 2026-06-07 21:44:55 +08:00
安正超 241e45d3b0 docs: add admin route action snapshot (#3258) 2026-06-07 21:20:16 +08:00
安正超 0f9584c8d9 docs: add startup timeline baseline (#3256) 2026-06-07 20:18:30 +08:00
安正超 ae9d25879d ci: stabilize architecture layer guard (#3255) 2026-06-07 19:26:35 +08:00
Marcelo Bartsch f00898d070 fix(tier): recover (#3182)
* fix(tier): stop sending nil/garbage versionId to warm backend S3

Three bugs caused NoSuchVersion errors when reading tiered objects:

1. warm_backend_s3sdk: GET and DELETE ignored rv/range opts entirely —
   fixed to forward version_id and byte-range to the SDK request.

2. version.rs (MetaObject + MetaDeleteMarker): transition_version_id was
   parsed with unwrap_or_default(), turning invalid/wrong-length bytes
   into Uuid::nil(). The nil UUID was then serialized and sent as
   ?versionId=00000000-... to the tier backend -> NoSuchVersion.
   Fixed: .and_then(.ok()).filter(!is_nil()) so only valid non-nil UUIDs
   are forwarded as versionId.

3. bucket_lifecycle_ops: add debug/error logs in
   get_transitioned_object_reader to record tier, tier_object, and
   tier_version_id before and on failure of the tier GET.

Also adds tier transition fields to dump_fileinfo example for offline
xl.meta inspection, and fixes Docker build (cargo path + entrypoint).
Adds CLAUDE.md with tier architecture and debugging notes.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* more fixes for versionId

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Signed-off-by: Marcelo Bartsch <marcelo@bartsch.cl>

* remove branch

* Add tests and fix cargo path, add load to build-docker

* update documentation (CLAUDE.md)

* more fixes for recover

* More fixes to ILM recover

* final fix

* chore: add missing-shard first-scene diagnostics (#3213)

chore(ecstore): add missing-shard first-scene diagnostics

Log rename_data quorum context behind RUSTFS_ISSUE3031_DIAG_ENABLE so partial-disk success can be correlated with later missing shard reads.

Also log put_object commit success and tmp cleanup boundaries to capture when successful quorum writes are followed by tmp_dir cleanup.

* fix test anmd fmt

* fix cargo path
fix test

* fix(tier): format copy_object self-copy guard

---------

Signed-off-by: Marcelo Bartsch <marcelo@bartsch.cl>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: loverustfs <hello@rustfs.com>
2026-06-07 09:38:28 +00:00
安正超 069d1e5a75 test: bypass proxy for embedded readiness probe (#3254) 2026-06-07 09:04:29 +00:00
Henry Guo 4ae070c8cc feat(scanner): expose scan partial source status (#3247)
* feat(scanner): expose scan partial source status

* fix(scanner): expose partial source in aggregated metrics

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
Co-authored-by: cxymds <Cxymds@qq.com>
Co-authored-by: houseme <housemecn@gmail.com>
2026-06-07 06:40:04 +00:00
安正超 f8aa4fa221 docs: add architecture migration guardrails (#3253) 2026-06-07 14:43:55 +08:00
cxymds 61f0dfbc40 fix(ecstore): invalidate wiped disk id cache (#3251) 2026-06-07 04:25:38 +00:00
cxymds 04b5e8f988 fix(replication): normalize local site endpoint port (#3249)
* fix(replication): normalize local site endpoint port

* fix(replication): respect runtime console endpoint

* fix(replication): allow valid peer endpoint ports

---------

Co-authored-by: loverustfs <hello@rustfs.com>
Co-authored-by: houseme <housemecn@gmail.com>
2026-06-07 04:19:04 +00:00
cxymds 316540aaf8 fix(docker): align TLS compose healthchecks (#3246)
* fix(docker): align TLS compose healthchecks

* fix(docker): keep simple compose TLS opt-in
2026-06-07 01:27:03 +00:00
cxymds dd5e2c8ae9 docs(docker): clarify bind mount permission setup (#3248)
Co-authored-by: houseme <housemecn@gmail.com>
2026-06-07 09:33:24 +08:00
houseme e73b1c11c0 chore(deps): update flake.lock (#3250)
Flake lock file updates:

• Updated input 'nixpkgs':
    'github:NixOS/nixpkgs/e9a7635' (2026-05-29)
  → 'github:NixOS/nixpkgs/cbb5cf3' (2026-06-06)
• Updated input 'rust-overlay':
    'github:oxalica/rust-overlay/85570ef' (2026-05-30)
  → 'github:oxalica/rust-overlay/27b7e78' (2026-06-06)
2026-06-07 01:07:23 +00:00
GatewayJ 4a28e3d671 fix: clean old data dirs on object overwrite (#3244)
Co-authored-by: cxymds <Cxymds@qq.com>
2026-06-07 00:18:56 +00:00
GatewayJ 8c742ede14 fix(admin): format policy JSON and improve error messages in service … (#3242)
* fix(admin): format policy JSON and improve error messages in service account API

- Use serde_json::to_string_pretty for policy serialization to match MinIO behavior
- Replace technical error messages with user-friendly ones for policy validation
- Fixes rustfs/rustfs#3233 (policy JSON not formatted)
- Fixes rustfs/rustfs#3232 (error messages not shown to users)

* Delete scripts/tempfile.zip

---------

Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
2026-06-06 16:39:59 +00:00
Henry Guo 7d84e89c36 feat(table-catalog): wire REST commit handler (#3239)
* feat(table-catalog): wire REST commit handler

* fix(table-catalog): hide MVP commit endpoint from config

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-06 13:38:12 +00:00
安正超 e2fe99d8bc refactor(ecstore): add server configuration accessors (#3238)
* refactor(ecstore): add server configuration accessors

Phase 4 of global singleton consolidation. Add ECStore accessor
methods for server-level configuration globals.

New methods:
- port() — delegates to global_rustfs_port()
- host() — delegates to GLOBAL_RUSTFS_HOST
- addr() — delegates to GLOBAL_RUSTFS_ADDR
- region() — delegates to get_global_region()
- endpoints() — delegates to get_global_endpoints()
- server_config() — delegates to get_global_server_config()
- storage_class() — delegates to get_global_storage_class()
- tier_config_mgr() — delegates to get_global_tier_config_mgr()
- notification_system() — delegates to get_global_notification_sys()
- bucket_metadata_sys() — delegates to get_global_bucket_metadata_sys()

1160 tests pass, clippy clean, formatting clean.

* fix(ecstore): avoid duplicate server accessors
2026-06-06 21:38:45 +08:00
Henry Guo 83a4e5712e feat(scanner): track scan cycle source work (#3240)
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-06 12:53:48 +00:00
Henry Guo fade13d950 feat(table-catalog): wire REST catalog MVP handlers (#3227)
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-06 10:47:49 +00:00
Henry Guo b36a730e48 feat(scanner): expose checkpoint and source work status (#3230)
* feat(scanner): expose checkpoint and source work status

* fix(scanner): count ignored checkpoints once per scan

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-06 10:47:39 +00:00
houseme 6db4c3538d fix(allocator): restore validated jemalloc target gating (#3236)
* fix(allocator): restore validated jemalloc target gating

Restrict the global allocator and jemalloc profiling paths to linux-gnu-x86_64 so Linux ARM64 builds fall back to mimalloc again.

Update the runtime profiling, allocator reclaim, and admin profiling handlers to use the same target gating and avoid exposing jemalloc-only code on unsupported targets.

Verification:
- cargo fmt --all
- cargo fmt --all --check
- cargo check -p rustfs
- make pre-commit

* fix(profiling): restore non-jemalloc platform behavior

Restore CPU profiling support on macOS while keeping jemalloc-backed memory profiling restricted to validated linux-gnu-x86_64 targets.

Also restore Unix log directory permission hardening so the allocator regression fix does not roll back unrelated observability behavior on other supported platforms.

Verification in progress:
- cargo fmt --all
- cargo check -p rustfs
- make pre-commit (running)

* fix(profiling): qualify periodic memory log macros

Use explicit tracing macro paths in periodic jemalloc memory profiling logging so builds do not fail when the local target configuration excludes those branches from the current import set.

Verification:
- cargo fmt --all
- cargo check -p rustfs

* fix(profiling): gate unsupported memory helper

Restrict the unsupported memory profiling helper to non-linux-gnu-x86_64 targets so Linux builds do not emit dead_code warnings for an unreachable fallback.

Verification:
- cargo fmt --all
- cargo check -p rustfs
2026-06-06 08:22:29 +00:00
安正超 60f14cb0f1 fix(bucket-encryption): populate default KMS key for SSE-KMS without key ID (#3225)
* fix(bucket-encryption): populate default KMS key for SSE-KMS without key ID

When PutBucketEncryption receives SSE-KMS configuration without a specific
KMS key ID, query the KMS service for the default key and populate
KMSMasterKeyID before storing. This ensures GetBucketEncryption responses
include the key ID, which S3 clients like mc rely on to distinguish SSE-KMS
from SSE-S3 in their display logic.

Fixes #3039

* test(kms): cover empty bucket encryption key ID

* fix(bucket-encryption): require default KMS key
2026-06-06 02:57:48 +00:00
安正超 06acf49672 refactor(ecstore): add accessor methods for service globals (#3226)
* refactor(ecstore): add accessor methods for service globals

Phase 3 of global singleton consolidation. Add ECStore accessor
methods that delegate to process-global service singletons,
providing a unified API through ECStore.

New methods:
- notification_system() — delegates to notification_sys global
- bucket_metadata_sys() — delegates to bucket metadata global
- endpoints() — delegates to endpoints global
- region() — delegates to region global
- tier_config_mgr() — delegates to tier config global
- server_config() — delegates to server config global
- storage_class() — delegates to storage class global

1161 tests pass, clippy clean, formatting clean.

* fix: use imported functions instead of fully-qualified paths

Address review comments: use imported functions instead of
crate:: paths to avoid unused import warnings.

1160 tests pass, clippy clean.

* fix(ecstore): remove unused service imports
2026-06-05 15:29:38 +00:00
Henry Guo 6d06b574f2 feat(table-catalog): add REST catalog route surface (#3211)
* feat(table-catalog): add REST catalog route surface

* fix(table-catalog): avoid test-only router import

* fix(table-catalog): align unsupported REST responses

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-05 13:52:13 +00:00
安正超 4d50c72e4c fix(ecstore): skip hidden metadata in walk limit (#3224)
* fix(ecstore): skip hidden metadata in walk limit

* fix(ecstore): match walk limit to visible versions

* fix(ecstore): avoid decoding all versions for limit
2026-06-05 13:50:02 +00:00
Henry Guo 0d445afda9 feat(scanner): add versioned scan checkpoints (#3220)
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-05 13:49:41 +00:00
GatewayJ f1b8f8312d fix(server): normalize empty request content length (#3215)
* fix(server): normalize empty S3 request content length

* fix(server): normalize empty console content length

* fix(server): keep console constants test scoped

* test(server): align empty body route expectations
2026-06-05 02:03:07 +00:00
安正超 8185376b90 refactor(ecstore): migrate config globals into ECStore struct fields (#3219)
* refactor(ecstore): migrate config globals into ECStore struct fields

Phase 2 of global singleton consolidation. Add server_config and
storage_class fields to ECStore, sharing the same underlying data
as the process-global LazyLock statics.

New ECStore fields:
- server_config: RwLock<Option<Config>>
- storage_class: RwLock<storageclass::Config>

New async accessor methods:
- get_server_config() / set_server_config()
- get_storage_class() / set_storage_class()

Fields initialized with defaults in ECStore::new(), synced from
globals after init() completes. Global functions preserved for
backward compatibility.

1160 tests pass, clippy clean, full workspace compiles.

* fix: address PR #3219 review comments - delegate to globals

Accessors now delegate to process-global statics to avoid state drift:
- get_server_config() delegates to get_global_server_config()
- set_server_config() updates both global and local field
- get_storage_class() delegates to get_global_storage_class()
- set_storage_class() updates both global and local field
- Removed stale sync code from init()
- Changed fields to std::sync::RwLock for sync access

1160 tests pass.

* fix: correct import order for cargo fmt

* fix: simplify config accessors - delegate to globals without local state

Remove local server_config and storage_class fields from ECStore.
Accessors now purely delegate to process-global statics, eliminating
the state drift risk identified in review.

1160 tests pass, formatting clean.
2026-06-05 01:06:08 +00:00
安正超 1dd3839a9f fix(signer): address post-merge review comments (#3216)
* fix: address PR #3150 review comments

- Restore get_host_addr as best-effort wrapper (String return type)
- Replace bare expect("err") with descriptive messages in unsigned trailer
- Simplify aws-chunked header construction
- 20 signer + 96 io-core tests pass

* fix(signer): preserve host fallback

* fix(signer): avoid signer fallback panics

---------

Co-authored-by: houseme <housemecn@gmail.com>
2026-06-04 22:45:50 +00:00
安正超 d6120f5788 refactor(ecstore): migrate mutable globals into ECStore struct fields (#3214)
* refactor(ecstore): migrate mutable globals into ECStore struct fields

Phase 1 of global singleton consolidation. Move mutable globals from
lazy_static into ECStore struct fields as the first step toward
dependency injection and multi-instance support.

New ECStore fields:
- is_erasure, is_dist_erasure, is_erasure_sd (erasure type flags)
- local_disk_map, local_disk_id_map, local_disk_set_drives
- root_disk_threshold
- tier_config_mgr, event_notifier, bucket_monitor

New accessor methods:
- is_erasure(), is_dist_erasure(), is_erasure_sd()
- update_erasure_type()
- tier_config_mgr(), event_notifier(), bucket_monitor()

Global functions in global.rs preserved for backward compatibility.
1151 ecstore tests pass.

* fix: address PR #3214 review comments

- Sync ECStore fields from globals after init()
- Enforce DistErasure => is_erasure invariant in update_erasure_type()
- Change bucket_monitor from Option to OnceLock for deferred initialization
- Restrict TypeLocalDiskSetDrives to pub(crate)
- 1151 tests pass

* fix: address PR #3150 review comments

- Restore get_host_addr as best-effort wrapper (String return type)
- Replace bare expect("err") with descriptive messages in unsigned trailer
- Simplify aws-chunked header construction
- 20 signer + 96 io-core tests pass

* fix(ecstore): format store imports

* fix(ecstore): keep migrated accessors in sync

* fix(signer): preserve host fallback and unsigned trailer errors

* fix(ecstore): defer migrated global accessors

* fix(signer): box unsigned trailer signing errors

* fix(ecstore): avoid lock awaits during sync

* fix(ecstore): narrow phase one globals

---------

Co-authored-by: houseme <housemecn@gmail.com>
2026-06-04 17:27:21 +00:00
houseme 0a74629c48 chore: add delete-objects lock batch diagnostics (#3218)
* chore(ecstore): add missing-shard first-scene diagnostics

Log rename_data quorum context behind RUSTFS_ISSUE3031_DIAG_ENABLE so partial-disk success can be correlated with later missing shard reads.

Also log put_object commit success and tmp cleanup boundaries to capture when successful quorum writes are followed by tmp_dir cleanup.

* chore(ecstore): add delete-objects lock batch diagnostics

Log DeleteObjects batch-lock request size, distributed lock quorum summary, and failed object keys behind RUSTFS_ISSUE3031_DIAG_ENABLE.

Include a Chinese operator checklist under docs/ that explains which logs to capture and how to interpret DeleteObjects lock_batch timeout incidents.

* chore: remove lock batch checklist from repo

Keep docs/issue-658-deleteobjects-lock-batch-checklist-zh.md only in the local checkout and drop it from the PR history.
2026-06-04 17:00:58 +00:00
安正超 3bd89944c2 perf(erasure): remove UUID from clone + increase encode inflight budget (#3212)
* perf(erasure): remove UUID from clone + increase encode inflight budget

Two targeted optimizations for the erasure encoding hot path:

1. Erasure::clone() no longer generates Uuid::new_v4() per clone.
   The _id field is unused in the hot path; reusing the original ID
   eliminates a CSPRNG call per block encode (100 calls for a 100MB
   object with 1MB blocks).

2. Default RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES raised from 8MB
   to 32MB. This increases the encode pipeline depth from ~5 to ~20
   blocks, allowing more read-ahead between the encoder and disk
   writer stages. The per-request memory bound is still controlled
   by the 8-block hard cap and the env var override.

3. Added encode_data_owned() utility method for zero-copy encoding
   when the caller already owns a heap buffer (Vec<u8> → BytesMut
   via Bytes::try_into_mut). Not used in the hot path yet but
   available for future callers.

All 1157 ecstore tests pass. Criterion micro-benchmarks show no
regression (< 2% variance). Single-machine warp E2E tests were
inconclusive due to high variance; a dedicated multi-disk test
environment is needed for reliable E2E comparison.

Ref: https://github.com/rustfs/backlog/issues/659

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

* chore: update Cargo.lock

* fix(erasure): align encode inflight cap and tests

---------

Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
2026-06-04 13:49:36 +00:00
houseme fde519910d chore: add missing-shard first-scene diagnostics (#3213)
chore(ecstore): add missing-shard first-scene diagnostics

Log rename_data quorum context behind RUSTFS_ISSUE3031_DIAG_ENABLE so partial-disk success can be correlated with later missing shard reads.

Also log put_object commit success and tmp cleanup boundaries to capture when successful quorum writes are followed by tmp_dir cleanup.
2026-06-04 10:35:33 +00:00
Henry Guo 542720a1f7 feat(scanner): add partial scan resume hints (#3207)
* feat(scanner): add partial scan resume hints

* test(scanner): cover clearing scan resume hints

* fix(scanner): apply resume hint to combined child order

---------

Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-04 09:21:30 +00:00
Henry Guo a7be7c558d feat(table-catalog): add object-backed catalog store (#3206)
Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
2026-06-04 07:50:13 +00:00
215 changed files with 36364 additions and 3406 deletions
@@ -53,6 +53,7 @@ For the full pattern map, read [advisory-patterns.md](references/advisory-patter
- Every admin or diagnostic route needs an explicit authn and authz story. Route registration, router whitelist, and handler-level authorization must agree.
- Match the admin action to the operation exactly. Copy-paste action constants are a known RustFS vulnerability class.
- Avoid authentication-only helpers for state-changing admin APIs; use `validate_admin_request` or the established equivalent with the right `AdminAction`.
- Read-only admin APIs such as metrics, server info, and diagnostics still require admin authorization; checking only that credentials exist is not enough.
- Do not assume admin-action `Resource` scoping constrains blast radius unless the policy engine actually enforces resources for that action.
### IAM and service accounts
@@ -20,6 +20,7 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
- `GHSA-mm2q-qcmx-gw4w`: `ListServiceAccount` used `UpdateServiceAccountAdminAction`, while update lacked target ownership checks. Lesson: exact action constants and ownership checks are both required; information disclosure can chain into secret rotation and takeover.
- `GHSA-vcwh-pff9-64cc`: `ImportIam` checked `ExportIAMAction` for an import/write operation. Lesson: every admin handler must authorize the action it actually performs.
- `GHSA-jqmc-mg33-v45g` and `GHSA-8784-9m7f-c6p6`: `/profile/cpu` and `/profile/memory` were whitelisted from auth and allowed expensive diagnostics plus path disclosure. Lesson: profiling/debug endpoints need admin auth, opt-in, rate limits, and non-sensitive responses.
- `GHSA-f5cv-v44x-2xgf`: `/rustfs/admin/v3/metrics` accepted any authenticated IAM user and skipped admin authorization. Lesson: read-only metrics and diagnostic admin endpoints still require an operation-specific admin action check.
- `GHSA-xp32-gxq2-3v52`: console license metadata endpoint was public and exposed subject and expiration fields. Lesson: management metadata endpoints should require admin auth or return only coarse public status.
### IAM import, service accounts, and privilege boundaries
+2 -2
View File
@@ -9,7 +9,7 @@ build-docker: SOURCE_BUILD_CONTAINER_NAME = rustfs-$(BUILD_OS)-build
build-docker: BUILD_CMD = /root/.cargo/bin/cargo build --release --bin rustfs --target-dir /root/s3-rustfs/target/$(BUILD_OS)
build-docker: ## Build using Docker container # e.g (make build-docker BUILD_OS=ubuntu22.04)
@echo "🐳 Building RustFS using Docker ($(BUILD_OS))..."
$(DOCKER_CLI) buildx build -t $(SOURCE_BUILD_IMAGE_NAME) -f $(DOCKERFILE_SOURCE) .
$(DOCKER_CLI) buildx build -t $(SOURCE_BUILD_IMAGE_NAME) -f $(DOCKERFILE_SOURCE) --load .
$(DOCKER_CLI) run --rm --name $(SOURCE_BUILD_CONTAINER_NAME) -v $(shell pwd):/root/s3-rustfs -it $(SOURCE_BUILD_IMAGE_NAME) $(BUILD_CMD)
.PHONY: docker-inspect-multiarch
@@ -19,4 +19,4 @@ docker-inspect-multiarch: ## Check image architecture support
exit 1; \
fi
@echo "🔍 Inspecting multi-architecture image: $(IMAGE)"
docker buildx imagetools inspect $(IMAGE)
docker buildx imagetools inspect $(IMAGE)
+5
View File
@@ -21,6 +21,11 @@ unsafe-code-check: ## Check unsafe_code allowances have SAFETY comments
@echo "🔒 Checking unsafe_code allowances..."
./scripts/check_unsafe_code_allowances.sh
.PHONY: architecture-migration-check
architecture-migration-check: ## Check architecture migration guardrails
@echo "🏗️ Checking architecture migration guardrails..."
./scripts/check_architecture_migration_rules.sh
.PHONY: compilation-check
compilation-check: core-deps ## Run compilation check
@echo "🔨 Running compilation check..."
+1 -1
View File
@@ -7,5 +7,5 @@ setup-hooks: ## Set up git hooks
@echo "✅ Git hooks setup complete!"
.PHONY: pre-commit
pre-commit: fmt unsafe-code-check clippy-check compilation-check test ## Run pre-commit checks
pre-commit: fmt unsafe-code-check architecture-migration-check clippy-check compilation-check test ## Run pre-commit checks
@echo "✅ All pre-commit checks passed!"
@@ -9627,6 +9627,416 @@
],
"title": "Cluster Config / API Rejected (All)",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 262
},
"id": 507,
"panels": [],
"title": "Internode Reliability",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": true,
"axisColorMode": "text",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "ops"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 263
},
"id": 508,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum by (operation, classification) (rate(rustfs_system_network_internode_operation_classified_errors_total{job=~\"$job\", operation=~\"put_file_stream|walk_dir|read_file_stream\"}[$__rate_interval]))",
"legendFormat": "{{operation}} | {{classification}}",
"range": true,
"refId": "A"
}
],
"title": "Internode Classified Errors",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": true,
"axisColorMode": "text",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "ops"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 263
},
"id": 509,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum by (operation, classification) (rate(rustfs_system_network_internode_operation_retries_total{job=~\"$job\", operation=\"put_file_stream\"}[$__rate_interval]))",
"legendFormat": "retry | {{classification}}",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum by (operation, classification) (rate(rustfs_system_network_internode_operation_retry_successes_total{job=~\"$job\", operation=\"put_file_stream\"}[$__rate_interval]))",
"legendFormat": "retry success | {{classification}}",
"range": true,
"refId": "B"
}
],
"title": "Internode Open Write Retries",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"max": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "yellow",
"value": 0.25
},
{
"color": "red",
"value": 0.5
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 271
},
"id": 510,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum by (classification) (rate(rustfs_system_network_internode_operation_retry_successes_total{job=~\"$job\", operation=\"put_file_stream\"}[$__rate_interval])) / clamp_min(sum by (classification) (rate(rustfs_system_network_internode_operation_retries_total{job=~\"$job\", operation=\"put_file_stream\"}[$__rate_interval])), 1e-9)",
"legendFormat": "{{classification}}",
"range": true,
"refId": "A"
}
],
"title": "Internode Retry Success Ratio",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": true,
"axisColorMode": "text",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "ops"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 271
},
"id": 511,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "desc"
}
},
"pluginVersion": "12.3.2",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum by (stage, dominant_error) (rate(rustfs_system_storage_erasure_write_quorum_failures_total{job=~\"$job\"}[$__rate_interval]))",
"legendFormat": "{{stage}} | {{dominant_error}}",
"range": true,
"refId": "A"
}
],
"title": "Erasure Write Quorum Failures",
"type": "timeseries"
}
],
"preload": false,
+5 -1
View File
@@ -32,5 +32,9 @@ Current `test-and-lint` gate includes:
- `cargo test -p e2e_test archive_multipart_roundtrip_preserves_bytes`
- `cargo test -p e2e_test presigned_get_and_reverse_proxy_preserve_multipart_bytes_with_fast_path`
- `cargo fmt --all --check`
- `cargo clippy --all-targets --all-features -- -D warnings`
- `cargo clippy --all-targets -- -D warnings`
- `cargo nextest run -p rustfs -p rustfs-ecstore --features rio-v2`
- `cargo test -p rustfs --doc --features rio-v2`
- `cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings`
- `./scripts/check_layer_dependencies.sh`
- `./scripts/check_architecture_migration_rules.sh`
+1
View File
@@ -54,6 +54,7 @@ runs:
build-essential \
pkg-config \
libssl-dev \
ripgrep \
unzip \
protobuf-compiler
@@ -0,0 +1,43 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Architecture Migration Rules
on:
pull_request:
branches: [ main ]
paths:
- "ARCHITECTURE.md"
- "docs/architecture/**"
- "scripts/check_architecture_migration_rules.sh"
- ".github/workflows/architecture-migration-rules.yml"
workflow_dispatch:
permissions:
contents: read
jobs:
architecture-migration-rules:
name: Architecture Migration Rules
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Install ripgrep
run: |
sudo apt-get update
sudo apt-get install -y ripgrep
- name: Check architecture migration rules
run: ./scripts/check_architecture_migration_rules.sh
+2 -2
View File
@@ -169,7 +169,7 @@ jobs:
{"target_id":"linux-x86_64-gnu","os":"ubicloud-standard-2","target":"x86_64-unknown-linux-gnu","cross":false,"platform":"linux","rustflags":""},
{"target_id":"linux-aarch64-gnu","os":"ubicloud-standard-2","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":""},
{"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":""},
{"target_id":"macos-x86_64","os":"macos-15-intel","target":"x86_64-apple-darwin","cross":true,"platform":"macos","rustflags":""},
{"target_id":"macos-x86_64","os":"macos-26-intel","target":"x86_64-apple-darwin","cross":false,"platform":"macos","rustflags":""},
{"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":""}
]}'
@@ -202,7 +202,7 @@ jobs:
needs: [ build-check, prepare-platform-matrix ]
if: needs.build-check.outputs.should_build == 'true' && needs.prepare-platform-matrix.result == 'success'
runs-on: ${{ matrix.os }}
timeout-minutes: 60
timeout-minutes: 90
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
# Always enable Tokio unstable features (required by dial9-tokio-telemetry).
+114 -3
View File
@@ -54,7 +54,7 @@ on:
- ".github/workflows/audit.yml"
- ".github/workflows/performance.yml"
merge_group:
types: [checks_requested]
types: [ checks_requested ]
schedule:
- cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
workflow_dispatch:
@@ -134,11 +134,42 @@ jobs:
run: ./scripts/check_unsafe_code_allowances.sh
- name: Run clippy lints
run: cargo clippy --all-targets --all-features -- -D warnings
run: cargo clippy --all-targets -- -D warnings
- name: Check layered dependencies
run: ./scripts/check_layer_dependencies.sh
- name: Check architecture migration rules
run: ./scripts/check_architecture_migration_rules.sh
test-and-lint-rio-v2:
name: Test and Lint (rio-v2)
needs: skip-check
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubicloud-standard-4
timeout-minutes: 60
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-test-rio-v2-${{ hashFiles('**/Cargo.lock') }}
github-token: ${{ secrets.GITHUB_TOKEN }}
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Run rio-v2 feature tests
run: |
cargo nextest run -p rustfs -p rustfs-ecstore --features rio-v2
cargo test -p rustfs --doc --features rio-v2
- name: Run rio-v2 clippy lints
run: cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings
build-rustfs-debug-binary:
name: Build RustFS Debug Binary
needs: skip-check
@@ -172,6 +203,39 @@ jobs:
if-no-files-found: error
retention-days: 1
build-rustfs-debug-binary-rio-v2:
name: Build RustFS Debug Binary (rio-v2)
needs: skip-check
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubicloud-standard-4
timeout-minutes: 30
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-rustfs-debug-binary-rio-v2-${{ hashFiles('**/Cargo.lock') }}
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Build debug binary with rio-v2
run: |
touch rustfs/build.rs
cargo build -p rustfs --bins --features rio-v2 --jobs 2
- name: Upload debug binary
uses: actions/upload-artifact@v6
with:
name: rustfs-debug-binary-rio-v2
path: target/debug/rustfs
if-no-files-found: error
retention-days: 1
e2e-tests:
name: End-to-End Tests
needs: [ skip-check, build-rustfs-debug-binary ]
@@ -219,9 +283,56 @@ jobs:
path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
retention-days: 3
e2e-tests-rio-v2:
name: End-to-End Tests (rio-v2)
needs: [ skip-check, build-rustfs-debug-binary-rio-v2 ]
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubicloud-standard-2
timeout-minutes: 30
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Clean up previous test run
run: |
rm -rf /tmp/rustfs
rm -f /tmp/rustfs.log
- name: Download debug binary
uses: actions/download-artifact@v7
with:
name: rustfs-debug-binary-rio-v2
path: target/debug
- name: Make binary executable
run: chmod +x ./target/debug/rustfs
- name: Setup Rust toolchain for s3s-e2e installation
uses: dtolnay/rust-toolchain@stable
- name: Install s3s-e2e test tool
uses: taiki-e/cache-cargo-install-action@v2
with:
tool: s3s-e2e
git: https://github.com/s3s-project/s3s.git
rev: 62cb4a71dd759a6ec56b64c4c42fcc183a2c6a52
- name: Run end-to-end tests
run: |
s3s-e2e --version
./scripts/e2e-run.sh ./target/debug/rustfs /tmp/rustfs
- name: Upload test logs
if: failure()
uses: actions/upload-artifact@v6
with:
name: e2e-test-logs-rio-v2-${{ github.run_number }}
path: /tmp/rustfs.log
retention-days: 3
s3-implemented-tests:
name: S3 Implemented Tests
needs: [ skip-check, build-rustfs-debug-binary ]
needs: [ skip-check, build-rustfs-debug-binary, e2e-tests ]
if: needs.skip-check.outputs.should_skip != 'true'
runs-on: ubicloud-standard-4
timeout-minutes: 60
+4 -1
View File
@@ -48,6 +48,8 @@ jobs:
permissions:
contents: read
id-token: write
env:
NIX_CURL_FLAGS: -A cargo/stable
steps:
- name: Checkout repository
uses: actions/checkout@v6
@@ -57,7 +59,8 @@ jobs:
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
extra-conf: |
experimental-features = nix-command flakes
experimental-features = nix-command flakes configurable-impure-env
impure-env = NIX_CURL_FLAGS
max-jobs = 1
- name: Cache Nix
+6
View File
@@ -42,6 +42,11 @@ artifacts/
PR_DESCRIPTION.md
scripts/s3-tests/selected_tests.txt
docs
__pycache__/
!docs/
docs/*
!docs/architecture/
!docs/architecture/**
# nix stuff
result*
@@ -53,3 +58,4 @@ rustfs-webdav.code-workspace
# Local test/benchmark artifacts
benchmarks.logs
tmp/
crates/*/docs
+42 -1
View File
@@ -224,6 +224,47 @@
"rust"
],
},
{
"type": "lldb",
"request": "launch",
"name": "Debug executable 'rustfs' with rio-v2 full matrix",
"cargo": {
"args": [
"build",
"--bin=rustfs",
"--package=rustfs",
"--features=full,rio-v2"
],
"filter": {
"name": "rustfs",
"kind": "bin"
}
},
"env": {
"RUST_LOG": "rustfs=error,ecstore=error,s3s=error,iam=error",
"RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS": "true",
"RUST_BACKTRACE": "full",
"RUSTFS_ACCESS_KEY": "rustfsadmin",
"RUSTFS_SECRET_KEY": "rustfsadmin",
"RUSTFS_VOLUMES": "./target/rio-v2/test{1...4}",
"RUSTFS_SCANNER_ENABLED": "false",
"RUSTFS_HEAL_ENABLED": "false",
"RUSTFS_REGION": "us-east-1",
"RUSTFS_ADDRESS": ":9000",
"RUSTFS_CONSOLE_ENABLE": "true",
"RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001",
"RUSTFS_OBS_LOG_DIRECTORY": "./target/logs/rio-v2",
"RUSTFS_UNSAFE_BYPASS_DISK_CHECK": "true",
"RUSTFS_COMPRESSION_ENABLED": "true",
"RUSTFS_KMS_ENABLE": "true",
"RUSTFS_KMS_BACKEND": "local",
"RUSTFS_KMS_KEY_DIR": "./target/rio-v2/kms-key-dir",
"RUSTFS_KMS_LOCAL_MASTER_KEY": "my-secret-key", // Some Password
"RUSTFS_KMS_DEFAULT_KEY_ID": "rustfs-master-key",
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"name": "E2E test executable target/debug/rustfs",
"type": "lldb",
@@ -256,4 +297,4 @@
],
},
]
}
}
+8
View File
@@ -103,6 +103,14 @@ Do not open a PR with code changes when the required checks fail.
- Use environment variables or vault tooling for sensitive configuration.
- For localhost-sensitive tests, verify proxy settings to avoid traffic leakage.
## Tools
### xl.meta decode tool Quick Use
```
cargo run -p rustfs-filemeta --example dump_fileinfo -- "/path/to/file/xl.meta"
```
## Serde Safety
- Add `#[serde(deny_unknown_fields)]` to structs deserialized from untrusted input (S3 API XML/JSON, lifecycle rules, bucket policies, replication configs).
+2
View File
@@ -6,6 +6,8 @@
> If you want to familiarize yourself with the code base, you are in the right place!
>
> See also [CONTRIBUTING.md](CONTRIBUTING.md) for development workflow.
> See also [docs/architecture](docs/architecture/overview.md) for active
> architecture migration guardrails.
## Bird's Eye View
+188
View File
@@ -0,0 +1,188 @@
# RustFS — CLAUDE.md
S3-compatible object store in Rust, derived from MinIO. Erasure-coded, multi-pool, supports ILM tiering/lifecycle.
## Commands
```bash
cargo build --release --bin rustfs # production binary
cargo build # dev build
cargo check -p <crate> # fast type-check one crate
cargo test -p <crate> # test one crate
cargo fmt --all # format (required before PR)
make pre-commit # full pre-PR gate (fmt + clippy + test)
make build-docker BUILD_OS=ubuntu22.04 # Docker cross-build
```
> **Docker build note**: `buildx build` without `--load` keeps the image in the buildx cache only — `docker run` will use a stale local image. The Makefile already includes `--load`; if you suspect a stale binary, add `--no-cache` to the `buildx build` invocation inside `.config/make/build-docker.mak`.
> Agent/PR rules: see `.github/copilot-instructions.md`.
> Crate membership: `Cargo.toml` `[workspace].members`.
> CI gates: `.github/workflows/ci.yml`.
## Workspace layout
```
rustfs/src/main.rs # binary entry point
crates/ecstore/src/
set_disk.rs # ErasureSet: transition_object, restore_transitioned_object
store.rs / store_api/ # ECStore trait + ObjectInfo / TransitionedObject types
bucket/lifecycle/
bucket_lifecycle_ops.rs # ILM actions: transition_object, expire_transitioned_object,
# get_transitioned_object_reader, gen_transition_objname
tier_sweeper.rs # background sweep: delete_object_from_remote_tier
tier/
warm_backend.rs # WarmBackend trait (put/get/remove/in_use)
warm_backend_s3.rs # HTTP-client based (TransitionClient) — used for S3/MinIO
warm_backend_s3sdk.rs # aws-sdk-s3 based — alternative S3 backend
warm_backend_minio.rs / _rustfs.rs / … # per-provider wrappers (all delegate to _s3 or _s3sdk)
tier.rs # TierConfigMgr, new_warm_backend dispatch
client/transition_api.rs # TransitionClient HTTP plumbing; UploadInfo, to_object_info
client/api_put_object_streaming.rs # put_object_do → UploadInfo (version_id from x-amz-version-id)
crates/filemeta/src/
filemeta.rs # FileMeta (xl.meta top-level), is_skip_meta_key
filemeta/version.rs # FileMetaVersion, MetaObject, MetaDeleteMarker
# → to_fileinfo() reads transition_version_id
# → set_transition() writes raw UUID bytes
# → From<FileInfo> for MetaObject writes all meta
fileinfo.rs # FileInfo struct (transition_version_id: Option<Uuid>)
examples/
dump_fileinfo.rs # CLI: parse xl.meta, print transition fields + metadata
dump_versions.rs # CLI: list all versions in xl.meta
crates/utils/src/http/metadata_compat.rs # SUFFIX_* constants, insert_bytes/get_bytes (dual RustFS+MinIO keys)
```
## Metadata key conventions
Internal metadata is stored under **both** `x-rustfs-internal-<suffix>` and `x-minio-internal-<suffix>` for MinIO interoperability. `get_bytes` prefers the RustFS key with MinIO fallback.
Key suffixes (from `metadata_compat.rs`):
| Suffix | Meaning |
|--------|---------|
| `transition-status` | `"complete"` when tiered |
| `transitioned-object` | tier key path (without prefix) |
| `transitioned-versionID` | S3 version_id returned by tier PUT (16 raw UUID bytes, or absent) |
| `transition-tier` | tier name |
| `tier-free-versionID` | delete-marker version for free-version sweep |
## Tier / ILM transition architecture
### Transition flow (hot → cold)
1. `transition_object` (lifecycle_ops) → `ECStore::transition_object``set_disk.rs`
2. `gen_transition_objname(bucket)``{sha256_hash[0..16]}/{uuid[0..2]}/{uuid[2..4]}/{uuid}` (unique per object version)
3. `tgt_client.put_with_meta(dest_obj, …)` → returns `rv: String` (remote S3 version_id, or `""`)
4. `fi.transition_version_id = if rv.is_empty() { None } else { Some(Uuid::parse_str(&rv)?) }`
5. `fi.transitioned_objname = dest_obj` (without tier prefix)
6. Written to xl.meta via `MetaObject::from(FileInfo)``insert_bytes(SUFFIX_TRANSITIONED_VERSION_ID, uuid.as_bytes())` (16 raw bytes)
### Tier GET flow (restore/read)
`get_transitioned_object_reader` (lifecycle_ops):
- reads `oi.transitioned_object.name` (= `fi.transitioned_objname`)
- reads `oi.transitioned_object.version_id` (= `fi.transition_version_id.to_string()` or `""`)
- calls `warm_backend.get(name, version_id, opts)`
- `warm_backend_s3.rs::get`: adds `?versionId=…` only when `rv != ""`
### Tier prefix handling
`WarmBackendS3::get_dest(object)` prepends `self.prefix` to the object name.
`transitioned_objname` is stored **without** the prefix — `get_dest` adds it on every call.
### xl.meta on disk
Path: `{disk}/{bucket}/{object}/xl.meta` — one per erasure shard disk.
All shards should be identical for a healthy object.
## Known bugs & fixes
### Bug 1: `NoSuchVersion` on tier GET — nil UUID sent as versionId
**Root cause**: `transitioned-versionID` metadata key exists with empty string value (0 bytes). Old reading code:
```rust
// OLD — unwrap_or_default() converts 0-byte or wrong-length slice to Uuid::nil()
get_bytes().map(|v| Uuid::from_slice(v.as_slice()).unwrap_or_default())
// → Some(Uuid::nil()) → sends ?versionId=00000000-… → NoSuchVersion
```
**Fix** (version.rs, `MetaObject::to_fileinfo` + `MetaDeleteMarker::to_fileinfo`):
```rust
get_bytes()
.and_then(|v| Uuid::from_slice(v.as_slice()).ok()) // None for wrong-length bytes
.filter(|u| !u.is_nil()) // None for nil UUID (old write-back)
```
**Regression tests** (`crates/filemeta/src/filemeta/version.rs` `mod tests`): 6 tests cover absent key, empty bytes, nil UUID, and valid UUID round-trip for both `MetaObject` and `MetaDeleteMarker` paths.
### Bug 2: `warm_backend_s3sdk.rs` ignored rv and range opts
**Fix**: added `req.version_id(rv)` and `req.range(…)` to GET; `req.version_id(rv)` to DELETE.
### Bug 4: `set_disk::copy_object` returns 501 for tiered objects (storage class restore)
**Root cause**: `set_disk::copy_object` immediately returns `StorageError::NotImplemented` when `src_info.metadata_only = false`. For tiered objects, `metadata_only` is never set to `true` (guarded by `transitioned_object.tier.is_empty()`). So `mc cp --storage-class STANDARD obj obj` on a tiered object always returns 501.
**Fix** (`crates/ecstore/src/set_disk.rs`, `copy_object`):
```rust
if !src_info.metadata_only {
if path_join_buf(&[src_bucket, src_object]) == path_join_buf(&[dst_bucket, dst_object]) {
if let Some(mut put_reader) = src_info.put_object_reader.take() {
return self.put_object(dst_bucket, dst_object, &mut put_reader, dst_opts).await;
}
}
return Err(StorageError::NotImplemented);
}
```
When a self-copy has a `put_object_reader` (data already fetched from tier in `execute_copy_object`), writes it back locally via `put_object`, effectively de-tiering the object.
**How `mc cp --storage-class STANDARD` flows**:
1. mc sends `PUT /bucket/key` with `x-amz-copy-source`, `x-amz-metadata-directive: REPLACE`, `x-amz-storage-class: STANDARD`
2. `execute_copy_object``get_object_reader` fetches data from tier backend → stores in `src_info.put_object_reader`
3. `store.copy_object(...)` → now calls `put_object` with tier data and STANDARD storage class in `dst_opts`
4. New xl.meta written locally with STANDARD class, no tier metadata → object de-tiered
### Bug 3 (open): race in `expire_transitioned_object`
Order is: delete remote tier version → delete local object.
A concurrent GET between those two steps fetches a valid stored version_id but the tier version is already gone → `NoSuchVersion`.
The proper fix is to delete local metadata first (making the object unreachable) before deleting the remote tier version.
## Debugging tier issues
### Inspect xl.meta directly
```bash
cargo build -p rustfs-filemeta --example dump_fileinfo
./target/debug/examples/dump_fileinfo /srv/rustfs/data/disk0/{bucket}/{object}/xl.meta
# Shows: transition_status, transition_tier, transitioned_obj, transition_ver_id
```
`transition_ver_id: <none>` → no versionId will be sent to tier (correct for non-versioned tier bucket).
`transition_ver_id: <uuid>` → that UUID will be sent as `?versionId=<uuid>`.
### Check what versionId is being sent at runtime
Enable debug logging:
```bash
RUST_LOG=rustfs_ecstore::bucket::lifecycle=debug rustfs …
```
Log line: `fetching transitioned object from tier` (DEBUG before request).
Log line: `tier GET failed` (ERROR on failure, includes `tier_version_id`).
### Metadata key to watch
```
x-minio-internal-transitioned-versionID= ← empty string = will cause NoSuchVersion with old code
x-rustfs-internal-transitioned-versionID= ← same
```
If both are empty string, the object was transitioned to a non-versioned tier bucket. The versionId should NOT be sent — fixed by Bug 1 above.
## Common patterns
### Writing internal metadata (binary values)
```rust
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, uuid.as_bytes().to_vec());
// stores under both x-rustfs-internal-* and x-minio-internal-* keys
```
### Reading internal metadata (binary values)
```rust
get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID)
.and_then(|v| Uuid::from_slice(v.as_slice()).ok())
.filter(|u| !u.is_nil())
// Returns None for: absent, wrong-length bytes, nil UUID
```
### WarmBackend trait
```rust
put_with_meta(object, reader, length, meta) -> Result<String> // returns S3 version_id or ""
put(object, reader, length) -> Result<String>
get(object, rv, opts) -> Result<ReadCloser> // rv="" means no versionId
remove(object, rv) -> Result<()>
in_use() -> Result<bool>
```
`rv` = remote version, always pass as empty string when `transition_version_id` is None.
Generated
+457 -458
View File
File diff suppressed because it is too large Load Diff
+57 -51
View File
@@ -38,13 +38,16 @@ members = [
"crates/protocols", # Protocol implementations (FTPS, SFTP, etc.)
"crates/protos", # Protocol buffer definitions
"crates/rio", # Rust I/O utilities and abstractions
"crates/rio-v2", # Next-generation Rust I/O compatibility layer
"crates/concurrency", # Concurrency management for RustFS - timeout, locking, backpressure, and I/O scheduling
"crates/s3-types", # S3 event type definitions
"crates/s3-ops", # S3 operation definitions and mapping
"crates/s3select-api", # S3 Select API interface
"crates/s3select-query", # S3 Select query engine
"crates/scanner", # Scanner for data integrity checks and health monitoring
"crates/security-governance", # Security governance contracts
"crates/signer", # client signer
"crates/storage-api", # Storage API contracts
"crates/targets", # Target-specific configurations and utilities
"crates/trusted-proxies", # Trusted proxies management
"crates/tls-runtime", # Project-wide TLS runtime foundation
@@ -59,8 +62,8 @@ resolver = "3"
edition = "2024"
license = "Apache-2.0"
repository = "https://github.com/rustfs/rustfs"
rust-version = "1.95.0"
version = "1.0.0-beta.7"
rust-version = "1.96.0"
version = "1.0.0-beta.8"
homepage = "https://rustfs.com"
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -77,43 +80,46 @@ redundant_clone = "warn"
[workspace.dependencies]
# RustFS Internal Crates
rustfs = { path = "./rustfs", version = "1.0.0-beta.7" }
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.7" }
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.7" }
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.7" }
rustfs-common = { path = "crates/common", version = "1.0.0-beta.7" }
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.7" }
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.7" }
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.7" }
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.7" }
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.7" }
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.7" }
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.7" }
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.7" }
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.7" }
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.7" }
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.7" }
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.7" }
rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.7" }
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.7" }
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.7" }
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.7" }
rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.7" }
rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.7" }
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.7" }
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.7" }
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.7" }
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.7" }
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.7" }
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.7" }
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.7" }
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.7" }
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.7" }
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.7" }
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.7" }
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.7" }
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.7" }
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.7" }
rustfs = { path = "./rustfs", version = "1.0.0-beta.8" }
rustfs-heal = { path = "crates/heal", version = "1.0.0-beta.8" }
rustfs-audit = { path = "crates/audit", version = "1.0.0-beta.8" }
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-beta.8" }
rustfs-common = { path = "crates/common", version = "1.0.0-beta.8" }
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-beta.8" }
rustfs-config = { path = "./crates/config", version = "1.0.0-beta.8" }
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-beta.8" }
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-beta.8" }
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-beta.8" }
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-beta.8" }
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-beta.8" }
rustfs-iam = { path = "crates/iam", version = "1.0.0-beta.8" }
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-beta.8" }
rustfs-kms = { path = "crates/kms", version = "1.0.0-beta.8" }
rustfs-lock = { path = "crates/lock", version = "1.0.0-beta.8" }
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-beta.8" }
rustfs-notify = { path = "crates/notify", version = "1.0.0-beta.8" }
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-beta.8" }
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-beta.8" }
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-beta.8" }
rustfs-obs = { path = "crates/obs", version = "1.0.0-beta.8" }
rustfs-policy = { path = "crates/policy", version = "1.0.0-beta.8" }
rustfs-protos = { path = "crates/protos", version = "1.0.0-beta.8" }
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-beta.8" }
rustfs-rio = { path = "crates/rio", version = "1.0.0-beta.8" }
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-beta.8" }
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-beta.8" }
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-beta.8" }
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-beta.8" }
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-beta.8" }
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-beta.8" }
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-beta.8" }
rustfs-signer = { path = "crates/signer", version = "1.0.0-beta.8" }
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-beta.8" }
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-beta.8" }
rustfs-targets = { path = "crates/targets", version = "1.0.0-beta.8" }
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-beta.8" }
rustfs-utils = { path = "crates/utils", version = "1.0.0-beta.8" }
rustfs-zip = { path = "./crates/zip", version = "1.0.0-beta.8" }
# Async Runtime and Networking
async-channel = "2.5.0"
@@ -121,7 +127,7 @@ mysql_async = { version = "0.37", default-features = false, features = ["default
async-compression = { version = "0.4.42" }
async-recursion = "1.1.1"
async-trait = "0.1.89"
async-nats = "0.49.0"
async-nats = "0.49.1"
axum = "0.8.9"
futures = "0.3.32"
futures-core = "0.3.32"
@@ -132,7 +138,7 @@ lapin = { version = "4.10.0", default-features = false, features = ["tokio", "ru
hyper = { version = "1.10.1", features = ["http2", "http1", "server"] }
hyper-rustls = { version = "0.27.9", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
hyper-util = { version = "0.1.20", features = ["tokio", "server-auto", "server-graceful", "tracing"] }
http = "1.4.1"
http = "1.4.2"
http-body = "1.0.1"
http-body-util = "0.1.3"
reqwest = { version = "0.13.4", default-features = false, features = ["rustls", "charset", "http2", "system-proxy", "stream", "json", "blocking", "query", "form"] }
@@ -155,7 +161,7 @@ bytesize = "2.3.1"
byteorder = "1.5.0"
flatbuffers = "25.12.19"
form_urlencoded = "1.2.2"
prost = "0.14.3"
prost = "0.14.4"
quick-xml = "0.40.1"
rmp = { version = "0.8.15" }
rmp-serde = { version = "1.3.1" }
@@ -183,7 +189,7 @@ subtle = "2.6"
zeroize = { version = "1.8.2", features = ["derive"] }
# Time and Date
chrono = { version = "0.4.44", features = ["serde"] }
chrono = { version = "0.4.45", features = ["serde"] }
humantime = "2.3.0"
jiff = { version = "0.2.28", features = ["serde"] }
time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros", "serde"] }
@@ -197,7 +203,7 @@ tokio-postgres-rustls = "0.14.0"
anyhow = "1.0.102"
arc-swap = "1.9.1"
astral-tokio-tar = "0.6.2"
atoi = "2.0.0"
atoi = "3.0.0"
atomic_enum = "0.3.0"
aws-config = { version = "1.8.18" }
aws-credential-types = { version = "1.2.14" }
@@ -215,14 +221,14 @@ crossbeam-queue = "0.3.12"
crossbeam-channel = "0.5.15"
crossbeam-deque = "0.8.6"
crossbeam-utils = "0.8.21"
datafusion = "53.1.0"
datafusion = "54.0.0"
derive_builder = "0.20.2"
enumset = "1.1.13"
faster-hex = "0.10.0"
flate2 = "1.1.9"
glob = "0.3.3"
google-cloud-storage = "1.12.0"
google-cloud-auth = "1.10.0"
google-cloud-storage = "1.14.0"
google-cloud-auth = "1.12.0"
hashbrown = { version = "0.17.1", features = ["serde", "rayon"] }
hex = "0.4.3"
hex-simd = "0.8.0"
@@ -254,7 +260,7 @@ ratelimit = "0.10.1"
rayon = "1.12.0"
reed-solomon-erasure = { version = "6.0", default-features = false, features = ["std", "simd-accel"] }
reed-solomon-simd = "3.1.0"
regex = { version = "1.12.3" }
regex = { version = "1.12.4" }
rumqttc = { package = "rumqttc-next", version = "0.33.2", features = ["websocket"] }
redis = { version = "1.2.2", features = ["connection-manager", "tokio-rustls-comp", "tls-rustls-insecure"] }
rustix = { version = "1.1.4", features = ["fs"] }
@@ -283,7 +289,7 @@ tracing-subscriber = { version = "0.3.23", features = ["env-filter", "time"] }
transform-stream = "0.3.1"
url = "2.5.8"
urlencoding = "2.1.3"
uuid = { version = "1.23.2", features = ["v4", "fast-rng", "macro-diagnostics"] }
uuid = { version = "1.23.3", features = ["v4", "fast-rng", "macro-diagnostics"] }
vaultrs = { version = "0.8.0" }
walkdir = "2.5.0"
wildmatch = { version = "2.6.1", features = ["serde"] }
@@ -306,9 +312,9 @@ pyroscope = { version = "=2.0.5", features = ["backend-pprof-rs", "backend-jemal
# FTP and SFTP
libunftp = { version = "0.23.0", features = ["experimental"] }
unftp-core = "0.1.0"
suppaftp = { version = "8.0.3", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
suppaftp = { version = "8.0.4", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
rcgen = "0.14.8"
russh = { version = "0.61.1",features = ["serde"] }
russh = { version = "0.61.2", features = ["serde"] }
russh-sftp = "2.3.0"
# WebDAV
+16 -2
View File
@@ -103,7 +103,7 @@ curl -O https://rustfs.com/install_rustfs.sh && bash install_rustfs.sh
### 2. Docker Quick Start (Option 2)
The RustFS container runs as a non-root user `rustfs` (UID `10001`). If you run Docker with `-v` to mount a host directory, please ensure the host directory owner is set to `10001`, otherwise you will encounter permission denied errors.
The RustFS container runs as a non-root user `rustfs` (UID/GID `10001:10001`). If you bind-mount host directories with Docker or Compose, every mounted path must be writable by that user, otherwise startup may fail with permission denied errors. This applies to data directories, log directories, and TLS certificate directories when `RUSTFS_TLS_PATH` is enabled.
```bash
# Create data and logs directories
@@ -116,7 +116,7 @@ chown -R 10001:10001 data logs
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
# Using specific version
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.7
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.8
```
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
@@ -125,12 +125,26 @@ If you use [podman](https://github.com/containers/podman) instead of docker, you
podman run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
```
If you enable TLS with a bind-mounted certificate directory, prepare that mount the same way:
```bash
mkdir -p certs
chown -R 10001:10001 certs
```
You can also use Docker Compose. Using the `docker-compose.yml` file in the root directory:
```bash
docker compose --profile observability up -d
```
Before running Compose with host bind mounts:
- Ensure every mounted host path is writable by `10001:10001`.
- If you enable TLS, ensure the certificate mount for `/opt/tls` is also readable by `10001:10001`.
- If matching host ownership is not practical, run the `rustfs` service with `user: "<host-uid>:<host-gid>"` instead.
- `docker-compose-simple.yml` includes a `volume-permission-helper` service for named volumes. `docker-compose.yml` relies on you to prepare bind-mounted host paths in advance.
Similarly, you can run the command with podman
```bash
+23 -9
View File
@@ -100,20 +100,27 @@ curl -O https://rustfs.com/install_rustfs.sh && bash install_rustfs.sh
### 2\. Docker 快速启动 (选项 2)
RustFS 容器以非 root 用户 `rustfs` (UID `10001`) 运行。如果您使用 Docker `-v` 参数挂载宿主机目录,请务必确保宿主机目录的所有者已更改为 `10001`,否则会遇到权限拒绝错误
RustFS 容器以非 root 用户 `rustfs` (UID/GID `10001:10001`) 运行。如果您通过 Docker 或 Compose 绑定挂载宿主机目录,请务必确保所有挂载路径都对该用户可写,否则启动时可能出现权限拒绝错误。这不仅适用于数据目录和日志目录,也适用于启用 `RUSTFS_TLS_PATH` 时挂载的 TLS 证书目录
```bash
# 创建数据和日志目录
mkdir -p data logs
# 创建数据和日志目录
mkdir -p data logs
# 更改这两个目录的所有者
chown -R 10001:10001 data logs
# 更改这两个目录的所有者
chown -R 10001:10001 data logs
# 使用最新版本运行
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
# 使用最新版本运行
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
# 使用指定版本运行
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.7
# 使用指定版本运行
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-beta.8
```
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
```bash
mkdir -p certs
chown -R 10001:10001 certs
```
您也可以使用 Docker Compose。使用根目录下的 `docker-compose.yml` 文件:
@@ -122,6 +129,13 @@ RustFS 容器以非 root 用户 `rustfs` (UID `10001`) 运行。如果您使用
docker compose --profile observability up -d
```
在使用 Compose 且包含宿主机绑定挂载时,请先确认:
- 所有挂载到容器内的宿主机路径都对 `10001:10001` 可写。
- 如果启用了 TLS,挂载到 `/opt/tls` 的证书目录也需要对 `10001:10001` 可读。
- 如果不方便调整宿主机目录归属,可以为 `rustfs` 服务显式设置 `user: "<host-uid>:<host-gid>"`
- `docker-compose-simple.yml` 为命名 volume 提供了 `volume-permission-helper` 服务;`docker-compose.yml` 使用宿主机绑定挂载,因此需要您在启动前自行准备目录权限。
**注意**: 我们建议您在运行前查看 `docker-compose.yml` 文件。该文件定义了包括 Grafana、Prometheus 和 Jaeger 在内的多个服务,有助于 RustFS 的可观测性监控。如果您还想启动 Redis 或 Nginx 容器,可以指定相应的 profile。
### 3\. 源码编译 (选项 3) - 进阶用户
File diff suppressed because it is too large Load Diff
+2
View File
@@ -65,6 +65,8 @@ Current guidance:
## Scanner environment aliases
- `RUSTFS_SCANNER_SPEED` (canonical, also accepts `MINIO_SCANNER_SPEED`)
- `RUSTFS_SCANNER_DELAY` (canonical)
- `RUSTFS_SCANNER_MAX_WAIT_SECS` (canonical)
- `RUSTFS_SCANNER_CYCLE` (canonical, also accepts `MINIO_SCANNER_CYCLE`)
- `RUSTFS_SCANNER_START_DELAY_SECS` (canonical)
- `RUSTFS_DATA_SCANNER_START_DELAY_SECS` (deprecated alias for compatibility)
+12
View File
@@ -12,6 +12,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
/// Heal admin config subsystem name.
pub const HEAL_SUB_SYS: &str = "heal";
/// Heal config key setting the scanner-driven periodic deep bitrot scan cycle in seconds.
pub const HEAL_BITROT_CYCLE: &str = "bitrot_cycle";
/// Heal config keys supported by the admin config subsystem.
pub const HEAL_KEYS: &[&str] = &[HEAL_BITROT_CYCLE];
/// Default scanner-driven bitrot scan cycle used by heal/scanner runtime config.
pub const DEFAULT_HEAL_BITROT_CYCLE_SECS: u64 = 30 * 24 * 60 * 60;
/// Environment variable name that enables or disables auto-heal functionality.
/// - Purpose: Control whether the system automatically performs heal operations.
/// - Valid values: "true" or "false" (case insensitive).
+20 -1
View File
@@ -20,6 +20,12 @@ pub const SCANNER_SUB_SYS: &str = "scanner";
/// Scanner config key selecting the speed preset.
pub const SCANNER_SPEED: &str = "speed";
/// Scanner config key overriding the scanner sleep multiplier.
pub const SCANNER_DELAY: &str = "delay";
/// Scanner config key overriding the maximum scanner sleep in seconds.
pub const SCANNER_MAX_WAIT: &str = "max_wait";
/// Scanner config key overriding the cycle interval in seconds.
pub const SCANNER_CYCLE: &str = "cycle";
@@ -68,6 +74,8 @@ pub const SCANNER_ALERT_EXCESS_FOLDERS: &str = "alert_excess_folders";
/// Scanner config keys supported by the admin config subsystem.
pub const SCANNER_KEYS: &[&str] = &[
SCANNER_SPEED,
SCANNER_DELAY,
SCANNER_MAX_WAIT,
SCANNER_CYCLE,
SCANNER_START_DELAY,
SCANNER_CYCLE_MAX_DURATION,
@@ -123,6 +131,15 @@ pub const ENV_SCANNER_CYCLE_MAX_DIRECTORIES: &str = "RUSTFS_SCANNER_CYCLE_MAX_DI
/// - Example: `export RUSTFS_SCANNER_SPEED=slow`
pub const ENV_SCANNER_SPEED: &str = "RUSTFS_SCANNER_SPEED";
/// Environment variable that overrides the scanner sleep multiplier.
/// - Example: `export RUSTFS_SCANNER_DELAY=30.0`
pub const ENV_SCANNER_DELAY: &str = "RUSTFS_SCANNER_DELAY";
/// Environment variable that overrides the maximum scanner sleep in seconds.
/// - Unit: seconds (u64).
/// - Example: `export RUSTFS_SCANNER_MAX_WAIT_SECS=15`
pub const ENV_SCANNER_MAX_WAIT_SECS: &str = "RUSTFS_SCANNER_MAX_WAIT_SECS";
/// Default scanner speed preset.
pub const DEFAULT_SCANNER_SPEED: &str = "default";
@@ -237,7 +254,9 @@ pub const DEFAULT_SCANNER_INLINE_HEAL_ENABLE: bool = false;
/// | `slow` | 10x | 15 seconds| 1 minute |
/// | `slowest` | 100x | 15 seconds| 30 minutes |
///
/// The cycle interval can be overridden by `RUSTFS_SCANNER_CYCLE`.
/// The sleep factor, max sleep, and cycle interval can be overridden by
/// `RUSTFS_SCANNER_DELAY`, `RUSTFS_SCANNER_MAX_WAIT_SECS`, and
/// `RUSTFS_SCANNER_CYCLE`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum ScannerSpeed {
Fastest,
@@ -0,0 +1,208 @@
// Copyright 2026 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::common::{RustFSTestClusterEnvironment, init_logging, local_http_client};
use aws_sdk_s3::primitives::ByteStream;
use http::header::HOST;
use reqwest::StatusCode;
use rustfs_madmin::{ITEM_OFFLINE, InfoMessage, StorageInfo};
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use serial_test::serial;
use std::error::Error;
use std::process::Command;
use tokio::time::{Duration, sleep, timeout};
use uuid::Uuid;
const BUCKET: &str = "issue-2525-admin-timeout";
async fn signed_admin_get(
url: &str,
access_key: &str,
secret_key: &str,
) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
let uri = url.parse::<http::Uri>()?;
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
let request = http::Request::builder()
.method(http::Method::GET)
.uri(uri)
.header(HOST, authority)
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
.body(Body::empty())?;
let signed = sign_v4(request, 0, access_key, secret_key, "", "us-east-1");
let mut builder = local_http_client().get(url);
for (name, value) in signed.headers() {
builder = builder.header(name, value);
}
Ok(builder.send().await?)
}
async fn fetch_info(cluster: &RustFSTestClusterEnvironment) -> Result<InfoMessage, Box<dyn Error + Send + Sync>> {
let url = format!("{}/rustfs/admin/v3/info", cluster.nodes[0].url);
let response = signed_admin_get(&url, &cluster.access_key, &cluster.secret_key).await?;
parse_json_response(response, "cluster info").await
}
async fn fetch_storage_info(cluster: &RustFSTestClusterEnvironment) -> Result<StorageInfo, Box<dyn Error + Send + Sync>> {
let url = format!("{}/rustfs/admin/v3/storageinfo", cluster.nodes[0].url);
let response = signed_admin_get(&url, &cluster.access_key, &cluster.secret_key).await?;
parse_json_response(response, "storage info").await
}
async fn parse_json_response<T: serde::de::DeserializeOwned>(
response: reqwest::Response,
label: &str,
) -> Result<T, Box<dyn Error + Send + Sync>> {
let status = response.status();
let body = response.bytes().await?;
if status != StatusCode::OK {
return Err(format!("{label} request failed: {status} {}", String::from_utf8_lossy(body.as_ref())).into());
}
Ok(serde_json::from_slice(&body)?)
}
fn signal_process(pid: u32, signal: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
let output = Command::new("kill").arg(format!("-{signal}")).arg(pid.to_string()).output()?;
if output.status.success() {
return Ok(());
}
Err(format!("kill -{signal} {pid} failed: {}", String::from_utf8_lossy(&output.stderr)).into())
}
fn offline_server_count(info: &InfoMessage) -> usize {
info.servers
.as_ref()
.map(|servers| servers.iter().filter(|server| server.state == ITEM_OFFLINE).count())
.unwrap_or_default()
}
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn test_single_admin_timeout_does_not_immediately_mark_peer_offline() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
cluster.start().await?;
cluster.create_test_bucket(BUCKET).await?;
let warm_info = fetch_info(&cluster).await?;
assert_eq!(
warm_info
.backend
.as_ref()
.map(|backend| backend.offline_disks)
.unwrap_or_default(),
0,
"cluster should start with zero offline disks"
);
let warm_storage = fetch_storage_info(&cluster).await?;
assert_eq!(
warm_storage.backend.offline_disks.sum(),
0,
"storage info should start with zero offline disks"
);
let suspended_pid = cluster.nodes[1]
.process
.as_ref()
.ok_or("cluster node 1 process missing")?
.id();
signal_process(suspended_pid, "STOP")?;
let resume = tokio::spawn(async move {
sleep(Duration::from_secs(6)).await;
signal_process(suspended_pid, "CONT")
});
let client = cluster.create_s3_client(0)?;
let key = format!("issue-2525/{}", Uuid::new_v4().simple());
let payload = b"admin timeout regression payload".to_vec();
let during = timeout(Duration::from_secs(20), async {
tokio::join!(fetch_info(&cluster), fetch_storage_info(&cluster), async {
client
.put_object()
.bucket(BUCKET)
.key(&key)
.body(ByteStream::from(payload.clone()))
.send()
.await?;
Ok::<(), Box<dyn Error + Send + Sync>>(())
})
})
.await;
resume.await??;
let (info_during, storage_during, put_result) =
during.map_err(|_| "timed out while waiting for admin queries during suspended peer")?;
let info_during = info_during?;
let storage_during = storage_during?;
put_result?;
assert_eq!(
info_during
.backend
.as_ref()
.map(|backend| backend.offline_disks)
.unwrap_or_default(),
0,
"single admin timeout must not synthesize offline disks in /info"
);
assert_eq!(
offline_server_count(&info_during),
0,
"single admin timeout must not mark any server offline in /info"
);
assert_eq!(
storage_during.backend.offline_disks.sum(),
0,
"single admin timeout must not synthesize offline disks in /storageinfo"
);
assert!(
storage_during.disks.iter().all(|disk| disk.state != ITEM_OFFLINE),
"single admin timeout must not synthesize offline disk entries in /storageinfo"
);
let recovered_info = fetch_info(&cluster).await?;
assert_eq!(
recovered_info
.backend
.as_ref()
.map(|backend| backend.offline_disks)
.unwrap_or_default(),
0,
"offline disk count should stay at zero after the peer resumes"
);
let body = client
.get_object()
.bucket(BUCKET)
.key(&key)
.send()
.await?
.body
.collect()
.await?
.into_bytes();
assert_eq!(body.as_ref(), b"admin timeout regression payload");
Ok(())
}
@@ -60,6 +60,125 @@ mod tests {
assert_eq!(body.as_ref(), expected, "object body changed for {key}");
}
#[tokio::test]
#[serial]
async fn test_auto_heal_rebuilds_runtime_wiped_disk_without_restart() {
init_logging();
info!("Issue #1533: auto heal should rebuild a runtime-wiped disk in a 4-disk single-node erasure set without restart");
let mut env = RustFSTestEnvironment::new().await.expect("Failed to create test environment");
let root = PathBuf::from(env.temp_dir.clone());
let disk0 = root.join("disk0");
let disk1 = root.join("disk1");
let disk2 = root.join("disk2");
let disk3 = root.join("disk3");
for disk in [&disk0, &disk1, &disk2, &disk3] {
std::fs::create_dir_all(disk).expect("disk directory should be created");
}
env.temp_dir = disk3.to_string_lossy().to_string();
let disk0_arg = disk0.to_string_lossy().to_string();
let disk1_arg = disk1.to_string_lossy().to_string();
let disk2_arg = disk2.to_string_lossy().to_string();
env.start_rustfs_server_with_env(
vec![disk0_arg.as_str(), disk1_arg.as_str(), disk2_arg.as_str()],
&[
("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true"),
("RUSTFS_HEAL_INTERVAL_SECS", "10"),
],
)
.await
.expect("Failed to start 4-disk RustFS");
env.temp_dir = root.to_string_lossy().to_string();
let client = env.create_s3_client();
let bucket = "heal-runtime-wiped-disk";
let heal_timeout_secs = std::env::var("RUSTFS_AUTO_HEAL_RUNTIME_WIPE_TIMEOUT_SECS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.unwrap_or(45);
let objects: Vec<(String, Vec<u8>, &'static str)> = vec![
(
"runtime/alpha.txt".to_string(),
b"alpha payload for runtime wipe heal".to_vec(),
"text/plain; charset=utf-8",
),
("runtime/beta.bin".to_string(), (0..=127).collect::<Vec<u8>>(), "application/octet-stream"),
(
"runtime/dir/emoji-free-name.json".to_string(),
br#"{"status":"runtime-heal"}"#.to_vec(),
"application/json",
),
(
"runtime/dir/gamma.txt".to_string(),
b"gamma payload for runtime wipe heal".to_vec(),
"text/plain; charset=utf-8",
),
];
client
.create_bucket()
.bucket(bucket)
.send()
.await
.expect("bucket create should succeed");
for (key, body, content_type) in &objects {
client
.put_object()
.bucket(bucket)
.key(key)
.content_type(*content_type)
.body(ByteStream::from(body.clone()))
.send()
.await
.expect("PUT should succeed");
}
for (key, body, _) in &objects {
assert_object_body(&env, bucket, key, body).await;
assert!(
object_metadata_exists_on_disk(&disk0, bucket, key),
"disk0 should contain xl.meta before runtime wipe for {key}"
);
}
std::fs::remove_dir_all(&disk0).expect("disk0 wipe should succeed while server is running");
std::fs::create_dir_all(&disk0).expect("disk0 should be recreated empty while server is running");
assert!(!has_file_under(&disk0), "disk0 must be empty immediately after runtime wipe");
let mut remaining_rebuild_keys: HashSet<String> = objects.iter().map(|(key, _, _)| key.clone()).collect();
for _ in 0..heal_timeout_secs {
for (key, body, _) in &objects {
assert_object_body(&env, bucket, key, body).await;
}
if !remaining_rebuild_keys.is_empty() {
let mut rebuilt = Vec::new();
for key in &remaining_rebuild_keys {
if object_metadata_exists_on_disk(&disk0, bucket, key) {
rebuilt.push(key.clone());
}
}
for key in rebuilt {
let _ = remaining_rebuild_keys.remove(&key);
}
}
if remaining_rebuild_keys.is_empty() {
assert!(
disk0.join(".rustfs.sys").join("format.json").is_file(),
"runtime-wiped disk should have format.json restored by auto heal"
);
return;
}
sleep(Duration::from_secs(1)).await;
}
panic!("auto heal did not rebuild all files on the runtime-wiped disk within timeout");
}
#[tokio::test]
#[serial]
async fn test_admin_deep_heal_rebuilds_cleared_disk_in_single_node_erasure_set() {
@@ -520,3 +520,75 @@ async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box<d
Ok(())
}
/// Test 5: Setting SSE-KMS without a specific key ID should auto-populate the
/// default KMS key ID so that GetBucketEncryption returns it (issue #3039).
#[tokio::test]
#[serial]
async fn test_sse_kms_without_key_id_populates_default() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
init_logging();
info!("Testing SSE-KMS without explicit key ID populates default key");
let mut kms_env = LocalKMSTestEnvironment::new().await?;
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
let s3_client = kms_env.base_env.create_s3_client();
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
// Set bucket encryption to SSE-KMS WITHOUT specifying a key ID
info!("Setting bucket default encryption to SSE-KMS without key ID");
let encryption_config = ServerSideEncryptionConfiguration::builder()
.rules(
ServerSideEncryptionRule::builder()
.apply_server_side_encryption_by_default(
ServerSideEncryptionByDefault::builder()
.sse_algorithm(ServerSideEncryption::AwsKms)
.kms_master_key_id("")
.build()
.unwrap(),
)
.build(),
)
.build()
.unwrap();
s3_client
.put_bucket_encryption()
.bucket(TEST_BUCKET)
.server_side_encryption_configuration(encryption_config)
.send()
.await
.expect("Failed to set bucket SSE-KMS encryption without key ID");
// GetBucketEncryption should return the default KMS key ID
info!("Verifying GetBucketEncryption returns default KMS key ID");
let get_response = s3_client
.get_bucket_encryption()
.bucket(TEST_BUCKET)
.send()
.await
.expect("Failed to get bucket encryption");
let config = get_response.server_side_encryption_configuration();
let rule = config
.and_then(|c| c.rules().first())
.expect("Should have at least one encryption rule");
let by_default = rule
.apply_server_side_encryption_by_default()
.expect("Should have ApplyServerSideEncryptionByDefault");
assert_eq!(by_default.sse_algorithm(), &ServerSideEncryption::AwsKms, "Algorithm should be aws:kms");
assert!(
by_default.kms_master_key_id().is_some(),
"KMS key ID should be populated with the default key (was None)"
);
assert_eq!(
by_default.kms_master_key_id().unwrap(),
&default_key_id,
"KMS key ID should match the configured default key"
);
info!("Test passed: SSE-KMS without key ID correctly populates default key '{}'", default_key_id);
Ok(())
}
+3
View File
@@ -149,4 +149,7 @@ mod snowball_auto_extract_test;
#[cfg(test)]
mod namespace_lock_quorum_test;
#[cfg(test)]
mod admin_timeout_regression_test;
pub mod tls_gen;
@@ -120,3 +120,115 @@ async fn test_concurrent_cluster_overwrites_do_not_fail_namespace_lock_quorum()
clients[0].delete_object().bucket(BUCKET).key(KEY).send().await?;
Ok(())
}
/// Regression test: concurrent PUTs to the same key must return 503 (ServiceUnavailable)
/// on lock contention, never 500 (InternalError).
///
/// Before the fix, `map_namespace_lock_error` wrapped lock timeout/conflict errors as
/// `StorageError::other(...)` → `StorageError::Io(...)`, which fell through to
/// `S3ErrorCode::InternalError` (500) in the error mapping.
#[tokio::test]
#[serial]
async fn test_concurrent_put_same_key_never_returns_500() -> TestResult {
crate::common::init_logging();
info!("Starting concurrent PUT 500 regression test");
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
// Short lock timeout to trigger contention errors quickly
cluster.set_env("RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT", "3");
cluster.start().await?;
cluster.create_test_bucket(BUCKET).await?;
let clients = cluster.create_all_clients()?;
let writer_count = clients.len() * 4; // 16 writers for heavy contention
let barrier = Arc::new(Barrier::new(writer_count));
// Seed initial object
let first_payload = b"initial object for 500 regression".to_vec();
put_object(clients[0].clone(), first_payload, 0).await?;
let err_500_count = Arc::new(std::sync::atomic::AtomicU64::new(0));
let err_503_count = Arc::new(std::sync::atomic::AtomicU64::new(0));
let unexpected_err_count = Arc::new(std::sync::atomic::AtomicU64::new(0));
let ok_count = Arc::new(std::sync::atomic::AtomicU64::new(0));
let mut handles = Vec::with_capacity(writer_count);
for writer_id in 0..writer_count {
let client = clients[writer_id % clients.len()].clone();
let barrier = barrier.clone();
let payload = format!("payload from writer {writer_id:02}").into_bytes();
let err_500 = err_500_count.clone();
let err_503 = err_503_count.clone();
let unexpected_err = unexpected_err_count.clone();
let ok = ok_count.clone();
handles.push(tokio::spawn(async move {
barrier.wait().await;
match client
.put_object()
.bucket(BUCKET)
.key(KEY)
.body(Bytes::from(payload).into())
.send()
.await
{
Ok(_) => {
ok.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
Err(err) => match &err {
SdkError::ServiceError(service_err) => {
let code = service_err.err().meta().code().unwrap_or("<unknown>");
if code == "500" || code == "InternalError" {
err_500.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
warn!("writer {writer_id} returned 500: {code}");
} else if code == "503" || code == "ServiceUnavailable" {
err_503.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
} else {
unexpected_err.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
warn!("writer {writer_id} returned unexpected error: {code}");
}
}
other => {
unexpected_err.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
warn!("writer {writer_id} returned SDK error: {other:?}");
}
},
}
}));
}
for handle in handles {
if let Err(err) = handle.await {
unexpected_err_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
warn!("writer task join failed: {err}");
}
}
let ok = ok_count.load(std::sync::atomic::Ordering::Relaxed);
let err_503 = err_503_count.load(std::sync::atomic::Ordering::Relaxed);
let err_500 = err_500_count.load(std::sync::atomic::Ordering::Relaxed);
let unexpected_err = unexpected_err_count.load(std::sync::atomic::Ordering::Relaxed);
let total = ok + err_503 + err_500 + unexpected_err;
info!("Concurrent PUT 500 regression: total={total}, ok={ok}, 503={err_503}, 500={err_500}, unexpected={unexpected_err}");
assert_eq!(
total, writer_count as u64,
"every concurrent PUT writer must be classified as success, 503, 500, or unexpected error"
);
assert_eq!(
unexpected_err, 0,
"Concurrent PUTs to the same key must only succeed or return 503. \
Got {unexpected_err} unexpected errors out of {total} requests. 503 count: {err_503}, ok: {ok}"
);
assert_eq!(
err_500, 0,
"Concurrent PUTs to the same key must NEVER return 500 InternalError. \
Got {err_500} out of {total} requests. 503 count: {err_503}, ok: {ok}"
);
clients[0].delete_object().bucket(BUCKET).key(KEY).send().await?;
Ok(())
}
+7
View File
@@ -32,12 +32,15 @@ workspace = true
[features]
default = []
rio-v2 = ["dep:rustfs-rio-v2"]
[dependencies]
rustfs-filemeta.workspace = true
rustfs-utils = { workspace = true, features = ["full"] }
rustfs-rio.workspace = true
rustfs-rio-v2 = { workspace = true, optional = true }
rustfs-signer.workspace = true
rustfs-storage-api.workspace = true
rustfs-tls-runtime.workspace = true
rustfs-checksums.workspace = true
rustfs-config = { workspace = true, features = ["constants", "notify", "audit"] }
@@ -153,5 +156,9 @@ harness = false
name = "rename_data_meta_benchmark"
harness = false
[[bench]]
name = "single_block_non_inline_benchmark"
harness = false
[lib]
doctest = false
@@ -0,0 +1,126 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
use rustfs_ecstore::erasure_coding::{BitrotWriterWrapper, CustomWriter, Erasure};
use rustfs_utils::HashAlgorithm;
use std::io::Cursor;
use std::sync::Arc;
use std::time::Duration;
#[derive(Clone, Debug)]
struct BenchConfig {
name: String,
payload_size: usize,
data_shards: usize,
parity_shards: usize,
block_size: usize,
}
impl BenchConfig {
fn new(payload_size: usize, data_shards: usize, parity_shards: usize, block_size: usize) -> Self {
Self {
name: format!(
"payload={}KB_{}+{}_block={}KB",
payload_size / 1024,
data_shards,
parity_shards,
block_size / 1024
),
payload_size,
data_shards,
parity_shards,
block_size,
}
}
}
fn generate_payload(size: usize) -> Vec<u8> {
(0..size).map(|i| (i % 251) as u8).collect()
}
fn build_non_inline_writers(config: &BenchConfig) -> Vec<Option<BitrotWriterWrapper>> {
let erasure = Erasure::new(config.data_shards, config.parity_shards, config.block_size);
let total_shards = config.data_shards + config.parity_shards;
let shard_size = erasure.shard_size();
(0..total_shards)
.map(|_| {
Some(BitrotWriterWrapper::new(
CustomWriter::new_tokio_writer(tokio::io::sink()),
shard_size,
HashAlgorithm::HighwayHash256S,
))
})
.collect()
}
fn bench_single_block_non_inline_fast_path(c: &mut Criterion) {
let configs = vec![
BenchConfig::new(4 * 1024, 4, 2, 128 * 1024),
BenchConfig::new(64 * 1024, 4, 2, 128 * 1024),
BenchConfig::new(128 * 1024, 4, 2, 128 * 1024),
];
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("benchmark runtime");
let mut group = c.benchmark_group("single_block_non_inline_write_path");
group.sample_size(20);
group.measurement_time(Duration::from_secs(6));
for config in configs {
let payload = generate_payload(config.payload_size);
let erasure = Arc::new(Erasure::new(config.data_shards, config.parity_shards, config.block_size));
group.throughput(Throughput::Bytes(config.payload_size as u64));
group.bench_with_input(BenchmarkId::new("encode_pipeline", &config.name), &config, |b, config| {
let payload = payload.clone();
let erasure = Arc::clone(&erasure);
b.iter(|| {
let mut writers = build_non_inline_writers(config);
let reader = tokio::io::BufReader::new(Cursor::new(payload.clone()));
rt.block_on(async {
erasure
.clone()
.encode(reader, &mut writers, config.data_shards)
.await
.expect("encode pipeline benchmark");
});
});
});
group.bench_with_input(BenchmarkId::new("single_block_candidate", &config.name), &config, |b, config| {
let payload = payload.clone();
let erasure = Arc::clone(&erasure);
b.iter(|| {
let mut writers = build_non_inline_writers(config);
let reader = tokio::io::BufReader::new(Cursor::new(payload.clone()));
rt.block_on(async {
erasure
.clone()
.encode_single_block_non_inline(reader, &mut writers, config.data_shards)
.await
.expect("single block candidate benchmark");
});
});
});
}
group.finish();
}
criterion_group!(benches, bench_single_block_non_inline_fast_path);
criterion_main!(benches);
@@ -45,7 +45,7 @@ use futures::Future;
use http::HeaderMap;
use lazy_static::lazy_static;
use rustfs_common::heal_channel::rep_has_active_rules;
use rustfs_common::metrics::{IlmAction, Metrics};
use rustfs_common::metrics::{IlmAction, Metrics, global_metrics};
use rustfs_config::{
DEFAULT_TRANSITION_QUEUE_CAPACITY, DEFAULT_TRANSITION_QUEUE_SEND_TIMEOUT_MS, DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX,
DEFAULT_TRANSITION_WORKERS_CAP, ENV_TRANSITION_QUEUE_CAPACITY, ENV_TRANSITION_QUEUE_SEND_TIMEOUT_MS, ENV_TRANSITION_WORKERS,
@@ -145,6 +145,12 @@ fn is_immediate_transition_source(src: &LcEventSrc) -> bool {
)
}
fn record_scanner_lifecycle_enqueue_result(src: &LcEventSrc, count: u64, queued: bool) {
if matches!(src, LcEventSrc::Scanner) {
global_metrics().record_scanner_ilm_enqueue_result(count, queued);
}
}
#[cfg(any(test, debug_assertions))]
fn should_force_immediate_transition_enqueue_timeout() -> bool {
env::var(rustfs_config::ENV_TEST_FORCE_IMMEDIATE_TRANSITION_ENQUEUE_TIMEOUT)
@@ -357,7 +363,7 @@ impl ExpiryState {
}
}
pub async fn enqueue_by_days(&mut self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
pub async fn enqueue_by_days(&mut self, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) -> bool {
let task = ExpiryTask {
obj_info: oi.clone(),
event: event.clone(),
@@ -366,22 +372,29 @@ impl ExpiryState {
let wrkr = self.get_worker_ch(task.op_hash());
if wrkr.is_none() {
*self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
return;
record_scanner_lifecycle_enqueue_result(src, 1, false);
return false;
}
let wrkr = wrkr.expect("worker channel should exist after None check");
select! {
//_ -> GlobalContext.Done() => {}
_ = wrkr.send(Some(Box::new(task))) => (),
else => {
*self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
}
let queued = wrkr.send(Some(Box::new(task))).await.is_ok();
if !queued {
*self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
}
record_scanner_lifecycle_enqueue_result(src, 1, queued);
queued
}
pub async fn enqueue_by_newer_noncurrent(&mut self, bucket: &str, versions: Vec<ObjectToDelete>, lc_event: lifecycle::Event) {
pub async fn enqueue_by_newer_noncurrent(
&mut self,
bucket: &str,
versions: Vec<ObjectToDelete>,
lc_event: lifecycle::Event,
src: &LcEventSrc,
) -> bool {
if versions.is_empty() {
return;
return true;
}
let version_count = u64::try_from(versions.len()).unwrap_or(u64::MAX);
let task = NewerNoncurrentTask {
bucket: String::from(bucket),
@@ -391,16 +404,16 @@ impl ExpiryState {
let wrkr = self.get_worker_ch(task.op_hash());
if wrkr.is_none() {
*self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
return;
record_scanner_lifecycle_enqueue_result(src, version_count, false);
return false;
}
let wrkr = wrkr.expect("worker channel should exist after None check");
select! {
//_ -> GlobalContext.Done() => {}
_ = wrkr.send(Some(Box::new(task))) => (),
else => {
*self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
}
let queued = wrkr.send(Some(Box::new(task))).await.is_ok();
if !queued {
*self.stats.as_mut().expect("stats lock").missed_expiry_tasks.get_mut() += 1;
}
record_scanner_lifecycle_enqueue_result(src, version_count, queued);
queued
}
pub fn get_worker_ch(&self, h: u64) -> Option<Sender<Option<ExpiryOpType>>> {
@@ -739,10 +752,11 @@ impl TransitionState {
}
}
pub async fn queue_transition_task(self: &Arc<Self>, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
pub async fn queue_transition_task(self: &Arc<Self>, oi: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) -> bool {
if is_immediate_transition_source(src) && should_force_immediate_transition_enqueue_timeout() {
self.handle_immediate_enqueue_failure(oi, src, ImmediateEnqueueFailure::ForcedTimeout);
return;
record_scanner_lifecycle_enqueue_result(src, 1, false);
return false;
}
let task = TransitionTask {
@@ -750,14 +764,15 @@ impl TransitionState {
src: src.clone(),
event: event.clone(),
};
let mut queued = false;
if is_immediate_transition_source(src) {
match self.transition_tx.try_send(Some(task)) {
Ok(()) => {}
Ok(()) => queued = true,
Err(async_channel::TrySendError::Full(task)) => {
Self::inc_counter(&self.queue_full_tasks);
let send_timeout = self.transition_queue_send_timeout;
match tokio::time::timeout(send_timeout, self.transition_tx.send(task)).await {
Ok(Ok(())) => {}
Ok(Ok(())) => queued = true,
Ok(Err(_)) => {
self.handle_immediate_enqueue_failure(
oi,
@@ -782,29 +797,31 @@ impl TransitionState {
self.handle_immediate_enqueue_failure(oi, src, ImmediateEnqueueFailure::QueueClosed { timeout_ms: None });
}
}
return;
record_scanner_lifecycle_enqueue_result(src, 1, queued);
return queued;
}
if let Err(err) = self.transition_tx.try_send(Some(task)) {
match err {
async_channel::TrySendError::Full(_) => {
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
"transition queue is full; deferring to scanner/backfill"
);
}
async_channel::TrySendError::Closed(_) => {
warn!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
"transition enqueue failed because the queue is closed"
);
}
match self.transition_tx.try_send(Some(task)) {
Ok(()) => queued = true,
Err(async_channel::TrySendError::Full(_)) => {
debug!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
"transition queue is full; deferring to scanner/backfill"
);
}
Err(async_channel::TrySendError::Closed(_)) => {
warn!(
bucket = %oi.bucket,
object = %oi.name,
source = ?src,
"transition enqueue failed because the queue is closed"
);
}
}
record_scanner_lifecycle_enqueue_result(src, 1, queued);
queued
}
pub async fn init(api: Arc<ECStore>) {
@@ -1482,7 +1499,7 @@ pub async fn enqueue_immediate_expiry(oi: &ObjectInfo, src: LcEventSrc) {
GLOBAL_ExpiryState
.write()
.await
.enqueue_by_newer_noncurrent(&oi.bucket, to_delete_objs, event)
.enqueue_by_newer_noncurrent(&oi.bucket, to_delete_objs, event, &src)
.await;
}
}
@@ -1770,11 +1787,31 @@ pub async fn get_transitioned_object_reader(
gopts.length = length;
}
//return Ok(HttpFileReader::new(rs, &oi, opts, &h));
//timeTierAction := auditTierActions(oi.transitioned_object.Tier, length)
debug!(
bucket = %bucket,
object = %object,
tier = %oi.transitioned_object.tier,
tier_object = %oi.transitioned_object.name,
tier_version_id = %oi.transitioned_object.version_id,
start_offset = gopts.start_offset,
length = gopts.length,
"fetching transitioned object from tier"
);
let reader = tgt_client
.get(&oi.transitioned_object.name, &oi.transitioned_object.version_id, gopts)
.await?;
.await
.map_err(|e| {
tracing::error!(
bucket = %bucket,
object = %object,
tier = %oi.transitioned_object.tier,
tier_object = %oi.transitioned_object.name,
tier_version_id = %oi.transitioned_object.version_id,
error = %e,
"tier GET failed"
);
e
})?;
Ok(get_fn(reader, h.clone()))
}
@@ -2025,8 +2062,7 @@ pub async fn apply_transition_rule(event: &lifecycle::Event, src: &LcEventSrc, o
if oi.delete_marker || oi.is_dir {
return false;
}
GLOBAL_TransitionState.queue_transition_task(oi, event, src).await;
true
GLOBAL_TransitionState.queue_transition_task(oi, event, src).await
}
pub async fn apply_expiry_on_transitioned_object(
@@ -2115,8 +2151,7 @@ pub async fn apply_expiry_on_non_transitioned_objects(
pub async fn apply_expiry_rule(event: &lifecycle::Event, src: &LcEventSrc, oi: &ObjectInfo) -> bool {
let mut expiry_state = GLOBAL_ExpiryState.write().await;
expiry_state.enqueue_by_days(oi, event, src).await;
true
expiry_state.enqueue_by_days(oi, event, src).await
}
fn lifecycle_deleted_object(oi: &ObjectInfo, dobj: &ObjectInfo) -> crate::store_api::DeletedObject {
@@ -2277,7 +2312,7 @@ pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc,
mod tests {
use super::{
DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS, DEFAULT_TRANSITION_QUEUE_CAPACITY, DEFAULT_TRANSITION_WORKERS_ABSOLUTE_MAX,
DEFAULT_TRANSITION_WORKERS_CAP, GLOBAL_TransitionState, StaleMultipartUploadCandidate, TransitionState,
DEFAULT_TRANSITION_WORKERS_CAP, ExpiryState, GLOBAL_TransitionState, StaleMultipartUploadCandidate, TransitionState,
cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at, lifecycle_deleted_object,
lifecycle_rule_has_date_expiration, lifecycle_version_purge_state_from_completed_targets,
mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate, replication_state_for_delete,
@@ -2285,6 +2320,7 @@ mod tests {
resolve_transition_workers_absolute_max, should_defer_date_expiry_for_recent_config_update,
should_reuse_lifecycle_delete_replication_state,
};
use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
use crate::bucket::metadata_sys;
use crate::disk::RUSTFS_META_MULTIPART_BUCKET;
@@ -2297,6 +2333,7 @@ mod tests {
BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations, ObjectInfo, ObjectOptions, PutObjReader,
};
use futures::FutureExt;
use rustfs_common::metrics::IlmAction;
use rustfs_config::ENV_TRANSITION_WORKERS_ABSOLUTE_MAX;
use rustfs_filemeta::{ReplicateDecision, VersionPurgeStatusType};
use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, Timestamp};
@@ -2313,6 +2350,48 @@ mod tests {
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
#[tokio::test]
async fn expiry_enqueue_reports_missed_without_worker_channel() {
let state = ExpiryState::new();
let mut state = state.write().await;
let object = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
..Default::default()
};
let event = crate::bucket::lifecycle::lifecycle::Event {
action: IlmAction::DeleteAction,
..Default::default()
};
let queued = state.enqueue_by_days(&object, &event, &LcEventSrc::Scanner).await;
assert!(!queued);
let stats = state.stats.as_ref().expect("expiry stats should exist");
assert_eq!(stats.missed_tasks(), 1);
}
#[tokio::test]
async fn scanner_transition_enqueue_reports_full_queue() {
let state = TransitionState::new_with_capacity(1);
let object = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
..Default::default()
};
let event = crate::bucket::lifecycle::lifecycle::Event {
action: IlmAction::TransitionAction,
..Default::default()
};
let first = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;
let second = state.queue_transition_task(&object, &event, &LcEventSrc::Scanner).await;
assert!(first);
assert!(!second);
assert_eq!(state.transition_rx.len(), 1);
}
#[test]
fn mark_delete_opts_skip_decommissioned_on_remote_success_sets_flag_on_success() {
let mut opts = ObjectOptions::default();
+23
View File
@@ -30,6 +30,7 @@ use s3s::dto::{
ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration,
};
use serde::Serializer;
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::io::{Read, Write};
use std::sync::Arc;
@@ -246,6 +247,28 @@ pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
pub const BUCKET_TABLE_CONFIG: &str = "table-bucket.json";
pub const BUCKET_TABLE_RESERVED_PREFIX: &str = ".rustfs-table";
pub const BUCKET_TABLE_CATALOG_META_PREFIX: &str = "s3tables/catalog";
pub const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str = "table-buckets";
pub fn table_catalog_path_hash(value: &str) -> String {
let digest = Sha256::digest(value.as_bytes());
let mut output = String::with_capacity(digest.len() * 2);
const HEX: &[u8; 16] = b"0123456789abcdef";
for byte in digest {
output.push(char::from(HEX[usize::from(byte >> 4)]));
output.push(char::from(HEX[usize::from(byte & 0x0f)]));
}
output
}
pub fn table_bucket_catalog_metadata_prefix(bucket: &str) -> String {
format!(
"{}/{}/{}",
BUCKET_TABLE_CATALOG_META_PREFIX,
BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX,
table_catalog_path_hash(bucket)
)
}
#[derive(Debug, Clone)]
pub struct BucketMetadata {
@@ -68,6 +68,30 @@ pub const MRF_WORKER_AUTO_DEFAULT: usize = 4;
pub const LARGE_WORKER_COUNT: usize = 10;
pub const MIN_LARGE_OBJ_SIZE: i64 = 128 * 1024 * 1024; // 128MiB
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum ReplicationQueueAdmission {
#[default]
Skipped,
Queued,
Missed,
}
impl ReplicationQueueAdmission {
fn merge(&mut self, other: Self) {
*self = match (*self, other) {
(Self::Missed, _) | (_, Self::Missed) => Self::Missed,
(Self::Queued, _) | (_, Self::Queued) => Self::Queued,
(Self::Skipped, Self::Skipped) => Self::Skipped,
};
}
}
#[derive(Debug, Clone, Default)]
pub struct ReplicationHealQueueResult {
pub object_info: ReplicateObjectInfo,
pub admission: ReplicationQueueAdmission,
}
/// Priority levels for replication
#[derive(Debug, Clone, PartialEq)]
pub enum ReplicationPriority {
@@ -513,7 +537,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
}
/// Queues a replica task
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) {
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
// If object is large, queue it to a static set of large workers
if ri.size >= MIN_LARGE_OBJ_SIZE {
use std::collections::hash_map::DefaultHasher;
@@ -532,7 +556,11 @@ impl<S: StorageAPI> ReplicationPool<S> {
&& worker.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_err()
{
// Queue to MRF if worker is busy
let _ = self.mrf_save_tx.try_send(ri.to_mrf_entry());
let admission = if self.mrf_save_tx.try_send(ri.to_mrf_entry()).is_ok() {
ReplicationQueueAdmission::Queued
} else {
ReplicationQueueAdmission::Missed
};
// Try to add more workers if possible
let max_l_workers = *self.max_l_workers.read().await;
@@ -543,9 +571,12 @@ impl<S: StorageAPI> ReplicationPool<S> {
drop(lrg_workers);
self.resize_lrg_workers(workers, existing).await;
}
return admission;
}
return ReplicationQueueAdmission::Queued;
}
return;
return ReplicationQueueAdmission::Missed;
}
// Handle regular sized objects
@@ -555,85 +586,105 @@ impl<S: StorageAPI> ReplicationPool<S> {
_ => self.get_worker_ch(&ri.bucket, &ri.name, ri.size).await,
};
if let Some(channel) = ch
&& channel.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_err()
{
// Queue to MRF if all workers are busy
let _ = self.mrf_save_tx.try_send(ri.to_mrf_entry());
let Some(channel) = ch else {
return ReplicationQueueAdmission::Missed;
};
// Try to scale up workers based on priority
let priority = self.priority.read().await.clone();
let max_workers = *self.max_workers.read().await;
if channel.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_ok() {
return ReplicationQueueAdmission::Queued;
}
match priority {
ReplicationPriority::Fast => {
// Log warning about unable to keep up
info!("Warning: Unable to keep up with incoming traffic");
// Queue to MRF if all workers are busy
let admission = if self.mrf_save_tx.try_send(ri.to_mrf_entry()).is_ok() {
ReplicationQueueAdmission::Queued
} else {
ReplicationQueueAdmission::Missed
};
// Try to scale up workers based on priority
let priority = self.priority.read().await.clone();
let max_workers = *self.max_workers.read().await;
match priority {
ReplicationPriority::Fast => {
// Log warning about unable to keep up
info!("Warning: Unable to keep up with incoming traffic");
}
ReplicationPriority::Slow => {
info!("Warning: Unable to keep up with incoming traffic - recommend increasing replication priority to auto");
}
ReplicationPriority::Auto => {
let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT);
let active_workers = self.active_workers();
if active_workers < max_w as i32 {
let workers = self.workers.read().await;
let new_count = std::cmp::min(workers.len() + 1, max_w);
let existing = workers.len();
drop(workers);
self.resize_workers(new_count, existing).await;
}
ReplicationPriority::Slow => {
info!("Warning: Unable to keep up with incoming traffic - recommend increasing replication priority to auto");
}
ReplicationPriority::Auto => {
let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT);
let active_workers = self.active_workers();
if active_workers < max_w as i32 {
let workers = self.workers.read().await;
let new_count = std::cmp::min(workers.len() + 1, max_w);
let existing = workers.len();
let max_mrf_workers = std::cmp::min(max_workers, MRF_WORKER_MAX_LIMIT);
let active_mrf = self.active_mrf_workers();
drop(workers);
self.resize_workers(new_count, existing).await;
}
if active_mrf < max_mrf_workers as i32 {
let current_mrf = self.mrf_worker_size.load(Ordering::SeqCst);
let new_mrf = std::cmp::min(current_mrf + 1, max_mrf_workers as i32);
let max_mrf_workers = std::cmp::min(max_workers, MRF_WORKER_MAX_LIMIT);
let active_mrf = self.active_mrf_workers();
if active_mrf < max_mrf_workers as i32 {
let current_mrf = self.mrf_worker_size.load(Ordering::SeqCst);
let new_mrf = std::cmp::min(current_mrf + 1, max_mrf_workers as i32);
self.resize_failed_workers(new_mrf).await;
}
self.resize_failed_workers(new_mrf).await;
}
}
}
admission
}
/// Queues a replica delete task
pub async fn queue_replica_delete_task(&self, doi: DeletedObjectReplicationInfo) {
pub async fn queue_replica_delete_task(&self, doi: DeletedObjectReplicationInfo) -> ReplicationQueueAdmission {
let ch = match doi.op_type {
ReplicationType::Heal | ReplicationType::ExistingObject => Some(self.mrf_replica_tx.clone()),
_ => self.get_worker_ch(&doi.bucket, &doi.delete_object.object_name, 0).await,
};
if let Some(channel) = ch
&& channel.try_send(ReplicationOperation::Delete(Box::new(doi.clone()))).is_err()
{
let _ = self.mrf_save_tx.try_send(doi.to_mrf_entry());
let Some(channel) = ch else {
return ReplicationQueueAdmission::Missed;
};
let priority = self.priority.read().await.clone();
let max_workers = *self.max_workers.read().await;
if channel.try_send(ReplicationOperation::Delete(Box::new(doi.clone()))).is_ok() {
return ReplicationQueueAdmission::Queued;
}
match priority {
ReplicationPriority::Fast => {
info!("Warning: Unable to keep up with incoming deletes");
}
ReplicationPriority::Slow => {
info!("Warning: Unable to keep up with incoming deletes - recommend increasing replication priority to auto");
}
ReplicationPriority::Auto => {
let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT);
if self.active_workers() < max_w as i32 {
let workers = self.workers.read().await;
let new_count = std::cmp::min(workers.len() + 1, max_w);
let existing = workers.len();
drop(workers);
self.resize_workers(new_count, existing).await;
}
let admission = if self.mrf_save_tx.try_send(doi.to_mrf_entry()).is_ok() {
ReplicationQueueAdmission::Queued
} else {
ReplicationQueueAdmission::Missed
};
let priority = self.priority.read().await.clone();
let max_workers = *self.max_workers.read().await;
match priority {
ReplicationPriority::Fast => {
info!("Warning: Unable to keep up with incoming deletes");
}
ReplicationPriority::Slow => {
info!("Warning: Unable to keep up with incoming deletes - recommend increasing replication priority to auto");
}
ReplicationPriority::Auto => {
let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT);
if self.active_workers() < max_w as i32 {
let workers = self.workers.read().await;
let new_count = std::cmp::min(workers.len() + 1, max_w);
let existing = workers.len();
drop(workers);
self.resize_workers(new_count, existing).await;
}
}
}
admission
}
/// Queues an MRF save operation
@@ -959,8 +1010,8 @@ pub trait ReplicationPoolTrait: std::fmt::Debug {
fn active_workers(&self) -> i32;
fn active_mrf_workers(&self) -> i32;
fn active_lrg_workers(&self) -> i32;
async fn queue_replica_task(&self, ri: ReplicateObjectInfo);
async fn queue_replica_delete_task(&self, ri: DeletedObjectReplicationInfo);
async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission;
async fn queue_replica_delete_task(&self, ri: DeletedObjectReplicationInfo) -> ReplicationQueueAdmission;
async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize);
async fn get_bucket_resync_status(&self, bucket: &str) -> Result<BucketReplicationResyncStatus, EcstoreError>;
async fn cancel_bucket_resync(&self, opts: ResyncOpts) -> Result<(), EcstoreError>;
@@ -987,12 +1038,12 @@ impl<S: StorageAPI> ReplicationPoolTrait for ReplicationPool<S> {
ReplicationPool::<S>::active_lrg_workers(self)
}
async fn queue_replica_task(&self, ri: ReplicateObjectInfo) {
self.queue_replica_task(ri).await;
async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
self.queue_replica_task(ri).await
}
async fn queue_replica_delete_task(&self, ri: DeletedObjectReplicationInfo) {
self.queue_replica_delete_task(ri).await;
async fn queue_replica_delete_task(&self, ri: DeletedObjectReplicationInfo) -> ReplicationQueueAdmission {
self.queue_replica_delete_task(ri).await
}
async fn resize(&self, priority: ReplicationPriority, max_workers: usize, max_l_workers: usize) {
@@ -1093,13 +1144,13 @@ pub async fn schedule_replication<S: StorageAPI>(oi: ObjectInfo, o: Arc<S>, dsc:
if dsc.is_synchronous() {
replicate_object(ri, o).await
} else if let Some(pool) = GLOBAL_REPLICATION_POOL.get() {
pool.queue_replica_task(ri).await;
let _ = pool.queue_replica_task(ri).await;
}
}
pub async fn schedule_replication_delete(dv: DeletedObjectReplicationInfo) {
if let Some(pool) = GLOBAL_REPLICATION_POOL.get() {
pool.queue_replica_delete_task(dv.clone()).await;
let _ = pool.queue_replica_delete_task(dv.clone()).await;
}
if let (Some(rs), Some(stats)) = (dv.delete_object.replication_state, GLOBAL_REPLICATION_STATS.get()) {
@@ -1153,23 +1204,32 @@ pub async fn queue_replication_heal_internal(
oi: ObjectInfo,
rcfg: ReplicationConfig,
retry_count: u32,
) -> ReplicateObjectInfo {
) -> ReplicationHealQueueResult {
let mut roi = ReplicateObjectInfo::default();
// ignore modtime zero objects
if oi.mod_time.is_none() || oi.mod_time == Some(OffsetDateTime::UNIX_EPOCH) {
return roi;
return ReplicationHealQueueResult {
object_info: roi,
admission: ReplicationQueueAdmission::Skipped,
};
}
if rcfg.config.is_none() || rcfg.remotes.is_none() {
return roi;
return ReplicationHealQueueResult {
object_info: roi,
admission: ReplicationQueueAdmission::Skipped,
};
}
roi = get_heal_replicate_object_info(&oi, &rcfg).await;
roi.retry_count = retry_count;
if !roi.dsc.replicate_any() {
return roi;
return ReplicationHealQueueResult {
object_info: roi,
admission: ReplicationQueueAdmission::Skipped,
};
}
// early return if replication already done, otherwise we need to determine if this
@@ -1178,7 +1238,10 @@ pub async fn queue_replication_heal_internal(
&& roi.version_purge_status.is_empty()
&& !roi.existing_obj_resync.must_resync()
{
return roi;
return ReplicationHealQueueResult {
object_info: roi,
admission: ReplicationQueueAdmission::Skipped,
};
}
if roi.delete_marker || !roi.version_purge_status.is_empty() {
@@ -1210,10 +1273,15 @@ pub async fn queue_replication_heal_internal(
|| roi.version_purge_status == VersionPurgeStatusType::Failed
|| roi.version_purge_status == VersionPurgeStatusType::Pending
{
if let Some(pool) = GLOBAL_REPLICATION_POOL.get() {
pool.queue_replica_delete_task(dv).await;
}
return roi;
let admission = if let Some(pool) = GLOBAL_REPLICATION_POOL.get() {
pool.queue_replica_delete_task(dv).await
} else {
ReplicationQueueAdmission::Missed
};
return ReplicationHealQueueResult {
object_info: roi,
admission,
};
}
// if replication status is Complete on DeleteMarker and existing object resync required
@@ -1221,11 +1289,17 @@ pub async fn queue_replication_heal_internal(
if existing_obj_resync.must_resync()
&& (roi.replication_status == ReplicationStatusType::Completed || roi.replication_status.is_empty())
{
queue_replicate_deletes_wrapper(dv, existing_obj_resync).await;
return roi;
let admission = queue_replicate_deletes_wrapper(dv, existing_obj_resync).await;
return ReplicationHealQueueResult {
object_info: roi,
admission,
};
}
return roi;
return ReplicationHealQueueResult {
object_info: roi,
admission: ReplicationQueueAdmission::Skipped,
};
}
if roi.existing_obj_resync.must_resync() {
@@ -1235,34 +1309,72 @@ pub async fn queue_replication_heal_internal(
match roi.replication_status {
ReplicationStatusType::Pending | ReplicationStatusType::Failed => {
roi.event_type = REPLICATE_HEAL.to_string();
if let Some(pool) = GLOBAL_REPLICATION_POOL.get() {
pool.queue_replica_task(roi.clone()).await;
}
return roi;
let admission = if let Some(pool) = GLOBAL_REPLICATION_POOL.get() {
pool.queue_replica_task(roi.clone()).await
} else {
ReplicationQueueAdmission::Missed
};
return ReplicationHealQueueResult {
object_info: roi,
admission,
};
}
_ => {}
}
if roi.existing_obj_resync.must_resync() {
roi.event_type = REPLICATE_EXISTING.to_string();
if let Some(pool) = GLOBAL_REPLICATION_POOL.get() {
pool.queue_replica_task(roi.clone()).await;
}
let admission = if let Some(pool) = GLOBAL_REPLICATION_POOL.get() {
pool.queue_replica_task(roi.clone()).await
} else {
ReplicationQueueAdmission::Missed
};
return ReplicationHealQueueResult {
object_info: roi,
admission,
};
}
roi
ReplicationHealQueueResult {
object_info: roi,
admission: ReplicationQueueAdmission::Skipped,
}
}
/// Wrapper function for queueing replicate deletes with resync decision
async fn queue_replicate_deletes_wrapper(doi: DeletedObjectReplicationInfo, existing_obj_resync: ResyncDecision) {
async fn queue_replicate_deletes_wrapper(
doi: DeletedObjectReplicationInfo,
existing_obj_resync: ResyncDecision,
) -> ReplicationQueueAdmission {
let mut admission = ReplicationQueueAdmission::Skipped;
for (k, v) in existing_obj_resync.targets.iter() {
if v.replicate {
let mut dv = doi.clone();
dv.reset_id = v.reset_id.clone();
dv.target_arn = k.clone();
if let Some(pool) = GLOBAL_REPLICATION_POOL.get() {
pool.queue_replica_delete_task(dv).await;
}
let target_admission = if let Some(pool) = GLOBAL_REPLICATION_POOL.get() {
pool.queue_replica_delete_task(dv).await
} else {
ReplicationQueueAdmission::Missed
};
admission.merge(target_admission);
}
}
admission
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn replication_queue_admission_combines_target_results() {
let mut admission = ReplicationQueueAdmission::Skipped;
admission.merge(ReplicationQueueAdmission::Queued);
assert_eq!(admission, ReplicationQueueAdmission::Queued);
admission.merge(ReplicationQueueAdmission::Missed);
assert_eq!(admission, ReplicationQueueAdmission::Missed);
}
}
@@ -56,6 +56,7 @@ async fn peek_with_timeout<R: AsyncRead + Unpin>(reader: &mut MetacacheReader<R>
pub(crate) enum TestReaderBehavior {
Eof,
Stall,
IgnoreCancel,
ProducerError(DiskError),
PartialThenTimeout(Vec<MetaCacheEntry>),
}
@@ -72,6 +73,7 @@ pub struct ListPathRawOptions {
pub min_disks: usize,
pub report_not_found: bool,
pub per_disk_limit: i32,
pub skip_walkdir_total_timeout: bool,
pub agreed: Option<AgreedFn>,
pub partial: Option<PartialFn>,
pub finished: Option<FinishedFn>,
@@ -97,6 +99,7 @@ impl Clone for ListPathRawOptions {
min_disks: self.min_disks,
report_not_found: self.report_not_found,
per_disk_limit: self.per_disk_limit,
skip_walkdir_total_timeout: self.skip_walkdir_total_timeout,
#[cfg(test)]
test_reader_behaviors: self.test_reader_behaviors.clone(),
#[cfg(test)]
@@ -137,6 +140,11 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
cancel_rx_clone.cancelled().await;
return Ok(());
}
TestReaderBehavior::IgnoreCancel => {
let _held_writer = wr;
std::future::pending::<()>().await;
return Ok(());
}
TestReaderBehavior::ProducerError(err) => {
record_producer_error(&producer_errs_clone, disk_idx, &err);
return Err(err);
@@ -162,6 +170,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
filter_prefix: opts_clone.filter_prefix.clone(),
forward_to: opts_clone.forward_to.clone(),
limit: opts_clone.per_disk_limit,
skip_total_timeout: opts_clone.skip_walkdir_total_timeout,
..Default::default()
};
@@ -213,6 +222,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
filter_prefix: opts_clone.filter_prefix.clone(),
forward_to: opts_clone.forward_to.clone(),
limit: opts_clone.per_disk_limit,
skip_total_timeout: opts_clone.skip_walkdir_total_timeout,
..Default::default()
},
&mut wr,
@@ -485,6 +495,9 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
if let Err(err) = revjob.await.map_err(std::io::Error::other)? {
error!("list_path_raw: revjob err {:?}", err);
cancel_rx.cancel();
for job in jobs {
job.abort();
}
return Err(err);
}
@@ -492,9 +505,15 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
// The merge consumer can finish successfully before every producer finishes
// (for example after reaching EOF quorum while a tolerated drive is stalled,
// or after the requested listing limit is satisfied). Cancel remaining walk
// jobs before joining them so list calls do not wait for slow remote streams.
// jobs before aborting them so list calls do not wait for slow remote streams.
cancel_rx.cancel();
for job in jobs.iter() {
if !job.is_finished() {
job.abort();
}
}
let results = join_all(jobs).await;
let mut job_errs = Vec::new();
for result in results {
@@ -509,6 +528,9 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
job_errs.push(err);
}
Err(err) => {
if err.is_cancelled() {
continue;
}
error!("list_path_raw join err {:?}", err);
job_errs.push(err.into());
}
@@ -582,6 +604,31 @@ mod tests {
.expect("listing should complete when healthy quorum reached EOF and only a tolerated drive stalled");
}
#[tokio::test]
async fn list_path_raw_aborts_unresponsive_producer_after_quorum_eof() {
let result = timeout(
Duration::from_millis(200),
list_path_raw(
CancellationToken::new(),
ListPathRawOptions {
disks: vec![None, None, None],
min_disks: 2,
test_reader_behaviors: vec![
TestReaderBehavior::Eof,
TestReaderBehavior::Eof,
TestReaderBehavior::IgnoreCancel,
],
peek_timeout: Some(Duration::from_millis(20)),
..Default::default()
},
),
)
.await;
let listing = result.expect("list_path_raw should abort unresponsive producer instead of hanging");
assert!(listing.is_ok());
}
#[tokio::test]
async fn list_path_raw_returns_timeout_when_producer_fails_after_partial_entry() {
let seen = Arc::new(Mutex::new(Vec::new()));
+37 -17
View File
@@ -17,23 +17,23 @@ use rustfs_config::audit::AUDIT_REDIS_DEFAULT_CHANNEL;
use rustfs_config::{
AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_QUEUE_LIMIT, AMQP_ROUTING_KEY,
AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY,
EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA,
KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL,
MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT,
MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST,
MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA,
MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR,
NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL,
WEBHOOK_SKIP_TLS_VERIFY,
EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_SASL_ENABLE,
KAFKA_SASL_MECHANISM, KAFKA_SASL_PASSWORD, KAFKA_SASL_USERNAME, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY,
KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT,
MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT,
MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING,
POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT,
POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR,
PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC,
PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS,
REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD, REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT,
REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT,
REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL, REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA,
WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR,
WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
};
use std::sync::LazyLock;
@@ -636,6 +636,26 @@ pub static DEFAULT_AUDIT_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: KAFKA_SASL_ENABLE.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: KAFKA_SASL_MECHANISM.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: KAFKA_SASL_USERNAME.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: KAFKA_SASL_PASSWORD.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: KAFKA_QUEUE_DIR.to_owned(),
value: EVENT_DEFAULT_DIR.to_owned(),
+11
View File
@@ -12,10 +12,21 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::config::{KV, KVS};
use crate::error::{Error, Result};
use rustfs_config::{DEFAULT_HEAL_BITROT_CYCLE_SECS, HEAL_BITROT_CYCLE};
use rustfs_utils::string::parse_bool;
use std::sync::LazyLock;
use std::time::Duration;
pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
KVS(vec![KV {
key: HEAL_BITROT_CYCLE.to_owned(),
value: DEFAULT_HEAL_BITROT_CYCLE_SECS.to_string(),
hidden_if_empty: false,
}])
});
#[derive(Debug, Default)]
pub struct Config {
pub bitrot: String,
+14 -1
View File
@@ -26,6 +26,7 @@ use crate::store::ECStore;
use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
use rustfs_config::COMMENT_KEY;
use rustfs_config::DEFAULT_DELIMITER;
use rustfs_config::HEAL_SUB_SYS;
use rustfs_config::audit::{
AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_SUB_SYS,
AUDIT_PULSAR_SUB_SYS, AUDIT_REDIS_SUB_SYS, AUDIT_WEBHOOK_SUB_SYS,
@@ -256,6 +257,7 @@ pub fn init() {
// Load storageclass default configuration
kvs.insert(STORAGE_CLASS_SUB_SYS.to_owned(), storageclass::DEFAULT_KVS.clone());
kvs.insert(rustfs_config::SCANNER_SUB_SYS.to_owned(), scanner::DEFAULT_KVS.clone());
kvs.insert(HEAL_SUB_SYS.to_owned(), heal::DEFAULT_KVS.clone());
// New: Loading default configurations for notify_webhook and notify_mqtt
// Referring subsystem names through constants to improve the readability and maintainability of the code
kvs.insert(NOTIFY_WEBHOOK_SUB_SYS.to_owned(), notify::DEFAULT_NOTIFY_WEBHOOK_KVS.clone());
@@ -285,7 +287,10 @@ pub fn init() {
#[cfg(test)]
mod tests {
use super::*;
use rustfs_config::{DEFAULT_DELIMITER, DEFAULT_SCANNER_SPEED, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_SPEED, SCANNER_SUB_SYS};
use rustfs_config::{
DEFAULT_DELIMITER, DEFAULT_HEAL_BITROT_CYCLE_SECS, DEFAULT_SCANNER_SPEED, HEAL_BITROT_CYCLE, SCANNER_CYCLE_MAX_OBJECTS,
SCANNER_DELAY, SCANNER_MAX_WAIT, SCANNER_SPEED, SCANNER_SUB_SYS,
};
#[test]
fn global_server_config_set_and_get_roundtrip() {
@@ -313,6 +318,14 @@ mod tests {
.expect("scanner defaults should exist");
assert_eq!(scanner_kvs.get(SCANNER_SPEED), DEFAULT_SCANNER_SPEED);
assert_eq!(scanner_kvs.get(SCANNER_DELAY), "");
assert_eq!(scanner_kvs.get(SCANNER_MAX_WAIT), "");
assert_eq!(scanner_kvs.get(SCANNER_CYCLE_MAX_OBJECTS), "0");
let heal_kvs = cfg
.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER)
.expect("heal defaults should exist");
assert_eq!(heal_kvs.get(HEAL_BITROT_CYCLE), DEFAULT_HEAL_BITROT_CYCLE_SECS.to_string());
}
}
+36 -16
View File
@@ -17,22 +17,22 @@ use rustfs_config::notify::NOTIFY_REDIS_DEFAULT_CHANNEL;
use rustfs_config::{
AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_QUEUE_LIMIT, AMQP_ROUTING_KEY,
AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY,
EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_TLS_CA,
KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY, KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL,
MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT,
MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST,
MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA,
MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR,
NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT,
WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_SASL_ENABLE,
KAFKA_SASL_MECHANISM, KAFKA_SASL_PASSWORD, KAFKA_SASL_USERNAME, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY,
KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT,
MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT,
MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT,
NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING,
POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT,
POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR,
PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC,
PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT, REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS,
REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD, REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT,
REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT,
REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL, REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA,
WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
};
use std::sync::LazyLock;
@@ -613,6 +613,26 @@ pub static DEFAULT_NOTIFY_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: KAFKA_SASL_ENABLE.to_owned(),
value: EnableState::Off.to_string(),
hidden_if_empty: false,
},
KV {
key: KAFKA_SASL_MECHANISM.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: KAFKA_SASL_USERNAME.to_owned(),
value: "".to_owned(),
hidden_if_empty: false,
},
KV {
key: KAFKA_SASL_PASSWORD.to_owned(),
value: "".to_owned(),
hidden_if_empty: true,
},
KV {
key: KAFKA_QUEUE_DIR.to_owned(),
value: EVENT_DEFAULT_DIR.to_owned(),
+12 -2
View File
@@ -19,8 +19,8 @@ use rustfs_config::{
DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS, DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS, DEFAULT_SCANNER_SPEED,
DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, SCANNER_ALERT_EXCESS_FOLDERS, SCANNER_ALERT_EXCESS_VERSION_SIZE,
SCANNER_ALERT_EXCESS_VERSIONS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE,
SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_IDLE_MODE,
SCANNER_MAX_CONCURRENT_DISK_SCANS, SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_SPEED, SCANNER_START_DELAY,
SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE,
SCANNER_MAX_CONCURRENT_DISK_SCANS, SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_MAX_WAIT, SCANNER_SPEED, SCANNER_START_DELAY,
SCANNER_YIELD_EVERY_N_OBJECTS,
};
use std::sync::LazyLock;
@@ -32,6 +32,16 @@ pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
value: DEFAULT_SCANNER_SPEED.to_owned(),
hidden_if_empty: false,
},
KV {
key: SCANNER_DELAY.to_owned(),
value: String::new(),
hidden_if_empty: true,
},
KV {
key: SCANNER_MAX_WAIT.to_owned(),
value: String::new(),
hidden_if_empty: true,
},
KV {
key: SCANNER_CYCLE.to_owned(),
value: String::new(),
+354 -16
View File
@@ -12,11 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::error::{Error, Result};
use crate::error::{Error, Result, is_err_data_movement_overwrite, is_err_object_not_found, is_err_version_not_found};
use crate::store::ECStore;
use crate::store_api::{CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader};
use crate::store_api::{
CompletePart, GetObjectReader, MultipartOperations, ObjectIO, ObjectInfo, ObjectOperations, ObjectOptions, PutObjReader,
};
use bytes::Bytes;
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
use rustfs_utils::path::encode_dir_object;
use std::io::Cursor;
use std::pin::Pin;
use std::sync::{
@@ -102,13 +105,14 @@ fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usiz
}
}
fn data_movement_complete_multipart_opts(object_info: &ObjectInfo) -> ObjectOptions {
fn data_movement_complete_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usize) -> ObjectOptions {
ObjectOptions {
versioned: object_info.version_id.is_some(),
version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
data_movement: true,
mod_time: object_info.mod_time,
preserve_etag: object_info.etag.clone(),
src_pool_idx,
..Default::default()
}
}
@@ -139,6 +143,111 @@ fn resolve_data_movement_abort_result(
))
}
fn data_movement_stage_error(op_label: &str, stage: &str, bucket: &str, object: &str, err: impl std::fmt::Display) -> Error {
Error::other(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"))
}
fn should_check_data_movement_overwrite_resume(err: &Error) -> bool {
is_err_data_movement_overwrite(err)
}
fn effective_actual_size(info: &ObjectInfo) -> Option<i64> {
info.get_actual_size().ok()
}
fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool {
source.version_id == target.version_id
&& source.delete_marker == target.delete_marker
&& source.size == target.size
&& effective_actual_size(source) == effective_actual_size(target)
&& source.etag == target.etag
&& source.checksum == target.checksum
&& source.mod_time == target.mod_time
}
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
target_pool_idx != src_pool_idx
}
async fn find_data_movement_target_info(
store: &ECStore,
target_pool_idx: usize,
bucket: &str,
object_info: &ObjectInfo,
) -> Result<Option<ObjectInfo>> {
let opts = ObjectOptions {
versioned: object_info.version_id.is_some(),
version_id: object_info.version_id.as_ref().map(|v| v.to_string()),
no_lock: true,
..Default::default()
};
let object = encode_dir_object(object_info.name.as_str());
let Some(pool) = store.pools.get(target_pool_idx) else {
return Err(Error::other(format!(
"data movement resume target pool {target_pool_idx} is out of range for {bucket}/{object}"
)));
};
match pool.get_object_info(bucket, object.as_str(), &opts).await {
Ok(target_info) => Ok(Some(target_info)),
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
Err(err) => Err(err),
}
}
fn resolve_data_movement_overwrite_resume_result(
err: &Error,
target_result: Result<Option<ObjectInfo>>,
source: &ObjectInfo,
src_pool_idx: usize,
target_pool_idx: usize,
) -> Result<bool> {
if !should_check_data_movement_overwrite_resume(err)
|| !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx)
{
return Ok(false);
}
let Some(target) = target_result? else {
return Ok(false);
};
Ok(is_equivalent_data_movement_object(source, &target))
}
async fn should_treat_data_movement_overwrite_as_complete(
store: &ECStore,
src_pool_idx: usize,
target_pool_idx: usize,
bucket: &str,
object_info: &ObjectInfo,
err: &Error,
) -> Result<bool> {
if !should_check_data_movement_overwrite_resume(err) {
return Ok(false);
}
resolve_data_movement_overwrite_resume_result(
err,
find_data_movement_target_info(store, target_pool_idx, bucket, object_info).await,
object_info,
src_pool_idx,
target_pool_idx,
)
}
fn data_movement_part_stage_error(
op_label: &str,
stage: &str,
bucket: &str,
object: &str,
part_number: usize,
err: impl std::fmt::Display,
) -> Error {
Error::other(format!("{op_label}: {stage} failed for {bucket}/{object} part {part_number}: {err}"))
}
pub(crate) async fn migrate_object(
store: Arc<ECStore>,
pool_idx: usize,
@@ -149,14 +258,24 @@ pub(crate) async fn migrate_object(
let object_info = rd.object_info.clone();
if object_info.is_multipart() {
let res = match store
.new_multipart_upload(&bucket, &object_info.name, &data_movement_new_multipart_opts(&object_info, pool_idx))
let (res, target_pool_idx) = match store
.handle_new_multipart_upload_with_pool_idx(
&bucket,
&object_info.name,
&data_movement_new_multipart_opts(&object_info, pool_idx),
)
.await
{
Ok(res) => res,
Err(err) => {
error!("{op_label}: new_multipart_upload err {:?}", &err);
return Err(err);
return Err(data_movement_stage_error(
op_label,
"new_multipart_upload",
bucket.as_str(),
object_info.name.as_str(),
err,
));
}
};
@@ -167,12 +286,39 @@ pub(crate) async fn migrate_object(
for (i, part) in object_info.parts.iter().enumerate() {
let mut chunk = vec![0u8; part.size];
reader.read_exact(&mut chunk).await?;
reader.read_exact(&mut chunk).await.map_err(|err| {
data_movement_part_stage_error(
op_label,
"read_part",
bucket.as_str(),
object_info.name.as_str(),
part.number,
Error::other(err.to_string()),
)
})?;
let part_size = i64::try_from(part.size).map_err(|_| Error::other("part size overflow"))?;
let part_size = i64::try_from(part.size).map_err(|_| {
data_movement_part_stage_error(
op_label,
"prepare_part",
bucket.as_str(),
object_info.name.as_str(),
part.number,
Error::other("part size overflow"),
)
})?;
let part_actual_size = if part.actual_size > 0 { part.actual_size } else { part_size };
let index = decode_part_index(part.index.as_ref());
let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index)?;
let mut data = put_obj_reader_from_chunk(chunk, part_size, part_actual_size, index).map_err(|err| {
data_movement_part_stage_error(
op_label,
"prepare_part",
bucket.as_str(),
object_info.name.as_str(),
part.number,
err,
)
})?;
let pi = match store
.put_object_part(
@@ -191,7 +337,14 @@ pub(crate) async fn migrate_object(
Ok(pi) => pi,
Err(err) => {
error!("{op_label}: put_object_part {i} err {:?}", &err);
return Err(err);
return Err(data_movement_part_stage_error(
op_label,
"put_object_part",
bucket.as_str(),
object_info.name.as_str(),
part.number,
err,
));
}
};
@@ -209,12 +362,32 @@ pub(crate) async fn migrate_object(
&object_info.name,
&res.upload_id,
parts,
&data_movement_complete_multipart_opts(&object_info),
&data_movement_complete_multipart_opts(&object_info, pool_idx),
)
.await
{
if should_treat_data_movement_overwrite_as_complete(
store.as_ref(),
pool_idx,
target_pool_idx,
bucket.as_str(),
&object_info,
&err,
)
.await?
{
mark_multipart_upload_completed(&abort_multipart_flag);
return Ok(());
}
error!("{op_label}: complete_multipart_upload err {:?}", &err);
return Err(err);
return Err(data_movement_stage_error(
op_label,
"complete_multipart_upload",
bucket.as_str(),
object_info.name.as_str(),
err,
));
}
mark_multipart_upload_completed(&abort_multipart_flag);
@@ -248,13 +421,18 @@ pub(crate) async fn migrate_object(
return Ok(());
}
let actual_size = object_info.get_actual_size()?;
let actual_size = object_info.get_actual_size().map_err(|err| {
data_movement_stage_error(op_label, "prepare_put_object", bucket.as_str(), object_info.name.as_str(), err)
})?;
let index = object_info
.parts
.first()
.and_then(|part| decode_part_index(part.index.as_ref()));
let reader = IndexedDataMovementReader::new(BufReader::new(rd.stream), index);
let hrd = HashReader::from_stream(reader, object_info.size, actual_size, object_info.etag.clone(), None, false)?;
let hrd =
HashReader::from_stream(reader, object_info.size, actual_size, object_info.etag.clone(), None, false).map_err(|err| {
data_movement_stage_error(op_label, "prepare_put_object", bucket.as_str(), object_info.name.as_str(), err)
})?;
let mut data = PutObjReader::new(hrd);
if let Err(err) = store
@@ -267,7 +445,13 @@ pub(crate) async fn migrate_object(
.await
{
error!("{op_label}: put_object err {:?}", &err);
return Err(err);
return Err(data_movement_stage_error(
op_label,
"put_object",
bucket.as_str(),
object_info.name.as_str(),
err,
));
}
Ok(())
@@ -309,6 +493,33 @@ mod tests {
assert!(message.contains(Error::SlowDown.to_string().as_str()));
}
#[test]
fn test_data_movement_stage_error_includes_stage_and_object() {
let err = data_movement_stage_error("rebalance_object", "put_object", "bucket-a", "object-a", Error::SlowDown);
let message = err.to_string();
assert!(message.contains("rebalance_object: put_object failed for bucket-a/object-a"));
assert!(message.contains(Error::SlowDown.to_string().as_str()));
}
#[test]
fn test_data_movement_part_stage_error_includes_stage_object_and_part() {
let err =
data_movement_part_stage_error("rebalance_object", "put_object_part", "bucket-a", "object-a", 7, Error::SlowDown);
let message = err.to_string();
assert!(message.contains("rebalance_object: put_object_part failed for bucket-a/object-a part 7"));
assert!(message.contains(Error::SlowDown.to_string().as_str()));
}
#[test]
fn test_should_check_data_movement_overwrite_resume_only_for_overwrite_error() {
assert!(should_check_data_movement_overwrite_resume(&Error::DataMovementOverwriteErr(
"bucket-a".to_string(),
"object-a".to_string(),
"version-a".to_string(),
)));
assert!(!should_check_data_movement_overwrite_resume(&Error::SlowDown));
}
#[test]
fn test_decode_part_index_returns_none_when_absent() {
assert!(decode_part_index(None).is_none());
@@ -366,13 +577,14 @@ mod tests {
..Default::default()
};
let opts = data_movement_complete_multipart_opts(&object_info);
let opts = data_movement_complete_multipart_opts(&object_info, 7);
assert!(opts.versioned);
assert!(opts.data_movement);
assert_eq!(opts.mod_time, Some(mod_time));
assert_eq!(opts.version_id.as_deref(), Some(version_id.to_string().as_str()));
assert_eq!(opts.preserve_etag.as_deref(), Some("etag-value"));
assert_eq!(opts.src_pool_idx, 7);
}
#[test]
@@ -396,4 +608,130 @@ mod tests {
assert!(opts.data_movement);
assert_eq!(opts.mod_time, object_info.mod_time);
}
#[test]
fn test_is_equivalent_data_movement_object_accepts_matching_metadata() {
let version_id = Uuid::nil();
let info = ObjectInfo {
version_id: Some(version_id),
size: 128,
actual_size: 96,
etag: Some("etag-value".to_string()),
checksum: Some(Bytes::from_static(b"checksum")),
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
assert!(is_equivalent_data_movement_object(&info, &info.clone()));
}
#[test]
fn test_is_equivalent_data_movement_object_rejects_content_mismatch() {
let source = ObjectInfo {
version_id: Some(Uuid::nil()),
size: 128,
actual_size: 96,
etag: Some("etag-source".to_string()),
checksum: Some(Bytes::from_static(b"checksum-source")),
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let target = ObjectInfo {
etag: Some("etag-target".to_string()),
checksum: Some(Bytes::from_static(b"checksum-target")),
..source.clone()
};
assert!(!is_equivalent_data_movement_object(&source, &target));
}
#[test]
fn test_is_equivalent_data_movement_object_uses_effective_actual_size() {
let source = ObjectInfo {
size: 128,
actual_size: 0,
etag: Some("etag-value".to_string()),
..Default::default()
};
let target = ObjectInfo {
size: 128,
actual_size: 128,
etag: Some("etag-value".to_string()),
..Default::default()
};
assert!(is_equivalent_data_movement_object(&source, &target));
}
#[test]
fn test_resolve_data_movement_overwrite_resume_result_accepts_equivalent_target() {
let source = ObjectInfo {
version_id: Some(Uuid::nil()),
size: 128,
etag: Some("etag-value".to_string()),
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(source.clone())), &source, 0, 1)
.expect("equivalent overwrite target should be evaluated");
assert!(should_resume);
}
#[test]
fn test_resolve_data_movement_overwrite_resume_result_rejects_source_pool_target() {
let source = ObjectInfo {
version_id: Some(Uuid::nil()),
size: 128,
etag: Some("etag-value".to_string()),
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(source.clone())), &source, 0, 0)
.expect("source-pool target should be rejected before target lookup");
assert!(!should_resume);
}
#[test]
fn test_resolve_data_movement_overwrite_resume_result_rejects_non_equivalent_target() {
let source = ObjectInfo {
version_id: Some(Uuid::nil()),
size: 128,
etag: Some("etag-source".to_string()),
..Default::default()
};
let target = ObjectInfo {
etag: Some("etag-target".to_string()),
..source.clone()
};
let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
let should_resume = resolve_data_movement_overwrite_resume_result(&err, Ok(Some(target)), &source, 0, 1)
.expect("non-equivalent overwrite target should be evaluated");
assert!(!should_resume);
}
#[test]
fn test_resolve_data_movement_overwrite_resume_result_propagates_target_lookup_error() {
let source = ObjectInfo::default();
let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
let result = resolve_data_movement_overwrite_resume_result(&err, Err(Error::SlowDown), &source, 0, 1);
assert!(matches!(result, Err(Error::SlowDown)));
}
#[test]
fn test_resolve_data_movement_overwrite_resume_result_ignores_non_overwrite_error() {
let source = ObjectInfo::default();
let result = resolve_data_movement_overwrite_resume_result(&Error::SlowDown, Err(Error::FileAccessDenied), &source, 0, 1)
.expect("non-overwrite errors should not query target equivalence");
assert!(!result);
}
}
+53 -1
View File
@@ -1136,10 +1136,16 @@ impl DiskAPI for LocalDiskWrapper {
}
async fn walk_dir<W: tokio::io::AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
let timeout_duration = if opts.skip_total_timeout {
Duration::ZERO
} else {
get_drive_walkdir_timeout()
};
self.track_disk_health_with_op_and_timeout_action(
"walk_dir",
|| async { self.disk.walk_dir(opts, wr).await },
get_drive_walkdir_timeout(),
timeout_duration,
// Listing/scanner backpressure should fail only the current walk, not poison drive health.
TimeoutHealthAction::IgnoreFailure,
)
@@ -1647,6 +1653,52 @@ mod tests {
.await;
}
#[tokio::test]
async fn walk_dir_skip_total_timeout_keeps_stream_pending() {
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("1"))], async {
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint =
Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")).expect("endpoint should parse");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
let wrapper = LocalDiskWrapper::new(disk, false);
let bucket = "test-bucket";
let object = "test-object";
wrapper.make_volume(bucket).await.expect("bucket should be created");
let mut file_info = FileInfo::new(&format!("{bucket}/{object}"), 1, 0);
file_info.volume = bucket.to_string();
file_info.name = object.to_string();
file_info.mod_time = Some(::time::OffsetDateTime::now_utc());
file_info.erasure.index = 1;
wrapper
.write_metadata("", bucket, object, file_info)
.await
.expect("object metadata should be written");
let mut writer = PendingWriter;
let result = tokio::time::timeout(
Duration::from_millis(20),
wrapper.walk_dir(
WalkDirOptions {
bucket: bucket.to_string(),
recursive: true,
skip_total_timeout: true,
..Default::default()
},
&mut writer,
),
)
.await;
assert!(result.is_err(), "skip_total_timeout should leave backpressured walk pending");
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
assert!(!wrapper.health.is_faulty());
})
.await;
}
#[tokio::test]
async fn walk_dir_timeout_does_not_break_followup_stat_volume() {
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("1"))], async {
+25 -1
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_rio::{InternodeHttpError, InternodeHttpErrorKind};
use std::hash::{Hash, Hasher};
use std::io::{self};
use std::path::PathBuf;
@@ -181,6 +182,26 @@ impl DiskError {
matches!(err, &DiskError::FileVersionNotFound)
}
pub fn is_retryable_internode_write_failure(&self) -> bool {
match self {
DiskError::Io(io_error) => io_error
.get_ref()
.and_then(|source| source.downcast_ref::<InternodeHttpError>())
.is_some_and(|err| err.kind().is_retryable()),
_ => false,
}
}
pub fn internode_http_error_kind(&self) -> Option<InternodeHttpErrorKind> {
match self {
DiskError::Io(io_error) => io_error
.get_ref()
.and_then(|source| source.downcast_ref::<InternodeHttpError>())
.map(InternodeHttpError::kind),
_ => None,
}
}
// /// If all errors are of the same fatal disk error type, returns the corresponding error.
// /// Otherwise, returns Ok.
// pub fn check_disk_fatal_errs(errs: &[Option<Error>]) -> Result<()> {
@@ -241,7 +262,10 @@ impl From<rustfs_filemeta::Error> for DiskError {
impl From<std::io::Error> for DiskError {
fn from(e: std::io::Error) -> Self {
e.downcast::<DiskError>().unwrap_or_else(DiskError::Io)
match e.downcast::<DiskError>() {
Ok(disk_error) => disk_error,
Err(io_error) => DiskError::Io(io_error),
}
}
}
+118
View File
@@ -14,6 +14,19 @@
use crate::disk::error::Error;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WriteQuorumFailureSummary {
pub required: usize,
pub achieved: usize,
pub failed: usize,
pub total: usize,
pub offline_disks: usize,
pub ignored_failures: usize,
pub retryable_failures: usize,
pub dominant_error: Option<Error>,
pub dominant_error_label: &'static str,
}
pub static OBJECT_OP_IGNORED_ERRS: &[Error] = &[
Error::DiskNotFound,
Error::FaultyDisk,
@@ -77,6 +90,58 @@ pub fn reduce_errs(errors: &[Option<Error>], ignored_errs: &[Error]) -> (usize,
}
}
pub fn build_write_quorum_failure_summary(
errors: &[Option<Error>],
ignored_errs: &[Error],
quorum: usize,
) -> WriteQuorumFailureSummary {
let total = errors.len();
let achieved = errors.iter().filter(|err| err.is_none()).count();
let failed = total.saturating_sub(achieved);
let offline_disks = count_errs(errors, &Error::DiskNotFound);
let ignored_failures = errors
.iter()
.filter_map(|err| err.as_ref())
.filter(|err| is_ignored_err(ignored_errs, err))
.count();
let retryable_failures = count_retryable_failures(errors);
let (_, dominant_error) = reduce_errs(errors, ignored_errs);
let dominant_error_label = dominant_error_label(errors, ignored_errs, dominant_error.as_ref());
WriteQuorumFailureSummary {
required: quorum,
achieved,
failed,
total,
offline_disks,
ignored_failures,
retryable_failures,
dominant_error,
dominant_error_label,
}
}
fn dominant_error_label(errors: &[Option<Error>], ignored_errs: &[Error], dominant_error: Option<&Error>) -> &'static str {
let Some(dominant_error) = dominant_error else {
return "nil_dominated";
};
if dominant_error == &Error::DiskNotFound {
return "disk_not_found";
}
if dominant_error == &Error::ShortWrite {
return "short_write";
}
errors
.iter()
.filter_map(|err| err.as_ref())
.find(|err| !is_ignored_err(ignored_errs, err) && *err == dominant_error)
.and_then(Error::internode_http_error_kind)
.map(|kind| kind.metric_label())
.unwrap_or("other_error")
}
pub fn is_ignored_err(ignored_errs: &[Error], err: &Error) -> bool {
ignored_errs.iter().any(|e| e == err)
}
@@ -85,6 +150,14 @@ pub fn count_errs(errors: &[Option<Error>], err: &Error) -> usize {
errors.iter().filter(|&e| e.as_ref() == Some(err)).count()
}
pub fn count_retryable_failures(errors: &[Option<Error>]) -> usize {
errors
.iter()
.filter_map(|err| err.as_ref())
.filter(|err| err.is_retryable_internode_write_failure())
.count()
}
pub fn is_all_buckets_not_found(errs: &[Option<Error>]) -> bool {
for err in errs.iter() {
if let Some(err) = err {
@@ -163,6 +236,51 @@ mod tests {
assert!(!is_ignored_err(&ignored, &e2));
}
#[test]
fn test_build_write_quorum_failure_summary() {
let retryable = Error::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::ConnectionReset,
));
let non_retryable = err_io("other");
let errors = vec![
None,
None,
None,
None,
None,
Some(retryable),
Some(non_retryable),
Some(Error::DiskNotFound),
];
let summary = build_write_quorum_failure_summary(&errors, OBJECT_OP_IGNORED_ERRS, 6);
assert_eq!(summary.required, 6);
assert_eq!(summary.achieved, 5);
assert_eq!(summary.failed, 3);
assert_eq!(summary.total, 8);
assert_eq!(summary.offline_disks, 1);
assert_eq!(summary.ignored_failures, 1);
assert_eq!(summary.retryable_failures, 1);
assert_eq!(summary.dominant_error, None);
assert_eq!(summary.dominant_error_label, "nil_dominated");
}
#[test]
fn test_build_write_quorum_failure_summary_preserves_internode_label() {
let retryable_a = Error::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::ConnectionReset,
));
let retryable_b = Error::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::ConnectionReset,
));
let errors = vec![Some(retryable_a), Some(retryable_b)];
let summary = build_write_quorum_failure_summary(&errors, OBJECT_OP_IGNORED_ERRS, 2);
assert_eq!(summary.retryable_failures, 2);
assert_eq!(summary.dominant_error_label, "connection_reset");
}
#[test]
fn test_reduce_errs_nil_tiebreak() {
// Error::Nil and another error have the same count, should prefer Nil
+198 -19
View File
@@ -1451,12 +1451,9 @@ impl LocalDisk {
let name = entry.trim_end_matches(SLASH_SEPARATOR);
let name = decode_dir_object(format!("{}/{}", &current, &name).as_str());
// if opts.limit > 0
// && let Ok(meta) = FileMeta::load(&metadata)
// && !meta.all_hidden(true)
// {
*objs_returned += 1;
// }
if opts.limit <= 0 || metadata_counts_toward_limit(&metadata) {
*objs_returned += 1;
}
out.write_obj(&MetaCacheEntry {
name: name.clone(),
@@ -1559,15 +1556,19 @@ impl LocalDisk {
out.write_obj(&meta).await?;
// if let Ok(meta) = FileMeta::load(&meta.metadata)
// && !meta.all_hidden(true)
// {
*objs_returned += 1;
// }
let file_meta = if opts.limit > 0 || opts.recursive {
FileMeta::load(&res).ok()
} else {
None
};
if opts.limit <= 0 || file_meta.as_ref().is_none_or(file_meta_counts_toward_limit) {
*objs_returned += 1;
}
if opts.recursive {
let mut dir_to_skip = HashSet::new();
if let Ok(file_meta) = FileMeta::load(&res)
if let Some(file_meta) = file_meta.as_ref()
&& let Ok(data_dirs) = file_meta.get_data_dirs()
{
for data_dir in data_dirs.iter().flatten() {
@@ -1636,6 +1637,15 @@ fn is_root_path(path: impl AsRef<Path>) -> bool {
path.as_ref().components().count() == 1 && path.as_ref().has_root()
}
fn metadata_counts_toward_limit(metadata: &[u8]) -> bool {
FileMeta::load(metadata).map_or(true, |meta| file_meta_counts_toward_limit(&meta))
}
fn file_meta_counts_toward_limit(meta: &FileMeta) -> bool {
meta.into_fileinfo("", "", "", false, true, false)
.map_or_else(|_| !meta.all_hidden(true), |latest| !latest.deleted && !latest.tier_free_version())
}
// Filter std::io::ErrorKind::NotFound
async fn read_file_exists(path: impl AsRef<Path>) -> Result<(Bytes, Option<Metadata>)> {
let p = path.as_ref();
@@ -1783,20 +1793,28 @@ impl DiskAPI for LocalDisk {
let id = format_info.id;
// if format_info.last_check_valid() {
// return Ok(id);
// }
if format_info.file_info.is_some() && id.is_some() {
// check last check time
// Reuse the cached disk id only when the cached format check is fresh.
if let Some(last_check) = format_info.last_check
&& last_check.unix_timestamp() + 1 < OffsetDateTime::now_utc().unix_timestamp()
&& last_check.unix_timestamp() + 1 >= OffsetDateTime::now_utc().unix_timestamp()
{
return Ok(id);
}
}
let file_meta = self.check_format_json().await?;
let file_meta = match self.check_format_json().await {
Ok(meta) => meta,
Err(err) => {
if matches!(err, DiskError::UnformattedDisk | DiskError::DiskNotFound) {
let mut format_info = self.format_info.write().await;
format_info.id = None;
format_info.data = Bytes::new();
format_info.file_info = None;
format_info.last_check = None;
}
return Err(err);
}
};
if let Some(file_info) = &format_info.file_info
&& super::fs::same_file(&file_meta, file_info)
@@ -3318,6 +3336,52 @@ mod test {
}
}
#[tokio::test]
async fn test_get_disk_id_invalidates_cache_after_format_removal() {
use crate::disk::FORMAT_CONFIG_FILE;
use crate::disk::format::FormatV3;
use tempfile::tempdir;
let dir = tempdir().unwrap();
let mut endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let meta_dir = dir.path().join(RUSTFS_META_BUCKET);
fs::create_dir_all(&meta_dir).await.expect("meta dir should be creatable");
let mut format = FormatV3::new(1, 1);
format.erasure.this = format.erasure.sets[0][0];
let format_json = format.to_json().expect("format should serialize");
fs::write(meta_dir.join(FORMAT_CONFIG_FILE), format_json)
.await
.expect("format.json should be writable");
let disk = LocalDisk::new(&endpoint, false)
.await
.expect("local disk should open after seeding format");
let initial_id = disk.get_disk_id().await.expect("disk id lookup should succeed");
assert!(initial_id.is_some(), "new disk should expose a disk id");
fs::remove_file(&disk.format_path)
.await
.expect("format.json should be removable");
tokio::time::sleep(Duration::from_secs(2)).await;
let err = disk
.get_disk_id()
.await
.expect_err("removed format.json should invalidate the cached disk id");
assert!(matches!(err, DiskError::UnformattedDisk));
let format_info = disk.format_info.read().await.clone();
assert!(format_info.id.is_none(), "cached disk id should be cleared");
assert!(format_info.data.is_empty(), "cached format bytes should be cleared");
assert!(format_info.file_info.is_none(), "cached file metadata should be cleared");
assert!(format_info.last_check.is_none(), "cached format timestamp should be cleared");
}
#[tokio::test]
async fn cleanup_tmp_on_startup_moves_existing_tmp_and_recreates_trash() {
use tempfile::tempdir;
@@ -3641,6 +3705,121 @@ mod test {
assert_eq!(double_count as usize, double_names.len());
}
#[tokio::test]
async fn test_scan_dir_hidden_delete_markers_do_not_exhaust_limit() {
use rustfs_filemeta::MetacacheReader;
use tempfile::tempdir;
fn delete_marker_metadata(version_id: &str) -> Vec<u8> {
let mut fm = FileMeta::default();
fm.add_version(FileInfo {
deleted: true,
version_id: Some(Uuid::parse_str(version_id).expect("test version id should parse")),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
})
.expect("delete marker metadata should be valid");
fm.marshal_msg().expect("delete marker metadata should encode")
}
fn delete_marker_with_old_object_metadata(delete_version_id: &str, object_version_id: &str) -> Vec<u8> {
let mut fm = FileMeta::default();
fm.add_version({
let mut fi = FileInfo::new("hidden", 1, 1);
fi.version_id = Some(Uuid::parse_str(object_version_id).expect("test version id should parse"));
fi.mod_time = Some(OffsetDateTime::now_utc() - time::Duration::seconds(1));
fi
})
.expect("object metadata should be valid");
fm.add_version(FileInfo {
deleted: true,
version_id: Some(Uuid::parse_str(delete_version_id).expect("test version id should parse")),
mod_time: Some(OffsetDateTime::now_utc()),
..Default::default()
})
.expect("delete marker metadata should be valid");
fm.marshal_msg().expect("delete marker metadata should encode")
}
fn object_metadata(version_id: &str) -> Vec<u8> {
let mut fm = FileMeta::default();
let mut fi = FileInfo::new("visible", 1, 1);
fi.version_id = Some(Uuid::parse_str(version_id).expect("test version id should parse"));
fi.mod_time = Some(OffsetDateTime::now_utc());
fm.add_version(fi).expect("object metadata should be valid");
fm.marshal_msg().expect("object metadata should encode")
}
let dir = tempdir().unwrap();
let bucket = "test-bucket";
let bucket_dir = dir.path().join(bucket);
for (name, version_id) in [
("shard/aaa-trash-0000", "11111111-1111-1111-1111-111111111111"),
("shard/aaa-trash-0001", "22222222-2222-2222-2222-222222222222"),
("shard/aaa-trash-0002", "33333333-3333-3333-3333-333333333333"),
] {
let object_dir = bucket_dir.join(name);
fs::create_dir_all(&object_dir).await.unwrap();
fs::write(object_dir.join(STORAGE_FORMAT_FILE), delete_marker_metadata(version_id))
.await
.unwrap();
}
let hidden_versioned_dir = bucket_dir.join("shard/aaa-trash-0003");
fs::create_dir_all(&hidden_versioned_dir).await.unwrap();
fs::write(
hidden_versioned_dir.join(STORAGE_FORMAT_FILE),
delete_marker_with_old_object_metadata(
"44444444-4444-4444-4444-444444444444",
"55555555-5555-5555-5555-555555555555",
),
)
.await
.unwrap();
let visible_dir = bucket_dir.join("shard/bbb-visible-0000");
fs::create_dir_all(&visible_dir).await.unwrap();
fs::write(
visible_dir.join(STORAGE_FORMAT_FILE),
object_metadata("66666666-6666-6666-6666-666666666666"),
)
.await
.unwrap();
let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
let disk = LocalDisk::new(&endpoint, false).await.unwrap();
let (reader, mut writer) = tokio::io::duplex(4096);
let mut out = MetacacheWriter::new(&mut writer);
let opts = WalkDirOptions {
bucket: bucket.to_string(),
base_dir: "".to_string(),
recursive: true,
limit: 1,
..Default::default()
};
let mut objs_returned = 0;
disk.scan_dir("".to_string(), "".to_string(), &opts, &mut out, &mut objs_returned, false, None)
.await
.unwrap();
out.close().await.unwrap();
drop(out);
drop(writer);
let mut reader = MetacacheReader::new(reader);
let has_visible_object = reader
.read_all()
.await
.unwrap()
.into_iter()
.any(|entry| !entry.metadata.is_empty() && entry.name == "shard/bbb-visible-0000");
assert!(has_visible_object);
assert_eq!(objs_returned, 1);
}
#[tokio::test]
async fn test_walk_dir_ignore_multipart_dirs() {
use rustfs_filemeta::MetacacheReader;
+6
View File
@@ -698,6 +698,10 @@ pub struct WalkDirOptions {
// DiskID contains the disk ID of the disk.
// Leave empty to not check disk ID.
pub disk_id: String,
// Skip the wrapper-level total timeout for long streaming walks.
#[serde(default)]
pub skip_total_timeout: bool,
}
#[derive(Clone, Debug, Default)]
@@ -894,6 +898,7 @@ mod tests {
forward_to: Some("object/path".to_string()),
limit: 100,
disk_id: "disk-123".to_string(),
skip_total_timeout: false,
};
assert_eq!(opts.bucket, "test-bucket");
@@ -904,6 +909,7 @@ mod tests {
assert_eq!(opts.forward_to, Some("object/path".to_string()));
assert_eq!(opts.limit, 100);
assert_eq!(opts.disk_id, "disk-123");
assert!(!opts.skip_total_timeout);
}
/// Test DeleteOptions structure
+72
View File
@@ -58,6 +58,40 @@ pub struct Node {
#[derive(Debug, Default, Clone)]
pub struct Endpoints(Vec<Endpoint>);
#[derive(Debug, Clone)]
struct LocalDiskValidationDiagnostic {
original_path: String,
canonical_path: Option<String>,
device_numbers: Option<String>,
device_ids: Option<Vec<String>>,
}
impl LocalDiskValidationDiagnostic {
fn new(original_path: &str) -> Self {
Self {
original_path: original_path.to_string(),
canonical_path: None,
device_numbers: None,
device_ids: None,
}
}
fn summary(&self) -> String {
let canonical_path = self.canonical_path.as_deref().unwrap_or("(unresolved)");
let device_numbers = self.device_numbers.as_deref().unwrap_or("(unavailable)");
let device_ids = self
.device_ids
.as_ref()
.map(|ids| ids.join(","))
.unwrap_or_else(|| "(unavailable)".to_string());
format!(
"path='{}', canonical='{}', st_dev='{}', device_ids=[{}]",
self.original_path, canonical_path, device_numbers, device_ids
)
}
}
impl AsRef<Vec<Endpoint>> for Endpoints {
fn as_ref(&self) -> &Vec<Endpoint> {
&self.0
@@ -683,10 +717,12 @@ fn validate_local_physical_disk_independence(pools: &[Endpoints]) -> Result<()>
}
let mut device_paths = BTreeMap::<String, BTreeSet<String>>::new();
let mut diagnostics = Vec::with_capacity(local_paths.len());
#[cfg(not(windows))]
let mut missing_paths = Vec::new();
for path in &local_paths {
let mut diagnostic = LocalDiskValidationDiagnostic::new(path);
let canonical = match rustfs_utils::canonicalize(path) {
Ok(path) => path,
Err(err) if err.kind() == ErrorKind::NotFound => {
@@ -697,8 +733,13 @@ fn validate_local_physical_disk_independence(pools: &[Endpoints]) -> Result<()>
match crate::disk::endpoint::windows_fallback_local_path(path, &err, "disk independence validation") {
Ok(absolute) => {
let abs_path = absolute.to_string_lossy().into_owned();
diagnostic.canonical_path = Some(abs_path.clone());
if let Ok(serial) = rustfs_utils::os::get_volume_serial_number(&abs_path) {
diagnostic.device_numbers = Some(format!("serial:{serial:#010x}"));
}
match rustfs_utils::os::get_physical_device_ids(&abs_path) {
Ok(ids) => {
diagnostic.device_ids = Some(ids.clone());
for device_id in ids {
device_paths.entry(device_id).or_default().insert(abs_path.clone());
}
@@ -709,6 +750,7 @@ fn validate_local_physical_disk_independence(pools: &[Endpoints]) -> Result<()>
)));
}
}
diagnostics.push(diagnostic);
continue;
}
Err(fallback_err) => {
@@ -721,6 +763,7 @@ fn validate_local_physical_disk_independence(pools: &[Endpoints]) -> Result<()>
#[cfg(not(windows))]
{
missing_paths.push(path.clone());
diagnostics.push(diagnostic);
continue;
}
}
@@ -731,9 +774,20 @@ fn validate_local_physical_disk_independence(pools: &[Endpoints]) -> Result<()>
}
};
let canonical_path = canonical.to_string_lossy().into_owned();
diagnostic.canonical_path = Some(canonical_path.clone());
#[cfg(not(windows))]
if let Ok(stat) = rustix::fs::stat(canonical.as_path()) {
diagnostic.device_numbers = Some(format!("{}:{}", rustix::fs::major(stat.st_dev), rustix::fs::minor(stat.st_dev)));
}
#[cfg(windows)]
if let Ok(serial) = rustfs_utils::os::get_volume_serial_number(&canonical_path) {
diagnostic.device_numbers = Some(format!("serial:{serial:#010x}"));
}
let device_ids = rustfs_utils::os::get_physical_device_ids(&canonical_path).map_err(|err| {
Error::other(format!("failed to inspect physical disk for local endpoint '{canonical_path}': {err}"))
})?;
diagnostic.device_ids = Some(device_ids.clone());
diagnostics.push(diagnostic);
for device_id in device_ids {
device_paths.entry(device_id).or_default().insert(canonical_path.clone());
@@ -748,6 +802,15 @@ fn validate_local_physical_disk_independence(pools: &[Endpoints]) -> Result<()>
);
}
warn!(
diagnostics = %diagnostics
.iter()
.map(LocalDiskValidationDiagnostic::summary)
.collect::<Vec<_>>()
.join("; "),
"Collected local endpoint disk-topology diagnostics before physical disk independence validation",
);
let shared_devices = device_paths
.into_iter()
.filter_map(|(device_id, paths)| {
@@ -768,9 +831,15 @@ fn validate_local_physical_disk_independence(pools: &[Endpoints]) -> Result<()>
.map(|(device_id, paths)| format!("{device_id} => {}", paths.join(", ")))
.collect::<Vec<_>>()
.join("; ");
let diagnostics_summary = diagnostics
.iter()
.map(LocalDiskValidationDiagnostic::summary)
.collect::<Vec<_>>()
.join("; ");
Err(Error::other(format!(
"local erasure endpoints must use distinct physical disks; detected shared devices [{details}]. \
validation diagnostics: [{diagnostics_summary}]. \
Set {ENV_UNSAFE_BYPASS_DISK_CHECK}=true only for local testing or CI to bypass this safety check"
)))
}
@@ -1578,6 +1647,9 @@ mod test {
let err_text = err.to_string();
assert!(err_text.contains("distinct physical disks"), "unexpected error: {err_text}");
assert!(err_text.contains(ENV_UNSAFE_BYPASS_DISK_CHECK), "unexpected error: {err_text}");
assert!(err_text.contains("validation diagnostics:"), "unexpected error: {err_text}");
assert!(err_text.contains("st_dev='"), "unexpected error: {err_text}");
assert!(err_text.contains("device_ids=["), "unexpected error: {err_text}");
})
.await;
}
@@ -433,6 +433,63 @@ mod tests {
}
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn test_erasure_decode_preserves_compressed_stream_near_block_boundary() {
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const BLOCK_SIZE: usize = 1024 * 1024;
use crate::rio::CompressReader;
use rustfs_utils::CompressionAlgorithm;
use tokio::io::AsyncReadExt;
let plaintext_size = 8 * BLOCK_SIZE + 123;
let plaintext = (0..plaintext_size)
.scan(0x9e37_79b9_7f4a_7c15u64, |state, _| {
*state ^= *state << 7;
*state ^= *state >> 9;
*state = state.wrapping_mul(0xbf58_476d_1ce4_e5b9);
Some((*state >> 32) as u8)
})
.collect::<Vec<_>>();
let mut compressor = CompressReader::new(Cursor::new(plaintext), CompressionAlgorithm::default());
let mut compressed = Vec::new();
compressor.read_to_end(&mut compressed).await.unwrap();
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
let total_shards = DATA_SHARDS + PARITY_SHARDS;
let shard_size = erasure.shard_size();
let hash_algo = HashAlgorithm::HighwayHash256;
let mut shard_writers: Vec<BitrotWriter<Cursor<Vec<u8>>>> = (0..total_shards)
.map(|_| BitrotWriter::new(Cursor::new(Vec::new()), shard_size, hash_algo.clone()))
.collect();
for block in compressed.chunks(BLOCK_SIZE) {
let shards = erasure.encode_data(block).unwrap();
for (i, shard) in shards.iter().enumerate() {
shard_writers[i].write(shard).await.unwrap();
}
}
let shard_bufs: Vec<Vec<u8>> = shard_writers.into_iter().map(|w| w.into_inner().into_inner()).collect();
let readers = shard_bufs
.iter()
.map(|buf| Some(BitrotReader::new(Cursor::new(buf.clone()), shard_size, hash_algo.clone(), false)))
.collect();
let mut decoded = Vec::new();
let (written, err) = erasure
.decode(&mut decoded, readers, 0, compressed.len(), compressed.len())
.await;
assert!(err.is_none(), "unexpected decode error: {err:?}");
assert_eq!(written, compressed.len());
assert_eq!(decoded, compressed);
}
#[tokio::test]
async fn test_parallel_reader_normal() {
const BLOCK_SIZE: usize = 64;
+197 -48
View File
@@ -13,8 +13,9 @@
// limitations under the License.
use crate::disk::error::Error;
use crate::disk::error_reduce::count_errs;
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_write_quorum_errs};
use crate::disk::error_reduce::{
OBJECT_OP_IGNORED_ERRS, WriteQuorumFailureSummary, build_write_quorum_failure_summary, reduce_write_quorum_errs,
};
use crate::erasure_coding::BitrotWriterWrapper;
use crate::erasure_coding::Erasure;
use bytes::Bytes;
@@ -27,8 +28,8 @@ use tokio::sync::mpsc;
use tracing::error;
const ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: &str = "RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES";
const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: usize = 8 * 1024 * 1024;
const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS: usize = 8;
const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: usize = 32 * 1024 * 1024;
const DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS: usize = 32;
fn encode_channel_capacity(expanded_block_bytes: usize, max_inflight_bytes: usize) -> usize {
if expanded_block_bytes == 0 {
@@ -50,6 +51,28 @@ async fn drain_queued_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Bytes>>) {
}
}
fn dominant_error_summary_label(summary: &WriteQuorumFailureSummary) -> &'static str {
summary.dominant_error_label
}
fn format_write_quorum_failure(summary: &WriteQuorumFailureSummary) -> String {
format!(
"erasure write quorum (required={}, achieved={}, failed={}, total={}, offline-disks={}/{}, retryable-failures={}, dominant-error={})",
summary.required,
summary.achieved,
summary.failed,
summary.total,
summary.offline_disks,
summary.total,
summary.retryable_failures,
dominant_error_summary_label(summary)
)
}
fn quorum_dominant_error_metric_label(summary: &WriteQuorumFailureSummary) -> &'static str {
dominant_error_summary_label(summary)
}
pub(crate) struct MultiWriter<'a> {
writers: &'a mut [Option<BitrotWriterWrapper>],
write_quorum: usize,
@@ -109,25 +132,18 @@ impl<'a> MultiWriter<'a> {
}
if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) {
error!(
"reduce_write_quorum_errs: {:?}, offline-disks={}/{}, errs={:?}",
write_err,
count_errs(&self.errs, &Error::DiskNotFound),
self.writers.len(),
self.errs
);
return Err(std::io::Error::other(format!(
"Failed to write data: {} (offline-disks={}/{})",
write_err,
count_errs(&self.errs, &Error::DiskNotFound),
self.writers.len()
)));
let summary = build_write_quorum_failure_summary(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum);
let summary_text = format_write_quorum_failure(&summary);
rustfs_io_metrics::internode_metrics::global_internode_metrics()
.record_erasure_write_quorum_failure("write", quorum_dominant_error_metric_label(&summary));
error!("reduce_write_quorum_errs: {:?}, {}, errs={:?}", write_err, summary_text, self.errs);
return Err(std::io::Error::other(format!("Failed to write data: {summary_text}")));
}
let summary = build_write_quorum_failure_summary(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum);
Err(std::io::Error::other(format!(
"Failed to write data: (offline-disks={}/{}): {}",
count_errs(&self.errs, &Error::DiskNotFound),
self.writers.len(),
"Failed to write data: {}: {}",
format_write_quorum_failure(&summary),
self.errs
.iter()
.map(|e| e.as_ref().map_or_else(|| "<nil>".to_string(), |e| e.to_string()))
@@ -171,25 +187,21 @@ impl<'a> MultiWriter<'a> {
}
if let Some(write_err) = reduce_write_quorum_errs(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum) {
let summary = build_write_quorum_failure_summary(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum);
let summary_text = format_write_quorum_failure(&summary);
rustfs_io_metrics::internode_metrics::global_internode_metrics()
.record_erasure_write_quorum_failure("shutdown", quorum_dominant_error_metric_label(&summary));
error!(
"reduce_write_quorum_errs during shutdown: {:?}, offline-disks={}/{}, errs={:?}",
write_err,
count_errs(&self.errs, &Error::DiskNotFound),
self.writers.len(),
self.errs
"reduce_write_quorum_errs during shutdown: {:?}, {}, errs={:?}",
write_err, summary_text, self.errs
);
return Err(std::io::Error::other(format!(
"Failed to shutdown writers: {} (offline-disks={}/{})",
write_err,
count_errs(&self.errs, &Error::DiskNotFound),
self.writers.len()
)));
return Err(std::io::Error::other(format!("Failed to shutdown writers: {summary_text}")));
}
let summary = build_write_quorum_failure_summary(&self.errs, OBJECT_OP_IGNORED_ERRS, self.write_quorum);
Err(std::io::Error::other(format!(
"Failed to shutdown writers: (offline-disks={}/{}): {}",
count_errs(&self.errs, &Error::DiskNotFound),
self.writers.len(),
"Failed to shutdown writers: {}: {}",
format_write_quorum_failure(&summary),
self.errs
.iter()
.map(|e| e.as_ref().map_or_else(|| "<nil>".to_string(), |e| e.to_string()))
@@ -200,6 +212,49 @@ impl<'a> MultiWriter<'a> {
}
impl Erasure {
async fn encode_small_direct<R>(
self: Arc<Self>,
mut reader: R,
writers: &mut [Option<BitrotWriterWrapper>],
quorum: usize,
require_single_block: bool,
) -> std::io::Result<(R, usize)>
where
R: AsyncRead + Send + Sync + Unpin,
{
use tokio::io::AsyncReadExt;
let mut buf = Vec::with_capacity(self.block_size);
let total = if require_single_block {
let read_limit = self
.block_size
.checked_add(1)
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "erasure block_size is too large"))?;
let read_limit = u64::try_from(read_limit)
.map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "erasure block_size exceeds u64"))?;
(&mut reader).take(read_limit).read_to_end(&mut buf).await?
} else {
reader.read_to_end(&mut buf).await?
};
if total == 0 {
return Ok((reader, 0));
}
if require_single_block && total > self.block_size {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"single-block non-inline fast path expects total <= block_size",
));
}
let shards = self.encode_data_owned(buf)?;
let mut mw = MultiWriter::new(writers, quorum);
mw.write(shards).await?;
mw.shutdown().await?;
Ok((reader, total))
}
pub async fn encode<R>(
self: Arc<Self>,
mut reader: R,
@@ -307,27 +362,28 @@ impl Erasure {
/// Reads all data, encodes directly, writes shards sequentially.
pub async fn encode_inline_small<R>(
self: Arc<Self>,
mut reader: R,
reader: R,
writers: &mut [Option<BitrotWriterWrapper>],
quorum: usize,
) -> std::io::Result<(R, usize)>
where
R: AsyncRead + Send + Sync + Unpin,
{
use tokio::io::AsyncReadExt;
self.encode_small_direct(reader, writers, quorum, false).await
}
let mut buf = Vec::with_capacity(self.block_size);
let total = reader.read_to_end(&mut buf).await?;
if total == 0 {
return Ok((reader, 0));
}
let shards = self.encode_data(&buf)?;
let mut mw = MultiWriter::new(writers, quorum);
mw.write(shards).await?;
mw.shutdown().await?;
Ok((reader, total))
/// Fast path for single-block non-inline objects: avoids the producer/consumer
/// pipeline in `encode()` while keeping the same writer/quorum/shutdown semantics.
pub async fn encode_single_block_non_inline<R>(
self: Arc<Self>,
reader: R,
writers: &mut [Option<BitrotWriterWrapper>],
quorum: usize,
) -> std::io::Result<(R, usize)>
where
R: AsyncRead + Send + Sync + Unpin,
{
self.encode_small_direct(reader, writers, quorum, true).await
}
}
@@ -491,6 +547,75 @@ mod tests {
}
}
#[tokio::test]
async fn encode_single_block_non_inline_payload_writes_all_shards() {
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const TOTAL_SHARDS: usize = DATA_SHARDS + PARITY_SHARDS;
const BLOCK_SIZE: usize = 64;
let committed: Vec<Arc<Mutex<Vec<u8>>>> = (0..TOTAL_SHARDS).map(|_| Arc::new(Mutex::new(Vec::new()))).collect();
let mut writers: Vec<Option<BitrotWriterWrapper>> = committed
.iter()
.map(|c| {
Some(BitrotWriterWrapper::new(
CustomWriter::new_tokio_writer(DeferredCommitWriter::new(c.clone())),
BLOCK_SIZE / DATA_SHARDS,
HashAlgorithm::HighwayHash256S,
))
})
.collect();
let payload = b"hello single block";
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
let reader = tokio::io::BufReader::new(std::io::Cursor::new(payload.to_vec()));
let (_reader, total) = erasure
.encode_single_block_non_inline(reader, &mut writers, DATA_SHARDS)
.await
.unwrap();
assert_eq!(total, payload.len());
for (i, c) in committed.iter().enumerate() {
assert!(!c.lock().unwrap().is_empty(), "shard {i} should have received data");
}
}
#[tokio::test]
async fn encode_single_block_non_inline_rejects_multi_block_payload() {
const DATA_SHARDS: usize = 2;
const PARITY_SHARDS: usize = 2;
const TOTAL_SHARDS: usize = DATA_SHARDS + PARITY_SHARDS;
const BLOCK_SIZE: usize = 64;
let committed: Vec<Arc<Mutex<Vec<u8>>>> = (0..TOTAL_SHARDS).map(|_| Arc::new(Mutex::new(Vec::new()))).collect();
let mut writers: Vec<Option<BitrotWriterWrapper>> = committed
.iter()
.map(|c| {
Some(BitrotWriterWrapper::new(
CustomWriter::new_tokio_writer(DeferredCommitWriter::new(c.clone())),
BLOCK_SIZE / DATA_SHARDS,
HashAlgorithm::HighwayHash256S,
))
})
.collect();
let payload = vec![1u8; BLOCK_SIZE + 1];
let erasure = Arc::new(Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE));
let reader = tokio::io::BufReader::new(std::io::Cursor::new(payload));
let err = erasure
.encode_single_block_non_inline(reader, &mut writers, DATA_SHARDS)
.await
.expect_err("single-block fast path must reject oversized readers");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput);
assert!(err.to_string().contains("single-block non-inline fast path"));
for c in committed {
assert!(c.lock().unwrap().is_empty());
}
}
#[test]
fn encode_channel_capacity_never_returns_zero() {
assert_eq!(encode_channel_capacity(0, 1024), 1);
@@ -498,9 +623,33 @@ mod tests {
assert_eq!(encode_channel_capacity(4096, 1024), 1);
}
#[test]
fn write_quorum_failure_summary_uses_stable_dominant_error_label() {
let err = Error::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::ConnectionReset,
));
let summary = WriteQuorumFailureSummary {
required: 2,
achieved: 0,
failed: 2,
total: 2,
offline_disks: 0,
ignored_failures: 0,
retryable_failures: 2,
dominant_error: Some(err),
dominant_error_label: "connection_reset",
};
let text = format_write_quorum_failure(&summary);
assert!(text.contains("dominant-error=connection_reset"));
assert!(!text.contains("/rustfs/rpc/put_file_stream"));
assert!(!text.contains("PUT "));
}
#[test]
fn encode_channel_capacity_respects_budget_and_hard_cap() {
assert_eq!(encode_channel_capacity(4 * 1024 * 1024, 32 * 1024 * 1024), 8);
assert_eq!(encode_channel_capacity(1536 * 1024, 32 * 1024 * 1024), 21);
assert_eq!(encode_channel_capacity(16 * 1024 * 1024, 32 * 1024 * 1024), 2);
assert_eq!(encode_channel_capacity(1, usize::MAX), DEFAULT_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BLOCKS);
}
+79 -1
View File
@@ -350,7 +350,7 @@ impl Clone for Erasure {
legacy_encoder: self.legacy_encoder.clone(),
block_size: self.block_size,
uses_legacy: self.uses_legacy,
_id: Uuid::new_v4(), // Generate new ID for clone
_id: self._id, // Shared by clones; this field is unused in hot paths.
}
}
}
@@ -413,6 +413,9 @@ impl Erasure {
calc_shard_size
};
let per_shard_size = shard_size_fn(data.len(), self.data_shards);
if per_shard_size == 0 {
return Ok(vec![Bytes::new(); self.total_shard_count()]);
}
let need_total_size = per_shard_size * self.total_shard_count();
let mut data_buffer = BytesMut::with_capacity(need_total_size);
@@ -448,6 +451,63 @@ impl Erasure {
Ok(shards)
}
/// Encode owned data, avoiding a copy when the caller already has a heap buffer.
/// Falls back to copying into a new buffer if zero-copy conversion fails.
pub fn encode_data_owned(&self, data: Vec<u8>) -> io::Result<Vec<Bytes>> {
let shard_size_fn = if self.uses_legacy {
calc_shard_size_legacy
} else {
calc_shard_size
};
let per_shard_size = shard_size_fn(data.len(), self.data_shards);
if per_shard_size == 0 {
return Ok(vec![Bytes::new(); self.total_shard_count()]);
}
let need_total_size = per_shard_size * self.total_shard_count();
// Try zero-copy: Vec<u8> -> Bytes -> BytesMut (succeeds when refcount == 1)
let mut data_buffer = match Bytes::from(data).try_into_mut() {
Ok(mut bm) => {
bm.resize(need_total_size, 0u8);
bm
}
Err(b) => {
// Rare path: refcount != 1, fall back to copy
let mut bm = BytesMut::with_capacity(need_total_size);
bm.extend_from_slice(&b);
bm.resize(need_total_size, 0u8);
bm
}
};
{
let data_slices: SmallVec<[&mut [u8]; 16]> = data_buffer.chunks_exact_mut(per_shard_size).collect();
if self.parity_shards > 0 {
if self.uses_legacy {
if let Some(encoder) = self.legacy_encoder.as_ref() {
encoder.encode(data_slices)?;
} else {
warn!("parity_shards > 0, uses_legacy but legacy_encoder is None");
}
} else if let Some(encoder) = self.encoder.as_ref() {
encoder.encode(data_slices)?;
} else {
warn!("parity_shards > 0, but encoder is None");
}
}
}
let mut data_buffer = data_buffer.freeze();
let mut shards = Vec::with_capacity(self.total_shard_count());
for _ in 0..self.total_shard_count() {
let shard = data_buffer.split_to(per_shard_size);
shards.push(shard);
}
Ok(shards)
}
/// Decode and reconstruct missing data shards in-place.
///
/// # Arguments
@@ -637,6 +697,24 @@ mod tests {
shards.iter().map(|shard| Some(shard.to_vec())).collect()
}
fn assert_owned_encode_matches_borrowed(erasure: &Erasure, data: Vec<u8>) {
let borrowed = erasure.encode_data(&data).expect("borrowed encode should succeed");
let owned = erasure.encode_data_owned(data).expect("owned encode should succeed");
assert_eq!(owned, borrowed);
}
#[test]
fn encode_data_owned_matches_borrowed_path() {
for uses_legacy in [false, true] {
let erasure = Erasure::new_with_options(4, 2, 64, uses_legacy);
assert_owned_encode_matches_borrowed(&erasure, Vec::new());
assert_owned_encode_matches_borrowed(&erasure, b"small payload".to_vec());
assert_owned_encode_matches_borrowed(&erasure, (0_u8..37).collect());
}
}
#[test]
fn decode_data_keeps_missing_parity_shard_unreconstructed() {
let erasure = Erasure::new(2, 2, 64);
+180 -135
View File
@@ -14,6 +14,7 @@
use crate::bucket::error::BucketMetadataError;
use crate::disk::error::DiskError;
use rustfs_storage_api::StorageErrorCode;
use rustfs_utils::path::decode_dir_object;
use s3s::{S3Error, S3ErrorCode};
@@ -460,156 +461,181 @@ impl Clone for StorageError {
}
impl StorageError {
pub fn to_u32(&self) -> u32 {
fn code(&self) -> StorageErrorCode {
match self {
StorageError::Io(_) => 0x01,
StorageError::FaultyDisk => 0x02,
StorageError::DiskFull => 0x03,
StorageError::VolumeNotFound => 0x04,
StorageError::VolumeExists => 0x05,
StorageError::FileNotFound => 0x06,
StorageError::FileVersionNotFound => 0x07,
StorageError::FileNameTooLong => 0x08,
StorageError::FileAccessDenied => 0x09,
StorageError::FileCorrupt => 0x0A,
StorageError::IsNotRegular => 0x0B,
StorageError::VolumeNotEmpty => 0x0C,
StorageError::VolumeAccessDenied => 0x0D,
StorageError::CorruptedFormat => 0x0E,
StorageError::CorruptedBackend => 0x0F,
StorageError::UnformattedDisk => 0x10,
StorageError::DiskNotFound => 0x11,
StorageError::DriveIsRoot => 0x12,
StorageError::FaultyRemoteDisk => 0x13,
StorageError::DiskAccessDenied => 0x14,
StorageError::Unexpected => 0x15,
StorageError::NotImplemented => 0x16,
StorageError::InvalidArgument(_, _, _) => 0x17,
StorageError::MethodNotAllowed => 0x18,
StorageError::BucketNotFound(_) => 0x19,
StorageError::BucketNotEmpty(_) => 0x1A,
StorageError::BucketNameInvalid(_) => 0x1B,
StorageError::ObjectNameInvalid(_, _) => 0x1C,
StorageError::BucketExists(_) => 0x1D,
StorageError::StorageFull => 0x1E,
StorageError::SlowDown => 0x1F,
StorageError::PrefixAccessDenied(_, _) => 0x20,
StorageError::InvalidUploadIDKeyCombination(_, _) => 0x21,
StorageError::MalformedUploadID(_) => 0x22,
StorageError::ObjectNameTooLong(_, _) => 0x23,
StorageError::ObjectNamePrefixAsSlash(_, _) => 0x24,
StorageError::ObjectNotFound(_, _) => 0x25,
StorageError::VersionNotFound(_, _, _) => 0x26,
StorageError::InvalidUploadID(_, _, _) => 0x27,
StorageError::InvalidVersionID(_, _, _) => 0x28,
StorageError::DataMovementOverwriteErr(_, _, _) => 0x29,
StorageError::ObjectExistsAsDirectory(_, _) => 0x2A,
// StorageError::InsufficientReadQuorum => 0x2B,
// StorageError::InsufficientWriteQuorum => 0x2C,
StorageError::DecommissionNotStarted => 0x2D,
StorageError::InvalidPart(_, _, _) => 0x2E,
StorageError::DoneForNow => 0x2F,
StorageError::DecommissionAlreadyRunning => 0x30,
StorageError::RebalanceAlreadyRunning => 0x40,
StorageError::OperationCanceled => 0x41,
StorageError::ErasureReadQuorum => 0x31,
StorageError::ErasureWriteQuorum => 0x32,
StorageError::NotFirstDisk => 0x33,
StorageError::FirstDiskWait => 0x34,
StorageError::ConfigNotFound => 0x35,
StorageError::TooManyOpenFiles => 0x36,
StorageError::NoHealRequired => 0x37,
StorageError::Lock(_) => 0x38,
StorageError::InsufficientReadQuorum(_, _) => 0x39,
StorageError::InsufficientWriteQuorum(_, _) => 0x3A,
StorageError::PreconditionFailed => 0x3B,
StorageError::EntityTooSmall(_, _, _) => 0x3C,
StorageError::InvalidRangeSpec(_) => 0x3D,
StorageError::NotModified => 0x3E,
StorageError::InvalidPartNumber(_) => 0x3F,
StorageError::NamespaceLockQuorumUnavailable { .. } => 0x42,
StorageError::Io(_) => StorageErrorCode::Io,
StorageError::FaultyDisk => StorageErrorCode::FaultyDisk,
StorageError::DiskFull => StorageErrorCode::DiskFull,
StorageError::VolumeNotFound => StorageErrorCode::VolumeNotFound,
StorageError::VolumeExists => StorageErrorCode::VolumeExists,
StorageError::FileNotFound => StorageErrorCode::FileNotFound,
StorageError::FileVersionNotFound => StorageErrorCode::FileVersionNotFound,
StorageError::FileNameTooLong => StorageErrorCode::FileNameTooLong,
StorageError::FileAccessDenied => StorageErrorCode::FileAccessDenied,
StorageError::FileCorrupt => StorageErrorCode::FileCorrupt,
StorageError::IsNotRegular => StorageErrorCode::IsNotRegular,
StorageError::VolumeNotEmpty => StorageErrorCode::VolumeNotEmpty,
StorageError::VolumeAccessDenied => StorageErrorCode::VolumeAccessDenied,
StorageError::CorruptedFormat => StorageErrorCode::CorruptedFormat,
StorageError::CorruptedBackend => StorageErrorCode::CorruptedBackend,
StorageError::UnformattedDisk => StorageErrorCode::UnformattedDisk,
StorageError::DiskNotFound => StorageErrorCode::DiskNotFound,
StorageError::DriveIsRoot => StorageErrorCode::DriveIsRoot,
StorageError::FaultyRemoteDisk => StorageErrorCode::FaultyRemoteDisk,
StorageError::DiskAccessDenied => StorageErrorCode::DiskAccessDenied,
StorageError::Unexpected => StorageErrorCode::Unexpected,
StorageError::NotImplemented => StorageErrorCode::NotImplemented,
StorageError::InvalidArgument(_, _, _) => StorageErrorCode::InvalidArgument,
StorageError::MethodNotAllowed => StorageErrorCode::MethodNotAllowed,
StorageError::BucketNotFound(_) => StorageErrorCode::BucketNotFound,
StorageError::BucketNotEmpty(_) => StorageErrorCode::BucketNotEmpty,
StorageError::BucketNameInvalid(_) => StorageErrorCode::BucketNameInvalid,
StorageError::ObjectNameInvalid(_, _) => StorageErrorCode::ObjectNameInvalid,
StorageError::BucketExists(_) => StorageErrorCode::BucketExists,
StorageError::StorageFull => StorageErrorCode::StorageFull,
StorageError::SlowDown => StorageErrorCode::SlowDown,
StorageError::PrefixAccessDenied(_, _) => StorageErrorCode::PrefixAccessDenied,
StorageError::InvalidUploadIDKeyCombination(_, _) => StorageErrorCode::InvalidUploadIDKeyCombination,
StorageError::MalformedUploadID(_) => StorageErrorCode::MalformedUploadID,
StorageError::ObjectNameTooLong(_, _) => StorageErrorCode::ObjectNameTooLong,
StorageError::ObjectNamePrefixAsSlash(_, _) => StorageErrorCode::ObjectNamePrefixAsSlash,
StorageError::ObjectNotFound(_, _) => StorageErrorCode::ObjectNotFound,
StorageError::VersionNotFound(_, _, _) => StorageErrorCode::VersionNotFound,
StorageError::InvalidUploadID(_, _, _) => StorageErrorCode::InvalidUploadID,
StorageError::InvalidVersionID(_, _, _) => StorageErrorCode::InvalidVersionID,
StorageError::DataMovementOverwriteErr(_, _, _) => StorageErrorCode::DataMovementOverwriteErr,
StorageError::ObjectExistsAsDirectory(_, _) => StorageErrorCode::ObjectExistsAsDirectory,
StorageError::DecommissionNotStarted => StorageErrorCode::DecommissionNotStarted,
StorageError::InvalidPart(_, _, _) => StorageErrorCode::InvalidPart,
StorageError::DoneForNow => StorageErrorCode::DoneForNow,
StorageError::DecommissionAlreadyRunning => StorageErrorCode::DecommissionAlreadyRunning,
StorageError::RebalanceAlreadyRunning => StorageErrorCode::RebalanceAlreadyRunning,
StorageError::OperationCanceled => StorageErrorCode::OperationCanceled,
StorageError::ErasureReadQuorum => StorageErrorCode::ErasureReadQuorum,
StorageError::ErasureWriteQuorum => StorageErrorCode::ErasureWriteQuorum,
StorageError::NotFirstDisk => StorageErrorCode::NotFirstDisk,
StorageError::FirstDiskWait => StorageErrorCode::FirstDiskWait,
StorageError::ConfigNotFound => StorageErrorCode::ConfigNotFound,
StorageError::TooManyOpenFiles => StorageErrorCode::TooManyOpenFiles,
StorageError::NoHealRequired => StorageErrorCode::NoHealRequired,
StorageError::Lock(_) => StorageErrorCode::Lock,
StorageError::InsufficientReadQuorum(_, _) => StorageErrorCode::InsufficientReadQuorum,
StorageError::InsufficientWriteQuorum(_, _) => StorageErrorCode::InsufficientWriteQuorum,
StorageError::PreconditionFailed => StorageErrorCode::PreconditionFailed,
StorageError::EntityTooSmall(_, _, _) => StorageErrorCode::EntityTooSmall,
StorageError::InvalidRangeSpec(_) => StorageErrorCode::InvalidRangeSpec,
StorageError::NotModified => StorageErrorCode::NotModified,
StorageError::InvalidPartNumber(_) => StorageErrorCode::InvalidPartNumber,
StorageError::NamespaceLockQuorumUnavailable { .. } => StorageErrorCode::NamespaceLockQuorumUnavailable,
}
}
pub fn to_u32(&self) -> u32 {
self.code().as_u32()
}
pub fn from_u32(error: u32) -> Option<Self> {
match error {
0x01 => Some(StorageError::Io(std::io::Error::other("Io error"))),
0x02 => Some(StorageError::FaultyDisk),
0x03 => Some(StorageError::DiskFull),
0x04 => Some(StorageError::VolumeNotFound),
0x05 => Some(StorageError::VolumeExists),
0x06 => Some(StorageError::FileNotFound),
0x07 => Some(StorageError::FileVersionNotFound),
0x08 => Some(StorageError::FileNameTooLong),
0x09 => Some(StorageError::FileAccessDenied),
0x0A => Some(StorageError::FileCorrupt),
0x0B => Some(StorageError::IsNotRegular),
0x0C => Some(StorageError::VolumeNotEmpty),
0x0D => Some(StorageError::VolumeAccessDenied),
0x0E => Some(StorageError::CorruptedFormat),
0x0F => Some(StorageError::CorruptedBackend),
0x10 => Some(StorageError::UnformattedDisk),
0x11 => Some(StorageError::DiskNotFound),
0x12 => Some(StorageError::DriveIsRoot),
0x13 => Some(StorageError::FaultyRemoteDisk),
0x14 => Some(StorageError::DiskAccessDenied),
0x15 => Some(StorageError::Unexpected),
0x16 => Some(StorageError::NotImplemented),
0x17 => Some(StorageError::InvalidArgument(Default::default(), Default::default(), Default::default())),
0x18 => Some(StorageError::MethodNotAllowed),
0x19 => Some(StorageError::BucketNotFound(Default::default())),
0x1A => Some(StorageError::BucketNotEmpty(Default::default())),
0x1B => Some(StorageError::BucketNameInvalid(Default::default())),
0x1C => Some(StorageError::ObjectNameInvalid(Default::default(), Default::default())),
0x1D => Some(StorageError::BucketExists(Default::default())),
0x1E => Some(StorageError::StorageFull),
0x1F => Some(StorageError::SlowDown),
0x20 => Some(StorageError::PrefixAccessDenied(Default::default(), Default::default())),
0x21 => Some(StorageError::InvalidUploadIDKeyCombination(Default::default(), Default::default())),
0x22 => Some(StorageError::MalformedUploadID(Default::default())),
0x23 => Some(StorageError::ObjectNameTooLong(Default::default(), Default::default())),
0x24 => Some(StorageError::ObjectNamePrefixAsSlash(Default::default(), Default::default())),
0x25 => Some(StorageError::ObjectNotFound(Default::default(), Default::default())),
0x26 => Some(StorageError::VersionNotFound(Default::default(), Default::default(), Default::default())),
0x27 => Some(StorageError::InvalidUploadID(Default::default(), Default::default(), Default::default())),
0x28 => Some(StorageError::InvalidVersionID(Default::default(), Default::default(), Default::default())),
0x29 => Some(StorageError::DataMovementOverwriteErr(
match StorageErrorCode::from_u32(error)? {
StorageErrorCode::Io => Some(StorageError::Io(std::io::Error::other("Io error"))),
StorageErrorCode::FaultyDisk => Some(StorageError::FaultyDisk),
StorageErrorCode::DiskFull => Some(StorageError::DiskFull),
StorageErrorCode::VolumeNotFound => Some(StorageError::VolumeNotFound),
StorageErrorCode::VolumeExists => Some(StorageError::VolumeExists),
StorageErrorCode::FileNotFound => Some(StorageError::FileNotFound),
StorageErrorCode::FileVersionNotFound => Some(StorageError::FileVersionNotFound),
StorageErrorCode::FileNameTooLong => Some(StorageError::FileNameTooLong),
StorageErrorCode::FileAccessDenied => Some(StorageError::FileAccessDenied),
StorageErrorCode::FileCorrupt => Some(StorageError::FileCorrupt),
StorageErrorCode::IsNotRegular => Some(StorageError::IsNotRegular),
StorageErrorCode::VolumeNotEmpty => Some(StorageError::VolumeNotEmpty),
StorageErrorCode::VolumeAccessDenied => Some(StorageError::VolumeAccessDenied),
StorageErrorCode::CorruptedFormat => Some(StorageError::CorruptedFormat),
StorageErrorCode::CorruptedBackend => Some(StorageError::CorruptedBackend),
StorageErrorCode::UnformattedDisk => Some(StorageError::UnformattedDisk),
StorageErrorCode::DiskNotFound => Some(StorageError::DiskNotFound),
StorageErrorCode::DriveIsRoot => Some(StorageError::DriveIsRoot),
StorageErrorCode::FaultyRemoteDisk => Some(StorageError::FaultyRemoteDisk),
StorageErrorCode::DiskAccessDenied => Some(StorageError::DiskAccessDenied),
StorageErrorCode::Unexpected => Some(StorageError::Unexpected),
StorageErrorCode::NotImplemented => Some(StorageError::NotImplemented),
StorageErrorCode::InvalidArgument => {
Some(StorageError::InvalidArgument(Default::default(), Default::default(), Default::default()))
}
StorageErrorCode::MethodNotAllowed => Some(StorageError::MethodNotAllowed),
StorageErrorCode::BucketNotFound => Some(StorageError::BucketNotFound(Default::default())),
StorageErrorCode::BucketNotEmpty => Some(StorageError::BucketNotEmpty(Default::default())),
StorageErrorCode::BucketNameInvalid => Some(StorageError::BucketNameInvalid(Default::default())),
StorageErrorCode::ObjectNameInvalid => Some(StorageError::ObjectNameInvalid(Default::default(), Default::default())),
StorageErrorCode::BucketExists => Some(StorageError::BucketExists(Default::default())),
StorageErrorCode::StorageFull => Some(StorageError::StorageFull),
StorageErrorCode::SlowDown => Some(StorageError::SlowDown),
StorageErrorCode::PrefixAccessDenied => {
Some(StorageError::PrefixAccessDenied(Default::default(), Default::default()))
}
StorageErrorCode::InvalidUploadIDKeyCombination => {
Some(StorageError::InvalidUploadIDKeyCombination(Default::default(), Default::default()))
}
StorageErrorCode::MalformedUploadID => Some(StorageError::MalformedUploadID(Default::default())),
StorageErrorCode::ObjectNameTooLong => Some(StorageError::ObjectNameTooLong(Default::default(), Default::default())),
StorageErrorCode::ObjectNamePrefixAsSlash => {
Some(StorageError::ObjectNamePrefixAsSlash(Default::default(), Default::default()))
}
StorageErrorCode::ObjectNotFound => Some(StorageError::ObjectNotFound(Default::default(), Default::default())),
StorageErrorCode::VersionNotFound => {
Some(StorageError::VersionNotFound(Default::default(), Default::default(), Default::default()))
}
StorageErrorCode::InvalidUploadID => {
Some(StorageError::InvalidUploadID(Default::default(), Default::default(), Default::default()))
}
StorageErrorCode::InvalidVersionID => {
Some(StorageError::InvalidVersionID(Default::default(), Default::default(), Default::default()))
}
StorageErrorCode::DataMovementOverwriteErr => Some(StorageError::DataMovementOverwriteErr(
Default::default(),
Default::default(),
Default::default(),
)),
0x2A => Some(StorageError::ObjectExistsAsDirectory(Default::default(), Default::default())),
// 0x2B => Some(StorageError::InsufficientReadQuorum),
// 0x2C => Some(StorageError::InsufficientWriteQuorum),
0x2D => Some(StorageError::DecommissionNotStarted),
0x2E => Some(StorageError::InvalidPart(Default::default(), Default::default(), Default::default())),
0x2F => Some(StorageError::DoneForNow),
0x30 => Some(StorageError::DecommissionAlreadyRunning),
0x40 => Some(StorageError::RebalanceAlreadyRunning),
0x41 => Some(StorageError::OperationCanceled),
0x31 => Some(StorageError::ErasureReadQuorum),
0x32 => Some(StorageError::ErasureWriteQuorum),
0x33 => Some(StorageError::NotFirstDisk),
0x34 => Some(StorageError::FirstDiskWait),
0x35 => Some(StorageError::ConfigNotFound),
0x36 => Some(StorageError::TooManyOpenFiles),
0x37 => Some(StorageError::NoHealRequired),
0x38 => Some(StorageError::Lock(rustfs_lock::LockError::internal("Generic lock error".to_string()))),
0x39 => Some(StorageError::InsufficientReadQuorum(Default::default(), Default::default())),
0x3A => Some(StorageError::InsufficientWriteQuorum(Default::default(), Default::default())),
0x3B => Some(StorageError::PreconditionFailed),
0x3C => Some(StorageError::EntityTooSmall(Default::default(), Default::default(), Default::default())),
0x3D => Some(StorageError::InvalidRangeSpec(Default::default())),
0x3E => Some(StorageError::NotModified),
0x3F => Some(StorageError::InvalidPartNumber(Default::default())),
0x42 => Some(StorageError::NamespaceLockQuorumUnavailable {
StorageErrorCode::ObjectExistsAsDirectory => {
Some(StorageError::ObjectExistsAsDirectory(Default::default(), Default::default()))
}
StorageErrorCode::DecommissionNotStarted => Some(StorageError::DecommissionNotStarted),
StorageErrorCode::InvalidPart => {
Some(StorageError::InvalidPart(Default::default(), Default::default(), Default::default()))
}
StorageErrorCode::DoneForNow => Some(StorageError::DoneForNow),
StorageErrorCode::DecommissionAlreadyRunning => Some(StorageError::DecommissionAlreadyRunning),
StorageErrorCode::RebalanceAlreadyRunning => Some(StorageError::RebalanceAlreadyRunning),
StorageErrorCode::OperationCanceled => Some(StorageError::OperationCanceled),
StorageErrorCode::ErasureReadQuorum => Some(StorageError::ErasureReadQuorum),
StorageErrorCode::ErasureWriteQuorum => Some(StorageError::ErasureWriteQuorum),
StorageErrorCode::NotFirstDisk => Some(StorageError::NotFirstDisk),
StorageErrorCode::FirstDiskWait => Some(StorageError::FirstDiskWait),
StorageErrorCode::ConfigNotFound => Some(StorageError::ConfigNotFound),
StorageErrorCode::TooManyOpenFiles => Some(StorageError::TooManyOpenFiles),
StorageErrorCode::NoHealRequired => Some(StorageError::NoHealRequired),
StorageErrorCode::Lock => {
Some(StorageError::Lock(rustfs_lock::LockError::internal("Generic lock error".to_string())))
}
StorageErrorCode::InsufficientReadQuorum => {
Some(StorageError::InsufficientReadQuorum(Default::default(), Default::default()))
}
StorageErrorCode::InsufficientWriteQuorum => {
Some(StorageError::InsufficientWriteQuorum(Default::default(), Default::default()))
}
StorageErrorCode::PreconditionFailed => Some(StorageError::PreconditionFailed),
StorageErrorCode::EntityTooSmall => {
Some(StorageError::EntityTooSmall(Default::default(), Default::default(), Default::default()))
}
StorageErrorCode::InvalidRangeSpec => Some(StorageError::InvalidRangeSpec(Default::default())),
StorageErrorCode::NotModified => Some(StorageError::NotModified),
StorageErrorCode::InvalidPartNumber => Some(StorageError::InvalidPartNumber(Default::default())),
StorageErrorCode::NamespaceLockQuorumUnavailable => Some(StorageError::NamespaceLockQuorumUnavailable {
mode: "write",
bucket: Default::default(),
object: Default::default(),
required: Default::default(),
achieved: Default::default(),
}),
_ => None,
}
}
}
@@ -1048,6 +1074,25 @@ mod tests {
assert!(StorageError::from_u32(0xFF).is_none());
}
#[test]
fn test_storage_error_code_contract_matches_storage_api() {
assert_eq!(StorageError::DiskFull.to_u32(), StorageErrorCode::DiskFull.as_u32());
assert_eq!(
StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()).to_u32(),
StorageErrorCode::ObjectNotFound.as_u32()
);
assert!(matches!(
StorageError::from_u32(StorageErrorCode::ErasureReadQuorum.as_u32()),
Some(StorageError::ErasureReadQuorum)
));
assert!(matches!(
StorageError::from_u32(StorageErrorCode::NamespaceLockQuorumUnavailable.as_u32()),
Some(StorageError::NamespaceLockQuorumUnavailable { .. })
));
assert!(StorageError::from_u32(0x2B).is_none());
assert!(StorageError::from_u32(0x2C).is_none());
}
#[test]
fn test_storage_error_partial_eq() {
// Test IO error comparison
+1 -1
View File
@@ -242,7 +242,7 @@ pub async fn update_erasure_type(setup_type: SetupType) {
// }
// }
type TypeLocalDiskSetDrives = Vec<Vec<Vec<Option<DiskStore>>>>;
pub(crate) type TypeLocalDiskSetDrives = Vec<Vec<Vec<Option<DiskStore>>>>;
/// Set the global region
///
+10
View File
@@ -34,6 +34,7 @@ pub mod metrics_realtime;
pub mod notification_sys;
pub mod pools;
pub mod rebalance;
pub mod rio;
pub mod rpc;
pub mod set_disk;
mod sets;
@@ -60,3 +61,12 @@ pub use global::{get_global_lock_client, get_global_lock_clients, set_global_loc
pub use global::GLOBAL_Endpoints;
pub use store_api::StorageAPI;
#[cfg(test)]
mod rio_tests {
#[test]
fn uses_expected_rio_backend() {
let expected = if cfg!(feature = "rio-v2") { "rio-v2" } else { "legacy-rio" };
assert_eq!(crate::rio::backend_name(), expected);
}
}
+70 -1
View File
@@ -18,7 +18,8 @@ use rustfs_common::{GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, heal_channel::Dr
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use rustfs_madmin::metrics::{
DiskIOStats, DiskMetric, LastMinute as MadminLastMinute, NetDevLine, NetMetrics, RPCMetrics, RealtimeMetrics,
ScannerMetrics as MadminScannerMetrics, TimedAction as MadminTimedAction,
ScannerMetrics as MadminScannerMetrics, ScannerPacingPressureSnapshot as MadminScannerPacingPressureSnapshot,
ScannerSourceCycleSnapshot as MadminScannerSourceCycleSnapshot, TimedAction as MadminTimedAction,
};
use rustfs_utils::os::get_drive_stats;
use serde::{Deserialize, Serialize};
@@ -99,6 +100,26 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
.collect(),
},
active_paths: metrics.active_paths,
last_cycle_partial_source: metrics.last_cycle_partial_source,
last_cycle_partial_source_code: metrics.last_cycle_partial_source_code,
pacing_pressure: MadminScannerPacingPressureSnapshot {
primary_pressure: metrics.pacing_pressure.primary_pressure,
current_queued_scans: metrics.pacing_pressure.current_queued_scans,
current_active_scans: metrics.pacing_pressure.current_active_scans,
last_cycle_budget_limited: metrics.pacing_pressure.last_cycle_budget_limited,
last_cycle_pause_observed: metrics.pacing_pressure.last_cycle_pause_observed,
last_cycle_throttle_sleep_ratio: metrics.pacing_pressure.last_cycle_throttle_sleep_ratio,
last_cycle_yield_ratio: metrics.pacing_pressure.last_cycle_yield_ratio,
last_cycle_total_pause_ratio: metrics.pacing_pressure.last_cycle_total_pause_ratio,
},
partial_cycles_by_source: metrics
.partial_cycles_by_source
.into_iter()
.map(|source| MadminScannerSourceCycleSnapshot {
source: source.source,
cycles: source.cycles,
})
.collect(),
}
}
@@ -343,4 +364,52 @@ mod test {
metrics.reset_for_test();
}
#[test]
fn scanner_metrics_mapping_preserves_partial_source_status() {
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
last_cycle_partial_source: "usage".to_string(),
last_cycle_partial_source_code: 1,
partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
source: "usage".to_string(),
cycles: 2,
}],
..Default::default()
});
assert_eq!(scanner.last_cycle_partial_source, "usage");
assert_eq!(scanner.last_cycle_partial_source_code, 1);
let usage = scanner
.partial_cycles_by_source
.iter()
.find(|source| source.source == "usage")
.expect("usage partial source should be mapped");
assert_eq!(usage.cycles, 2);
}
#[test]
fn scanner_metrics_mapping_preserves_pacing_pressure() {
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
pacing_pressure: rustfs_common::metrics::ScannerPacingPressureSnapshot {
primary_pressure: "cycle_budget".to_string(),
current_queued_scans: 4,
current_active_scans: 2,
last_cycle_budget_limited: true,
last_cycle_pause_observed: true,
last_cycle_throttle_sleep_ratio: 0.25,
last_cycle_yield_ratio: 0.05,
last_cycle_total_pause_ratio: 0.3,
},
..Default::default()
});
assert_eq!(scanner.pacing_pressure.primary_pressure, "cycle_budget");
assert_eq!(scanner.pacing_pressure.current_queued_scans, 4);
assert_eq!(scanner.pacing_pressure.current_active_scans, 2);
assert!(scanner.pacing_pressure.last_cycle_budget_limited);
assert!(scanner.pacing_pressure.last_cycle_pause_observed);
assert_eq!(scanner.pacing_pressure.last_cycle_throttle_sleep_ratio, 0.25);
assert_eq!(scanner.pacing_pressure.last_cycle_yield_ratio, 0.05);
assert_eq!(scanner.pacing_pressure.last_cycle_total_pause_ratio, 0.3);
}
}
+393 -20
View File
@@ -25,15 +25,37 @@ use lazy_static::lazy_static;
use rustfs_madmin::health::{Cpus, MemInfo, OsInfo, Partitions, ProcInfo, SysConfig, SysErrors, SysServices};
use rustfs_madmin::metrics::RealtimeMetrics;
use rustfs_madmin::net::NetInfo;
use rustfs_madmin::{ItemState, ServerProperties};
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
use std::collections::hash_map::DefaultHasher;
use std::future::Future;
use std::hash::{Hash, Hasher};
use std::sync::OnceLock;
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, SystemTime};
use tokio::time::timeout;
use tracing::{error, warn};
/// After this many consecutive admin-call failures, mark the peer as offline.
const CONSECUTIVE_FAILURE_THRESHOLD: u32 = 3;
/// Cached result from the last successful admin call to a peer.
struct PeerAdminCache {
last_storage_info: Option<StorageInfo>,
last_server_info: Option<ServerProperties>,
storage_failures: u32,
server_failures: u32,
}
impl PeerAdminCache {
fn new() -> Self {
Self {
last_storage_info: None,
last_server_info: None,
storage_failures: 0,
server_failures: 0,
}
}
}
lazy_static! {
pub static ref GLOBAL_NotificationSys: OnceLock<NotificationSys> = OnceLock::new();
}
@@ -53,14 +75,17 @@ pub struct NotificationSys {
pub peer_clients: Vec<Option<PeerRestClient>>,
#[allow(dead_code)]
pub all_peer_clients: Vec<Option<PeerRestClient>>,
peer_admin_caches: Vec<Mutex<PeerAdminCache>>,
}
impl NotificationSys {
pub async fn new(eps: EndpointServerPools) -> Self {
let (peer_clients, all_peer_clients) = PeerRestClient::new_clients(eps).await;
let peer_admin_caches = (0..peer_clients.len()).map(|_| Mutex::new(PeerAdminCache::new())).collect();
Self {
peer_clients,
all_peer_clients,
peer_admin_caches,
}
}
}
@@ -251,30 +276,27 @@ impl NotificationSys {
pub async fn storage_info<S: StorageAPI>(&self, api: &S) -> rustfs_madmin::StorageInfo {
let mut futures = Vec::with_capacity(self.peer_clients.len());
let endpoints = get_global_endpoints();
let peer_timeout = Duration::from_secs(2); // Same timeout as server_info
let peer_timeout = Duration::from_secs(5);
for client in self.peer_clients.iter() {
for (idx, client) in self.peer_clients.iter().enumerate() {
let endpoints = endpoints.clone();
let cache = self.peer_admin_caches.get(idx);
futures.push(async move {
if let Some(client) = client {
let host = client.host.to_string();
// Wrap in timeout to ensure we don't hang on dead peers
match timeout(peer_timeout, client.local_storage_info()).await {
Ok(Ok(info)) => Some(info),
Ok(Ok(info)) => {
update_storage_info_cache(cache, &host, &info);
Some(info)
}
Ok(Err(err)) => {
warn!("peer {} storage_info failed: {}", host, err);
Some(rustfs_madmin::StorageInfo {
disks: get_offline_disks(&host, &endpoints),
..Default::default()
})
handle_peer_failure(cache, &host, &endpoints)
}
Err(_) => {
warn!("peer {} storage_info timed out after {:?}", host, peer_timeout);
client.evict_connection().await;
Some(rustfs_madmin::StorageInfo {
disks: get_offline_disks(&host, &endpoints),
..Default::default()
})
handle_peer_failure(cache, &host, &endpoints)
}
}
} else {
@@ -299,24 +321,27 @@ impl NotificationSys {
pub async fn server_info(&self) -> Vec<ServerProperties> {
let mut futures = Vec::with_capacity(self.peer_clients.len());
let endpoints = get_global_endpoints();
let peer_timeout = Duration::from_secs(2);
let peer_timeout = Duration::from_secs(5);
for client in self.peer_clients.iter() {
for (idx, client) in self.peer_clients.iter().enumerate() {
let endpoints = endpoints.clone();
let cache = self.peer_admin_caches.get(idx);
futures.push(async move {
if let Some(client) = client {
let host = client.host.to_string();
match timeout(peer_timeout, client.server_info()).await {
Ok(Ok(info)) => info,
Ok(Ok(info)) => {
update_server_info_cache(cache, &host, &info);
info
}
Ok(Err(err)) => {
warn!("peer {} server_info failed: {}", host, err);
// client.server_info handles eviction internally on error, but fallback needed
offline_server_properties(&host, &endpoints)
handle_server_info_failure(cache, &host, &endpoints)
}
Err(_) => {
warn!("peer {} server_info timed out after {:?}", host, peer_timeout);
client.evict_connection().await;
offline_server_properties(&host, &endpoints)
handle_server_info_failure(cache, &host, &endpoints)
}
}
} else {
@@ -841,6 +866,113 @@ where
}
}
/// Handle a peer failure for storage_info: return cached data if available,
/// or mark offline only after consecutive failures exceed the threshold.
fn handle_peer_failure(
cache: Option<&Mutex<PeerAdminCache>>,
host: &str,
endpoints: &EndpointServerPools,
) -> Option<StorageInfo> {
let cache = cache?;
let mut c = match cache.lock() {
Ok(cache) => cache,
Err(poisoned) => {
warn!("peer {host} storage_info cache mutex poisoned");
poisoned.into_inner()
}
};
c.storage_failures += 1;
if let Some(ref cached) = c.last_storage_info
&& c.storage_failures < CONSECUTIVE_FAILURE_THRESHOLD
{
return Some(cached.clone());
}
if c.storage_failures >= CONSECUTIVE_FAILURE_THRESHOLD {
return Some(StorageInfo {
disks: get_offline_disks(host, endpoints),
..Default::default()
});
}
None
}
fn update_storage_info_cache(cache: Option<&Mutex<PeerAdminCache>>, host: &str, info: &StorageInfo) {
let Some(cache) = cache else {
return;
};
let mut c = match cache.lock() {
Ok(cache) => cache,
Err(poisoned) => {
warn!("peer {host} storage_info cache mutex poisoned");
poisoned.into_inner()
}
};
c.last_storage_info = Some(info.clone());
c.storage_failures = 0;
}
/// Handle a peer failure for server_info: return cached data if available,
/// or mark offline only after consecutive failures exceed the threshold.
fn handle_server_info_failure(
cache: Option<&Mutex<PeerAdminCache>>,
host: &str,
endpoints: &EndpointServerPools,
) -> ServerProperties {
let Some(cache) = cache else {
return initializing_server_properties(host);
};
let mut c = match cache.lock() {
Ok(cache) => cache,
Err(poisoned) => {
warn!("peer {host} server_info cache mutex poisoned");
poisoned.into_inner()
}
};
c.server_failures += 1;
if let Some(ref cached) = c.last_server_info
&& c.server_failures < CONSECUTIVE_FAILURE_THRESHOLD
{
return cached.clone();
}
if c.server_failures >= CONSECUTIVE_FAILURE_THRESHOLD {
return offline_server_properties(host, endpoints);
}
initializing_server_properties(host)
}
fn update_server_info_cache(cache: Option<&Mutex<PeerAdminCache>>, host: &str, info: &ServerProperties) {
let Some(cache) = cache else {
return;
};
let mut c = match cache.lock() {
Ok(cache) => cache,
Err(poisoned) => {
warn!("peer {host} server_info cache mutex poisoned");
poisoned.into_inner()
}
};
c.last_server_info = Some(info.clone());
c.server_failures = 0;
}
fn initializing_server_properties(host: &str) -> ServerProperties {
ServerProperties {
endpoint: host.to_string(),
state: ItemState::Initializing.to_string().to_owned(),
..Default::default()
}
}
fn offline_server_properties(host: &str, endpoints: &EndpointServerPools) -> ServerProperties {
ServerProperties {
uptime: GLOBAL_BOOT_TIME
@@ -964,6 +1096,7 @@ mod tests {
let sys = NotificationSys {
peer_clients: vec![None],
all_peer_clients: Vec::new(),
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
};
let err = sys
@@ -982,6 +1115,7 @@ mod tests {
let sys = NotificationSys {
peer_clients: vec![None],
all_peer_clients: Vec::new(),
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
};
let results = sys.load_transition_tier_config().await;
@@ -990,4 +1124,243 @@ mod tests {
assert!(results[0].err.is_some());
assert!(results[0].err.as_ref().unwrap().to_string().contains("peer is not reachable"));
}
// --- Tests for handle_peer_failure / handle_server_info_failure caching ---
#[test]
fn handle_peer_failure_first_failure_returns_none_when_no_cache() {
let cache = Mutex::new(PeerAdminCache::new());
let endpoints = EndpointServerPools::default();
let result = handle_peer_failure(Some(&cache), "peer-1", &endpoints);
assert!(result.is_none());
assert_eq!(cache.lock().unwrap().storage_failures, 1);
}
#[test]
fn handle_peer_failure_returns_cached_data_on_single_failure() {
let cached_info = StorageInfo {
disks: vec![rustfs_madmin::Disk {
endpoint: "disk-0".to_string(),
state: "ok".to_string(),
..Default::default()
}],
..Default::default()
};
let cache = Mutex::new(PeerAdminCache {
last_storage_info: Some(cached_info),
last_server_info: None,
storage_failures: 0,
server_failures: 0,
});
let endpoints = EndpointServerPools::default();
// First failure: should return cached data
let result = handle_peer_failure(Some(&cache), "peer-1", &endpoints);
let info = result.unwrap();
assert_eq!(info.disks.len(), 1);
assert_eq!(info.disks[0].state, "ok");
assert_eq!(cache.lock().unwrap().storage_failures, 1);
}
#[test]
fn handle_peer_failure_returns_offline_after_threshold_exceeded() {
let cached_info = StorageInfo {
disks: vec![rustfs_madmin::Disk {
endpoint: "disk-0".to_string(),
state: "ok".to_string(),
..Default::default()
}],
..Default::default()
};
let cache = Mutex::new(PeerAdminCache {
last_storage_info: Some(cached_info),
last_server_info: None,
storage_failures: CONSECUTIVE_FAILURE_THRESHOLD - 1,
server_failures: 0,
});
let endpoints = EndpointServerPools::default();
// This failure pushes us to the threshold => offline
let result = handle_peer_failure(Some(&cache), "peer-1", &endpoints);
assert!(result.is_some());
assert_eq!(cache.lock().unwrap().storage_failures, CONSECUTIVE_FAILURE_THRESHOLD);
}
#[test]
fn handle_server_info_failure_returns_cached_on_single_failure() {
let cached_props = ServerProperties {
endpoint: "peer-1".to_string(),
state: "online".to_string(),
..Default::default()
};
let cache = Mutex::new(PeerAdminCache {
last_storage_info: None,
last_server_info: Some(cached_props),
storage_failures: 0,
server_failures: 0,
});
let endpoints = EndpointServerPools::default();
let result = handle_server_info_failure(Some(&cache), "peer-1", &endpoints);
assert_eq!(result.endpoint, "peer-1");
assert_eq!(result.state, "online");
assert_eq!(cache.lock().unwrap().server_failures, 1);
}
#[test]
fn handle_server_info_failure_returns_initializing_before_threshold_without_cache() {
let cache = Mutex::new(PeerAdminCache::new());
let endpoints = EndpointServerPools::default();
let result = handle_server_info_failure(Some(&cache), "peer-1", &endpoints);
assert_eq!(result.endpoint, "peer-1");
assert_eq!(result.state, ItemState::Initializing.to_string());
assert!(result.disks.is_empty());
assert_eq!(cache.lock().unwrap().server_failures, 1);
}
#[test]
fn handle_server_info_failure_returns_offline_after_threshold() {
let cached_props = ServerProperties {
endpoint: "peer-1".to_string(),
state: "online".to_string(),
..Default::default()
};
let cache = Mutex::new(PeerAdminCache {
last_storage_info: None,
last_server_info: Some(cached_props),
storage_failures: 0,
server_failures: CONSECUTIVE_FAILURE_THRESHOLD - 1,
});
let endpoints = EndpointServerPools::default();
let result = handle_server_info_failure(Some(&cache), "peer-1", &endpoints);
assert_eq!(result.state, ItemState::Offline.to_string());
assert_eq!(cache.lock().unwrap().server_failures, CONSECUTIVE_FAILURE_THRESHOLD);
}
#[test]
fn success_resets_failure_counters_independently() {
let cache = Mutex::new(PeerAdminCache {
last_storage_info: None,
last_server_info: None,
storage_failures: 2,
server_failures: 2,
});
{
let mut c = cache.lock().unwrap();
c.last_storage_info = Some(StorageInfo::default());
c.storage_failures = 0;
}
let cache = cache.lock().unwrap();
assert_eq!(cache.storage_failures, 0);
assert_eq!(cache.server_failures, 2);
}
#[test]
fn storage_failures_do_not_affect_server_failures() {
let cache = Mutex::new(PeerAdminCache {
last_storage_info: Some(StorageInfo::default()),
last_server_info: Some(ServerProperties {
endpoint: "peer-1".to_string(),
state: "online".to_string(),
..Default::default()
}),
storage_failures: CONSECUTIVE_FAILURE_THRESHOLD - 1,
server_failures: 0,
});
let endpoints = EndpointServerPools::default();
let storage_result = handle_peer_failure(Some(&cache), "peer-1", &endpoints);
assert!(storage_result.is_some());
let server_result = handle_server_info_failure(Some(&cache), "peer-1", &endpoints);
assert_eq!(server_result.state, "online");
assert_eq!(cache.lock().unwrap().server_failures, 1);
}
#[test]
fn poisoned_admin_cache_mutex_still_returns_fallbacks() {
let storage_cache = Mutex::new(PeerAdminCache::new());
let server_cache = Mutex::new(PeerAdminCache::new());
let endpoints = EndpointServerPools::default();
let _ = std::panic::catch_unwind(|| {
let _guard = storage_cache.lock().expect("test: poison storage cache mutex");
panic!("poison storage cache mutex");
});
let _ = std::panic::catch_unwind(|| {
let _guard = server_cache.lock().expect("test: poison server cache mutex");
panic!("poison server cache mutex");
});
let storage_result = handle_peer_failure(Some(&storage_cache), "peer-1", &endpoints);
assert!(storage_result.is_none());
let server_result = handle_server_info_failure(Some(&server_cache), "peer-1", &endpoints);
assert_eq!(server_result.endpoint, "peer-1");
assert_eq!(server_result.state, ItemState::Initializing.to_string());
}
#[test]
fn poisoned_admin_cache_recovers_on_success_and_resets_failures() {
let storage_cache = Mutex::new(PeerAdminCache {
last_storage_info: None,
last_server_info: None,
storage_failures: CONSECUTIVE_FAILURE_THRESHOLD - 1,
server_failures: 0,
});
let server_cache = Mutex::new(PeerAdminCache {
last_storage_info: None,
last_server_info: None,
storage_failures: 0,
server_failures: CONSECUTIVE_FAILURE_THRESHOLD - 1,
});
let endpoints = EndpointServerPools::default();
let _ = std::panic::catch_unwind(|| {
let _guard = storage_cache.lock().expect("test: poison storage cache mutex");
panic!("poison storage cache mutex");
});
let _ = std::panic::catch_unwind(|| {
let _guard = server_cache.lock().expect("test: poison server cache mutex");
panic!("poison server cache mutex");
});
update_storage_info_cache(
Some(&storage_cache),
"peer-1",
&StorageInfo {
disks: vec![rustfs_madmin::Disk {
endpoint: "disk-0".to_string(),
state: "ok".to_string(),
..Default::default()
}],
..Default::default()
},
);
update_server_info_cache(
Some(&server_cache),
"peer-1",
&ServerProperties {
endpoint: "peer-1".to_string(),
state: "online".to_string(),
..Default::default()
},
);
let storage_result = handle_peer_failure(Some(&storage_cache), "peer-1", &endpoints);
assert!(storage_result.is_some());
assert_eq!(storage_result.unwrap().disks[0].state, "ok");
let server_result = handle_server_info_failure(Some(&server_cache), "peer-1", &endpoints);
assert_eq!(server_result.state, "online");
}
}
File diff suppressed because it is too large Load Diff
+767
View File
@@ -0,0 +1,767 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(feature = "rio-v2")]
pub use rustfs_rio_v2::*;
#[cfg(not(feature = "rio-v2"))]
pub use rustfs_rio::*;
use bytes::Bytes;
use rustfs_utils::CompressionAlgorithm;
use std::str::FromStr;
use tokio::io::AsyncRead;
#[cfg(feature = "rio-v2")]
const MINIO_S2_COMPRESSION_SCHEME: &str = "klauspost/compress/s2";
#[cfg(feature = "rio-v2")]
const ENCRYPTED_S2_PADDING_MULTIPLE: usize = 256;
pub const fn backend_name() -> &'static str {
#[cfg(feature = "rio-v2")]
{
"rio-v2"
}
#[cfg(not(feature = "rio-v2"))]
{
"legacy-rio"
}
}
pub fn compression_metadata_value(algorithm: CompressionAlgorithm) -> String {
#[cfg(feature = "rio-v2")]
{
let _ = algorithm;
MINIO_S2_COMPRESSION_SCHEME.to_string()
}
#[cfg(not(feature = "rio-v2"))]
{
algorithm.to_string()
}
}
pub fn compression_scheme_to_algorithm(scheme: &str) -> std::io::Result<CompressionAlgorithm> {
#[cfg(feature = "rio-v2")]
if scheme.eq_ignore_ascii_case(MINIO_S2_COMPRESSION_SCHEME) {
// rio_v2 currently routes all compressed-object handling through the S2
// reader implementation, so the enum is only a placeholder token here.
return Ok(CompressionAlgorithm::default());
}
CompressionAlgorithm::from_str(scheme)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReadCompressionBackend {
Legacy,
V2,
}
pub fn compression_scheme_to_read_plan(scheme: &str) -> std::io::Result<(CompressionAlgorithm, ReadCompressionBackend)> {
#[cfg(feature = "rio-v2")]
if scheme.eq_ignore_ascii_case(MINIO_S2_COMPRESSION_SCHEME) {
return Ok((CompressionAlgorithm::default(), ReadCompressionBackend::V2));
}
Ok((CompressionAlgorithm::from_str(scheme)?, ReadCompressionBackend::Legacy))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReadEncryptionBackend {
Legacy,
V2,
}
pub fn compression_index_storage_bytes(index: &Index) -> Bytes {
#[cfg(feature = "rio-v2")]
{
minio_index_storage_bytes(index)
}
#[cfg(not(feature = "rio-v2"))]
{
index.clone().into_vec()
}
}
pub fn decode_compression_index_bytes(bytes: &Bytes) -> Option<Index> {
#[cfg(feature = "rio-v2")]
{
if let Some(decoded) = decode_minio_index_bytes(bytes) {
return Some(decoded);
}
}
let mut decoded = Index::new();
if decoded.load(bytes.as_ref()).is_ok() {
return Some(decoded);
}
#[cfg(feature = "rio-v2")]
{
let restored = restore_legacy_index_headers(bytes.as_ref());
let mut decoded = Index::new();
if decoded.load(&restored).is_ok() {
return Some(decoded);
}
}
None
}
pub fn compression_reader<R>(reader: R, algorithm: CompressionAlgorithm, encrypted: bool) -> CompressReader<R>
where
R: AsyncRead + Unpin + Send + Sync,
{
#[cfg(feature = "rio-v2")]
{
if encrypted {
return CompressReader::with_encrypted_padding(reader, algorithm);
}
}
#[cfg(not(feature = "rio-v2"))]
let _ = encrypted;
CompressReader::new(reader, algorithm)
}
pub fn decompression_reader<R>(
reader: R,
algorithm: CompressionAlgorithm,
backend: ReadCompressionBackend,
) -> Box<dyn AsyncRead + Unpin + Send + Sync>
where
R: AsyncRead + Unpin + Send + Sync + 'static,
{
#[cfg(feature = "rio-v2")]
{
match backend {
ReadCompressionBackend::Legacy => Box::new(rustfs_rio::DecompressReader::new(reader, algorithm)),
ReadCompressionBackend::V2 => Box::new(rustfs_rio_v2::DecompressReader::new(reader, algorithm)),
}
}
#[cfg(not(feature = "rio-v2"))]
{
let _ = backend;
Box::new(rustfs_rio::DecompressReader::new(reader, algorithm))
}
}
pub fn decrypt_reader<R>(
reader: R,
key: [u8; 32],
base_nonce: [u8; 12],
backend: ReadEncryptionBackend,
sequence_number: u32,
) -> Box<dyn AsyncRead + Unpin + Send + Sync>
where
R: AsyncRead + Unpin + Send + Sync + 'static,
{
#[cfg(feature = "rio-v2")]
{
match backend {
ReadEncryptionBackend::Legacy => Box::new(rustfs_rio::DecryptReader::new(reader, key, base_nonce)),
ReadEncryptionBackend::V2 => {
Box::new(rustfs_rio_v2::DecryptReader::new_with_sequence(reader, key, base_nonce, sequence_number))
}
}
}
#[cfg(not(feature = "rio-v2"))]
{
let _ = (backend, sequence_number);
Box::new(rustfs_rio::DecryptReader::new(reader, key, base_nonce))
}
}
pub fn decrypt_reader_with_object_key<R>(
reader: R,
object_key: [u8; 32],
sequence_number: u32,
) -> Box<dyn AsyncRead + Unpin + Send + Sync>
where
R: AsyncRead + Unpin + Send + Sync + 'static,
{
#[cfg(feature = "rio-v2")]
{
Box::new(rustfs_rio_v2::DecryptReader::new_with_object_key_and_sequence(
reader,
object_key,
sequence_number,
))
}
#[cfg(not(feature = "rio-v2"))]
{
let _ = sequence_number;
Box::new(rustfs_rio::DecryptReader::new(reader, object_key, [0u8; 12]))
}
}
pub fn decrypt_multipart_reader<R>(
reader: R,
key: [u8; 32],
base_nonce: [u8; 12],
multipart_parts: Vec<usize>,
backend: ReadEncryptionBackend,
sequence_number: u32,
) -> Box<dyn AsyncRead + Unpin + Send + Sync>
where
R: AsyncRead + Unpin + Send + Sync + 'static,
{
#[cfg(feature = "rio-v2")]
{
match backend {
ReadEncryptionBackend::Legacy => {
Box::new(rustfs_rio::DecryptReader::new_multipart(reader, key, base_nonce, multipart_parts))
}
ReadEncryptionBackend::V2 => Box::new(rustfs_rio_v2::DecryptReader::new_multipart_with_sequence(
reader,
key,
base_nonce,
multipart_parts,
sequence_number,
)),
}
}
#[cfg(not(feature = "rio-v2"))]
{
let _ = (backend, sequence_number);
Box::new(rustfs_rio::DecryptReader::new_multipart(reader, key, base_nonce, multipart_parts))
}
}
pub fn decrypt_multipart_reader_with_object_key<R>(
reader: R,
object_key: [u8; 32],
multipart_parts: Vec<usize>,
sequence_number: u32,
) -> Box<dyn AsyncRead + Unpin + Send + Sync>
where
R: AsyncRead + Unpin + Send + Sync + 'static,
{
#[cfg(feature = "rio-v2")]
{
Box::new(rustfs_rio_v2::DecryptReader::new_multipart_with_object_key_and_sequence(
reader,
object_key,
multipart_parts,
sequence_number,
))
}
#[cfg(not(feature = "rio-v2"))]
{
let _ = sequence_number;
Box::new(rustfs_rio::DecryptReader::new_multipart(reader, object_key, [0u8; 12], multipart_parts))
}
}
#[cfg(feature = "rio-v2")]
fn restore_legacy_index_headers(bytes: &[u8]) -> Vec<u8> {
if bytes.is_empty() {
return Vec::new();
}
const S2_INDEX_HEADER: &[u8] = b"s2idx\x00";
const S2_INDEX_TRAILER: &[u8] = b"\x00xdi2s";
let mut restored = Vec::with_capacity(4 + S2_INDEX_HEADER.len() + bytes.len() + 4 + S2_INDEX_TRAILER.len());
restored.extend_from_slice(&[0x99, 0x2A, 0x4D, 0x18]);
restored.extend_from_slice(S2_INDEX_HEADER);
restored.extend_from_slice(bytes);
let total_size = (restored.len() + 4 + S2_INDEX_TRAILER.len()) as u32;
restored.extend_from_slice(&total_size.to_le_bytes());
restored.extend_from_slice(S2_INDEX_TRAILER);
let chunk_len = restored.len() - 4;
restored[1] = chunk_len as u8;
restored[2] = (chunk_len >> 8) as u8;
restored[3] = (chunk_len >> 16) as u8;
restored
}
#[derive(Debug, Clone, Copy)]
pub struct WriteEncryption {
key_bytes: [u8; 32],
mode: WriteEncryptionMode,
}
#[derive(Debug, Clone, Copy)]
enum WriteEncryptionMode {
SinglepartObjectKey,
Singlepart {
base_nonce: [u8; 12],
},
MultipartLegacy {
base_nonce: [u8; 12],
multipart_part_number: usize,
},
MultipartObjectKey {
multipart_part_number: u32,
},
}
impl WriteEncryption {
pub const fn singlepart_object_key(object_key: [u8; 32]) -> Self {
Self {
key_bytes: object_key,
mode: WriteEncryptionMode::SinglepartObjectKey,
}
}
pub const fn singlepart(key_bytes: [u8; 32], base_nonce: [u8; 12]) -> Self {
Self {
key_bytes,
mode: WriteEncryptionMode::Singlepart { base_nonce },
}
}
pub const fn multipart(key_bytes: [u8; 32], base_nonce: [u8; 12], multipart_part_number: usize) -> Self {
Self {
key_bytes,
mode: WriteEncryptionMode::MultipartLegacy {
base_nonce,
multipart_part_number,
},
}
}
pub const fn multipart_object_key(object_key: [u8; 32], multipart_part_number: u32) -> Self {
Self {
key_bytes: object_key,
mode: WriteEncryptionMode::MultipartObjectKey { multipart_part_number },
}
}
}
#[derive(Debug, Clone, Default)]
pub struct WritePlan {
compression: Option<CompressionAlgorithm>,
encryption: Option<WriteEncryption>,
}
impl WritePlan {
pub const fn new() -> Self {
Self {
compression: None,
encryption: None,
}
}
pub const fn with_compression(mut self, algorithm: CompressionAlgorithm) -> Self {
self.compression = Some(algorithm);
self
}
pub const fn with_encryption(mut self, encryption: WriteEncryption) -> Self {
self.encryption = Some(encryption);
self
}
pub const fn is_passthrough(&self) -> bool {
self.compression.is_none() && self.encryption.is_none()
}
pub fn apply(self, mut reader: HashReader, actual_size: i64) -> std::io::Result<HashReader> {
let encrypted = self.encryption.is_some();
if let Some(algorithm) = self.compression {
reader = HashReader::from_reader(
compression_reader(reader, algorithm, encrypted),
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
None,
None,
false,
)?;
}
if let Some(encryption) = self.encryption {
reader = match encryption.mode {
WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
#[cfg(feature = "rio-v2")]
EncryptReader::new_with_object_key(reader, encryption.key_bytes),
#[cfg(not(feature = "rio-v2"))]
EncryptReader::new(reader, encryption.key_bytes, [0u8; 12]),
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
None,
None,
false,
)?,
WriteEncryptionMode::Singlepart { base_nonce } => HashReader::from_reader(
EncryptReader::new(reader, encryption.key_bytes, base_nonce),
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
None,
None,
false,
)?,
WriteEncryptionMode::MultipartLegacy {
base_nonce,
multipart_part_number,
} => HashReader::from_reader(
EncryptReader::new_multipart(reader, encryption.key_bytes, base_nonce, multipart_part_number),
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
None,
None,
false,
)?,
WriteEncryptionMode::MultipartObjectKey { multipart_part_number } => HashReader::from_reader(
#[cfg(feature = "rio-v2")]
EncryptReader::new_multipart_with_object_key(reader, encryption.key_bytes, multipart_part_number),
#[cfg(not(feature = "rio-v2"))]
EncryptReader::new_multipart(reader, encryption.key_bytes, [0u8; 12], multipart_part_number as usize),
HashReader::SIZE_PRESERVE_LAYER,
actual_size,
None,
None,
false,
)?,
};
}
Ok(reader)
}
}
#[cfg(test)]
mod tests {
use super::*;
use rustfs_utils::CompressionAlgorithm;
use std::io::Cursor;
use tokio::io::AsyncReadExt;
#[cfg(feature = "rio-v2")]
fn s2_chunk_types(stream: &[u8]) -> Vec<u8> {
let mut chunk_types = Vec::new();
let mut offset = 0usize;
while offset + 4 <= stream.len() {
let chunk_type = stream[offset];
let chunk_len =
(stream[offset + 1] as usize) | ((stream[offset + 2] as usize) << 8) | ((stream[offset + 3] as usize) << 16);
chunk_types.push(chunk_type);
offset += 4 + chunk_len;
}
chunk_types
}
#[tokio::test]
async fn write_plan_passthrough_keeps_plaintext() {
let plaintext = b"write-plan-plain".to_vec();
let reader = HashReader::from_stream(
Cursor::new(plaintext.clone()),
plaintext.len() as i64,
plaintext.len() as i64,
None,
None,
false,
)
.expect("create hash reader");
let mut reader = WritePlan::new()
.apply(reader, plaintext.len() as i64)
.expect("apply passthrough plan");
let mut actual = Vec::new();
reader.read_to_end(&mut actual).await.expect("read passthrough stream");
assert_eq!(actual, plaintext);
}
#[tokio::test]
async fn write_plan_compress_then_encrypt_multipart_roundtrip() {
let plaintext = b"abcdefghijklmnopqrstuvwxyz".repeat(128);
let actual_size = plaintext.len() as i64;
let key_bytes = [0x5Au8; 32];
let base_nonce = [0xA5u8; 12];
let part_number = 7;
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
.expect("create hash reader");
let mut transformed = WritePlan::new()
.with_compression(CompressionAlgorithm::default())
.with_encryption(WriteEncryption::multipart(key_bytes, base_nonce, part_number))
.apply(reader, actual_size)
.expect("apply transform plan");
let mut ciphertext = Vec::new();
transformed
.read_to_end(&mut ciphertext)
.await
.expect("read transformed ciphertext");
let decrypt_reader = DecryptReader::new_multipart(Cursor::new(ciphertext), key_bytes, base_nonce, vec![part_number]);
let mut decompressed = DecompressReader::new(Box::new(decrypt_reader), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressed
.read_to_end(&mut actual)
.await
.expect("decrypt and decompress transformed stream");
assert_eq!(actual, plaintext);
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn write_plan_supports_singlepart_object_key_encryption_roundtrip() {
let plaintext = b"singlepart-object-key".repeat(512);
let actual_size = plaintext.len() as i64;
let object_key = [0x7Cu8; 32];
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
.expect("create hash reader");
let mut transformed = WritePlan::new()
.with_encryption(WriteEncryption::singlepart_object_key(object_key))
.apply(reader, actual_size)
.expect("apply singlepart object-key plan");
let mut encrypted = Vec::new();
transformed
.read_to_end(&mut encrypted)
.await
.expect("read encrypted object-key stream");
let mut decrypted = DecryptReader::new_with_object_key(Cursor::new(encrypted), object_key);
let mut actual = Vec::new();
decrypted.read_to_end(&mut actual).await.expect("decrypt object-key stream");
assert_eq!(actual, plaintext);
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn write_plan_supports_multipart_object_key_encryption_roundtrip() {
let plaintext = b"multipart-object-key-".repeat(4096);
let actual_size = plaintext.len() as i64;
let object_key = [0x2Du8; 32];
let part_number = 3u32;
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
.expect("create hash reader");
let mut transformed = WritePlan::new()
.with_encryption(WriteEncryption::multipart_object_key(object_key, part_number))
.apply(reader, actual_size)
.expect("apply multipart object-key encryption");
let mut ciphertext = Vec::new();
transformed
.read_to_end(&mut ciphertext)
.await
.expect("read multipart object-key ciphertext");
let mut actual = Vec::new();
DecryptReader::new_multipart_with_object_key(Cursor::new(ciphertext), object_key, vec![part_number as usize])
.read_to_end(&mut actual)
.await
.expect("decrypt multipart object-key ciphertext");
assert_eq!(actual, plaintext);
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn write_plan_rio_v2_compression_emits_s2_stream_and_seekable_index() {
let plaintext = b"rustfs-rio-v2-s2-".repeat(600_000);
let actual_size = plaintext.len() as i64;
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
.expect("create hash reader");
let mut transformed = WritePlan::new()
.with_compression(CompressionAlgorithm::default())
.apply(reader, actual_size)
.expect("apply compression plan");
let mut compressed = Vec::new();
transformed
.read_to_end(&mut compressed)
.await
.expect("read compressed stream");
assert!(
compressed.starts_with(b"\xff\x06\x00\x00S2sTwO"),
"rio_v2 compressed stream must start with the S2 stream identifier"
);
let index = transformed
.try_get_index()
.cloned()
.expect("rio_v2 compressed stream should expose a compression index");
let (compressed_offset, uncompressed_offset) = index.find(2 * 1024 * 1024).expect("seek into compression index");
assert!(compressed_offset > 0, "expected a non-zero compressed offset for the second block");
assert!(uncompressed_offset > 0, "expected a non-zero uncompressed offset for the second block");
let mut decompressed = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressed.read_to_end(&mut actual).await.expect("decompress rio_v2 stream");
assert_eq!(actual, plaintext);
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn write_plan_rio_v2_small_compression_skips_index_below_minio_threshold() {
let plaintext = b"rustfs-rio-v2-s2-".repeat(32_768);
let actual_size = plaintext.len() as i64;
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
.expect("create hash reader");
let mut transformed = WritePlan::new()
.with_compression(CompressionAlgorithm::default())
.apply(reader, actual_size)
.expect("apply compression plan");
let mut compressed = Vec::new();
transformed
.read_to_end(&mut compressed)
.await
.expect("read compressed stream");
assert!(
transformed.try_get_index().is_none(),
"rio_v2 should match MinIO and skip compression indexes for small objects"
);
let mut decompressed = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressed.read_to_end(&mut actual).await.expect("decompress rio_v2 stream");
assert_eq!(actual, plaintext);
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn rio_v2_singlepart_encrypt_decrypt_roundtrip_preserves_small_compressed_stream() {
let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec();
let key_bytes = [0x33u8; 32];
let base_nonce = [0x55u8; 12];
let mut compressed = Vec::new();
CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
.read_to_end(&mut compressed)
.await
.expect("compress plaintext");
let mut encrypted = Vec::new();
EncryptReader::new(Cursor::new(compressed), key_bytes, base_nonce)
.read_to_end(&mut encrypted)
.await
.expect("encrypt compressed stream");
let decrypt_reader = DecryptReader::new(Cursor::new(encrypted), key_bytes, base_nonce);
let mut decompressed = DecompressReader::new(Box::new(decrypt_reader), CompressionAlgorithm::default());
let mut actual = Vec::new();
decompressed
.read_to_end(&mut actual)
.await
.expect("decrypt and decompress small stream");
assert_eq!(actual, plaintext);
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn rio_v2_compress_then_encrypt_adds_s2_padding_frames() {
let plaintext = b"padding-check-".repeat(4097);
let actual_size = plaintext.len() as i64;
let key_bytes = [0x1Bu8; 32];
let base_nonce = [0xC4u8; 12];
let reader = HashReader::from_stream(Cursor::new(plaintext.clone()), actual_size, actual_size, None, None, false)
.expect("create hash reader");
let mut transformed = WritePlan::new()
.with_compression(CompressionAlgorithm::default())
.with_encryption(WriteEncryption::singlepart(key_bytes, base_nonce))
.apply(reader, actual_size)
.expect("apply transform plan");
let mut ciphertext = Vec::new();
transformed
.read_to_end(&mut ciphertext)
.await
.expect("read transformed ciphertext");
let mut decrypted_compressed = Vec::new();
DecryptReader::new(Cursor::new(ciphertext), key_bytes, base_nonce)
.read_to_end(&mut decrypted_compressed)
.await
.expect("decrypt compressed stream");
assert_eq!(decrypted_compressed.len() % ENCRYPTED_S2_PADDING_MULTIPLE, 0);
let chunk_types = s2_chunk_types(&decrypted_compressed);
assert!(
chunk_types.contains(&0xfe),
"rio_v2 compressed+encrypted streams must include S2 padding frames before encryption"
);
let mut actual = Vec::new();
DecompressReader::new(Cursor::new(decrypted_compressed), CompressionAlgorithm::default())
.read_to_end(&mut actual)
.await
.expect("decompress padded stream");
assert_eq!(actual, plaintext);
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn rio_v2_decompress_reader_returns_bytes_on_first_read() {
let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec();
let mut compressed = Vec::new();
CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
.read_to_end(&mut compressed)
.await
.expect("compress plaintext");
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut buf = [0u8; 64];
let n = decompressor.read(&mut buf).await.expect("read first decompressed chunk");
assert!(n > 0);
assert_eq!(&buf[..n], plaintext.as_slice());
}
#[cfg(feature = "rio-v2")]
#[tokio::test]
async fn rio_v2_decompress_reader_returns_bytes_on_first_large_read() {
let plaintext = b"abcdefghijklmnopqrstuvwxyz".to_vec();
let mut compressed = Vec::new();
CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default())
.read_to_end(&mut compressed)
.await
.expect("compress plaintext");
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
let mut buf = [0u8; 8192];
let n = decompressor.read(&mut buf).await.expect("read first decompressed chunk");
assert!(n > 0);
assert_eq!(&buf[..n], plaintext.as_slice());
}
}
+208 -22
View File
@@ -36,7 +36,7 @@ use metrics::counter;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC,
global_internode_metrics,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
};
use rustfs_protos::evict_failed_connection;
use rustfs_protos::proto_gen::node_service::RenamePartRequest;
@@ -74,6 +74,9 @@ enum FailureHealthAction {
IgnoreFailure,
}
const REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS: usize = 2;
const REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(20);
async fn copy_stream_with_buffer<R, W>(reader: &mut R, writer: &mut W, buffer_size: usize) -> io::Result<u64>
where
R: AsyncRead + Unpin,
@@ -119,6 +122,10 @@ impl RemoteDisk {
err_text.contains("httpreader stream error") || err_text.contains("error decoding response body")
}
fn is_retryable_open_write_error(err: &DiskError) -> bool {
err.is_retryable_internode_write_failure()
}
pub(crate) async fn new(ep: &Endpoint, opt: &DiskOption, data_transport: Arc<dyn InternodeDataTransport>) -> Result<Self> {
let addr = if let Some(port) = ep.url.port() {
format!("{}://{}:{}", ep.url.scheme(), ep.url.host_str().unwrap(), port)
@@ -156,6 +163,50 @@ impl RemoteDisk {
self.health.last_capacity_snapshot()
}
async fn open_write_with_retry(&self, request: WriteStreamRequest) -> Result<FileWriter> {
let mut attempt = 1;
let mut last_retry_classification = None;
loop {
match self.data_transport.open_write(request.clone()).await {
Ok(writer) => {
if attempt > 1
&& let Some(classification) = last_retry_classification
{
global_internode_metrics().record_retry_success_for_operation_and_backend(
rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
classification,
);
}
return Ok(writer);
}
Err(err) if attempt < REMOTE_DISK_OPEN_WRITE_MAX_ATTEMPTS && Self::is_retryable_open_write_error(&err) => {
if let Some(classification) = err.internode_http_error_kind() {
let classification = classification.metric_label();
global_internode_metrics().record_retry_for_operation_and_backend(
rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
classification,
);
last_retry_classification = Some(classification);
}
debug!(
endpoint = %request.endpoint,
volume = %request.volume,
path = %request.path,
append = request.append,
size = request.size,
attempt,
"retrying remote open_write after retryable transport error"
);
tokio::time::sleep(REMOTE_DISK_OPEN_WRITE_RETRY_BACKOFF).await;
attempt += 1;
}
Err(err) => return Err(err),
}
}
}
pub fn record_capacity_probe(&self, total: u64, used: u64, free: u64) {
self.health.record_capacity_probe(total, used, free);
}
@@ -1231,6 +1282,11 @@ impl DiskAPI for RemoteDisk {
let bucket = opts.bucket.clone();
let base_dir = opts.base_dir.clone();
let disk_for_log = disk.clone();
let timeout_duration = if opts.skip_total_timeout {
Duration::ZERO
} else {
get_drive_walkdir_timeout()
};
self.execute_with_timeout_for_op_and_health_action(
"walk_dir",
@@ -1278,7 +1334,7 @@ impl DiskAPI for RemoteDisk {
Err(last_err.unwrap_or_else(|| DiskError::other("walk_dir retry exhausted without captured error")))
},
get_drive_walkdir_timeout(),
timeout_duration,
FailureHealthAction::IgnoreFailure,
)
.await
@@ -1342,16 +1398,15 @@ impl DiskAPI for RemoteDisk {
return Err(DiskError::FaultyDisk);
}
let disk = self.disk_ref().await;
self.data_transport
.open_write(WriteStreamRequest {
endpoint: self.endpoint.grid_host(),
disk,
volume: volume.to_string(),
path: path.to_string(),
append: true,
size: 0,
})
.await
self.open_write_with_retry(WriteStreamRequest {
endpoint: self.endpoint.grid_host(),
disk,
volume: volume.to_string(),
path: path.to_string(),
append: true,
size: 0,
})
.await
}
#[tracing::instrument(level = "debug", skip(self))]
@@ -1368,16 +1423,15 @@ impl DiskAPI for RemoteDisk {
return Err(DiskError::FaultyDisk);
}
let disk = self.disk_ref().await;
self.data_transport
.open_write(WriteStreamRequest {
endpoint: self.endpoint.grid_host(),
disk,
volume: volume.to_string(),
path: path.to_string(),
append: false,
size: file_size,
})
.await
self.open_write_with_retry(WriteStreamRequest {
endpoint: self.endpoint.grid_host(),
disk,
volume: volume.to_string(),
path: path.to_string(),
append: false,
size: file_size,
})
.await
}
#[tracing::instrument(level = "debug", skip(self))]
@@ -1832,6 +1886,35 @@ mod tests {
}
}
#[derive(Debug, Clone)]
enum OpenWriteTestStep {
Error(DiskError),
Success,
}
#[derive(Debug, Clone, Default)]
struct RetryingOpenWriteInternodeDataTransport {
calls: Arc<StdMutex<Vec<RecordedTransportCall>>>,
steps: Arc<StdMutex<Vec<OpenWriteTestStep>>>,
}
impl RetryingOpenWriteInternodeDataTransport {
fn with_steps(steps: Vec<OpenWriteTestStep>) -> Self {
Self {
calls: Arc::new(StdMutex::new(Vec::new())),
steps: Arc::new(StdMutex::new(steps)),
}
}
fn calls(&self) -> Vec<RecordedTransportCall> {
self.calls.lock().expect("recorded transport calls lock poisoned").clone()
}
fn record(&self, call: RecordedTransportCall) {
self.calls.lock().expect("recorded transport calls lock poisoned").push(call);
}
}
#[derive(Debug, Default)]
struct EmptyTestReader;
@@ -1916,6 +1999,34 @@ mod tests {
}
}
#[async_trait::async_trait]
impl InternodeDataTransport for RetryingOpenWriteInternodeDataTransport {
async fn open_read(&self, _request: ReadStreamRequest) -> Result<FileReader> {
panic!("open_read should not be used in open_write retry test");
}
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
self.record(RecordedTransportCall::Write(request));
let step = self.steps.lock().expect("open_write retry steps lock poisoned").remove(0);
match step {
OpenWriteTestStep::Error(err) => Err(err),
OpenWriteTestStep::Success => Ok(Box::new(SinkTestWriter)),
}
}
async fn open_walk_dir(&self, _request: WalkDirStreamRequest) -> Result<FileReader> {
panic!("open_walk_dir should not be used in open_write retry test");
}
fn name(&self) -> &'static str {
"retrying-open-write"
}
fn capabilities(&self) -> InternodeDataTransportCapabilities {
InternodeDataTransportCapabilities::tcp_http()
}
}
async fn new_remote_disk_with_transport(data_transport: Arc<dyn InternodeDataTransport>) -> RemoteDisk {
let endpoint = Endpoint {
url: url::Url::parse("http://remote-node:9000/data/rustfs0").unwrap(),
@@ -2283,6 +2394,47 @@ mod tests {
}
}
#[tokio::test]
async fn test_remote_disk_create_file_retries_once_on_retryable_open_write_error() {
let transport = RetryingOpenWriteInternodeDataTransport::with_steps(vec![
OpenWriteTestStep::Error(DiskError::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::ConnectionReset,
))),
OpenWriteTestStep::Success,
]);
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
rustfs_io_metrics::internode_metrics::global_internode_metrics().reset_for_test();
let _created = remote_disk
.create_file("orig-bucket", "bucket", "object/part.1", 4096)
.await
.expect("retryable open_write error should recover");
let calls = transport.calls();
assert_eq!(calls.len(), 2, "create_file should retry exactly once");
let snapshot = rustfs_io_metrics::internode_metrics::global_internode_metrics().snapshot();
assert_eq!(snapshot.outgoing_requests_total, 0);
}
#[tokio::test]
async fn test_remote_disk_append_file_does_not_retry_non_retryable_open_write_error() {
let transport = RetryingOpenWriteInternodeDataTransport::with_steps(vec![OpenWriteTestStep::Error(DiskError::from(
rustfs_rio::new_test_internode_http_io_error(rustfs_rio::InternodeHttpErrorKind::DnsResolutionFailed),
))]);
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
let err = match remote_disk.append_file("bucket", "object/part.2").await {
Ok(_) => panic!("non-retryable open_write error should be returned directly"),
Err(err) => err,
};
assert_eq!(
err.internode_http_error_kind(),
Some(rustfs_rio::InternodeHttpErrorKind::DnsResolutionFailed)
);
assert_eq!(transport.calls().len(), 1, "append_file should not retry non-retryable errors");
}
#[tokio::test]
async fn test_remote_disk_walk_dir_uses_configured_data_transport() {
let transport = RecordingInternodeDataTransport::default();
@@ -2297,6 +2449,7 @@ mod tests {
forward_to: None,
limit: 10,
disk_id: String::new(),
..Default::default()
};
let expected_body = serde_json::to_vec(&opts).unwrap();
let mut writer = Vec::new();
@@ -2316,6 +2469,37 @@ mod tests {
}
}
#[tokio::test]
async fn test_remote_disk_walk_dir_preserves_skip_total_timeout_option() {
let transport = RecordingInternodeDataTransport::default();
let remote_disk = new_remote_disk_with_transport(Arc::new(transport.clone())).await;
let opts = WalkDirOptions {
bucket: "bucket".to_string(),
base_dir: "prefix".to_string(),
recursive: true,
skip_total_timeout: true,
..Default::default()
};
let mut writer = Vec::new();
remote_disk
.walk_dir(opts, &mut writer)
.await
.expect("walk_dir should be sent through configured data transport");
let calls = transport.calls();
assert_eq!(calls.len(), 1);
match &calls[0] {
RecordedTransportCall::WalkDir(request) => {
let sent_opts: WalkDirOptions =
serde_json::from_slice(&request.body).expect("walk_dir request body should deserialize");
assert!(sent_opts.skip_total_timeout);
assert_eq!(request.stall_timeout, Some(get_drive_walkdir_stall_timeout()));
}
other => panic!("expected walk-dir transport call, got {other:?}"),
}
}
#[tokio::test]
async fn test_remote_disk_walk_dir_retries_once_on_retryable_transport_error() {
let transport = RetryingWalkDirInternodeDataTransport::with_steps(vec![
@@ -2332,6 +2516,7 @@ mod tests {
forward_to: None,
limit: 10,
disk_id: String::new(),
..Default::default()
};
let mut writer = Vec::new();
@@ -2363,6 +2548,7 @@ mod tests {
forward_to: None,
limit: 10,
disk_id: String::new(),
..Default::default()
};
let mut writer = Vec::new();
+259 -22
View File
@@ -85,7 +85,6 @@ use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_object_capacity::capacity_scope::{
CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope,
};
use rustfs_rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _};
use rustfs_s3_types::EventName;
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
@@ -130,6 +129,8 @@ use tracing::error;
use tracing::{debug, info, warn};
use uuid::Uuid;
use crate::rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _};
pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
pub const MAX_PARTS_COUNT: usize = 10000;
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
@@ -773,6 +774,37 @@ fn delete_file_info_version_id(version_id: Option<Uuid>) -> Option<Uuid> {
}
}
fn object_fits_single_block(object_size: i64, block_size: usize) -> bool {
match usize::try_from(object_size) {
Ok(size) => size > 0 && size <= block_size,
Err(_) => false,
}
}
fn should_use_inline_small_fast_path(is_inline_buffer: bool, object_size: i64, block_size: usize) -> bool {
is_inline_buffer && object_fits_single_block(object_size, block_size)
}
fn should_use_single_block_non_inline_fast_path(is_inline_buffer: bool, object_size: i64, block_size: usize) -> bool {
!is_inline_buffer && object_fits_single_block(object_size, block_size)
}
enum SmallWritePath {
Inline,
SingleBlockNonInline,
Pipeline,
}
fn classify_small_write_path(is_inline_buffer: bool, object_size: i64, block_size: usize) -> SmallWritePath {
if should_use_inline_small_fast_path(is_inline_buffer, object_size, block_size) {
SmallWritePath::Inline
} else if should_use_single_block_non_inline_fast_path(is_inline_buffer, object_size, block_size) {
SmallWritePath::SingleBlockNonInline
} else {
SmallWritePath::Pipeline
}
}
#[async_trait::async_trait]
impl ObjectIO for SetDisks {
#[tracing::instrument(level = "debug", skip(self))]
@@ -1065,10 +1097,10 @@ impl ObjectIO for SetDisks {
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
);
let use_fast_path = is_inline_buffer && data.size() <= fi.erasure.block_size as i64;
let write_path = classify_small_write_path(is_inline_buffer, data.size(), fi.erasure.block_size);
let (reader, w_size) = if use_fast_path {
match Arc::new(erasure)
let (reader, w_size) = match write_path {
SmallWritePath::Inline => match Arc::new(erasure)
.encode_inline_small(stream, &mut writers, write_quorum)
.await
{
@@ -1077,15 +1109,24 @@ impl ObjectIO for SetDisks {
error!("encode_inline_small err {:?}", e);
return Err(e.into());
}
}
} else {
match Arc::new(erasure).encode(stream, &mut writers, write_quorum).await {
},
SmallWritePath::SingleBlockNonInline => match Arc::new(erasure)
.encode_single_block_non_inline(stream, &mut writers, write_quorum)
.await
{
Ok((r, w)) => (r, w),
Err(e) => {
error!("encode_single_block_non_inline err {:?}", e);
return Err(e.into());
}
},
SmallWritePath::Pipeline => match Arc::new(erasure).encode(stream, &mut writers, write_quorum).await {
Ok((r, w)) => (r, w),
Err(e) => {
error!("encode err {:?}", e);
return Err(e.into());
}
}
},
};
let _ = mem::replace(&mut data.stream, reader);
@@ -1106,7 +1147,7 @@ impl ObjectIO for SetDisks {
insert_str(&mut user_defined, SUFFIX_COMPRESSION_SIZE, w_size.to_string());
}
let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec());
let index_op = data.stream.try_get_index().map(crate::rio::compression_index_storage_bytes);
//TODO: userDefined
@@ -1171,7 +1212,7 @@ impl ObjectIO for SetDisks {
object_lock_guard = Some(self.acquire_write_lock_diag("put_object_commit", bucket, object).await?);
}
let (online_disks, _, op_old_dir) = Self::rename_data(
let (online_disks, _, op_old_dir, cleanup_disks) = Self::rename_data(
&shuffle_disks,
RUSTFS_META_TMP_BUCKET,
tmp_dir.as_str(),
@@ -1183,7 +1224,7 @@ impl ObjectIO for SetDisks {
.await?;
if let Some(old_dir) = op_old_dir {
self.commit_rename_data_dir(&online_disks, bucket, object, &old_dir.to_string(), write_quorum)
self.commit_rename_data_dir(&cleanup_disks, bucket, object, &old_dir.to_string(), write_quorum)
.await?;
}
@@ -1204,12 +1245,46 @@ impl ObjectIO for SetDisks {
fi.is_latest = true;
if issue3031_diag_enabled() {
let online_success_count = online_disks.iter().filter(|disk| disk.is_some()).count();
warn!(
target: "rustfs_ecstore::set_disk",
bucket = %bucket,
object = %object,
tmp_dir = %tmp_dir,
data_dir = ?fi.data_dir,
write_quorum,
online_success_count,
op_old_dir = ?op_old_dir,
"issue3031_put_object_commit_succeeded"
);
}
Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended))
}
.await;
if issue3031_diag_enabled() {
warn!(
target: "rustfs_ecstore::set_disk",
bucket = %bucket,
object = %object,
tmp_dir = %tmp_dir,
result = ?result.as_ref().map(|_| ()).map_err(|err| err.to_string()),
"issue3031_put_object_tmp_cleanup_start"
);
}
if let Err(err) = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await {
warn!(tmp_dir = %tmp_dir, error = ?err, "failed to cleanup put_object temporary data");
} else if issue3031_diag_enabled() {
warn!(
target: "rustfs_ecstore::set_disk",
bucket = %bucket,
object = %object,
tmp_dir = %tmp_dir,
"issue3031_put_object_tmp_cleanup_done"
);
}
result
@@ -1341,6 +1416,33 @@ impl SetDisks {
}
}
if issue3031_diag_enabled() {
let succeeded_count = resolution_by_object
.iter()
.filter(|resolution| matches!(resolution, ObjectLockResolution::Succeeded))
.count();
let failed_count = resolution_by_object
.iter()
.filter(|resolution| matches!(resolution, ObjectLockResolution::Failed))
.count();
let pending_count = resolution_by_object
.iter()
.filter(|resolution| matches!(resolution, ObjectLockResolution::Pending))
.count();
warn!(
target: "rustfs_ecstore::set_disk",
request_count = requests.len(),
locker_count = self.lockers.len(),
write_quorum,
succeeded_count,
failed_count,
pending_count,
pending_clients,
errors_by_object = ?errors_by_object,
"issue3031_delete_objects_dist_batch_lock_summary"
);
}
if !pending.is_empty() {
let cleanup_requests = requests.clone();
let lockers = self.lockers.clone();
@@ -1574,10 +1676,17 @@ impl ObjectOperations for SetDisks {
src_opts: &ObjectOptions,
dst_opts: &ObjectOptions,
) -> Result<ObjectInfo> {
// FIXME: TODO:
if !src_info.metadata_only {
return Err(StorageError::NotImplemented);
if path_join_buf(&[src_bucket, src_object]) != path_join_buf(&[dst_bucket, dst_object]) {
return Err(StorageError::NotImplemented);
}
// Self-copy with a data reader: write tier data back locally (de-tiering).
// Handles `mc cp --storage-class STANDARD obj obj` on a transitioned object.
if let Some(mut put_reader) = src_info.put_object_reader.take() {
return self.put_object(dst_bucket, dst_object, &mut put_reader, dst_opts).await;
}
// Same-key tiered copy without a pre-fetched reader: fall through to the metadata
// path so the caller gets a disk/quorum error rather than NotImplemented.
}
if path_join_buf(&[src_bucket, src_object]) != path_join_buf(&[dst_bucket, dst_object]) {
@@ -1775,6 +1884,7 @@ impl ObjectOperations for SetDisks {
batch = batch.add_write_lock(ObjectKey::new(bucket, dobj.object_name.clone()));
}
}
let unique_lock_count = batch.requests.len();
let mut failed_map = HashMap::new();
let mut _local_batch_guards: Vec<FastLockGuard> = Vec::with_capacity(batch.requests.len());
@@ -1798,6 +1908,24 @@ impl ObjectOperations for SetDisks {
}
}
if issue3031_diag_enabled() {
let failed_lock_count = failed_map.len();
let locked_object_count = locked_objects.len();
let dist_lock_id_count = dist_batch_lock_ids.iter().map(Vec::len).sum::<usize>();
warn!(
target: "rustfs_ecstore::set_disk",
bucket = %bucket,
requested_object_count = objects.len(),
unique_lock_count,
locked_object_count,
failed_lock_count,
dist_erasure,
dist_lock_id_count,
failed_objects = ?failed_map.keys().collect::<Vec<_>>(),
"issue3031_delete_objects_lock_batch_context"
);
}
// Mark failures for objects that could not be locked
for (i, dobj) in objects.iter().enumerate() {
if let Some(err) = failed_map.get(&(bucket.to_string(), dobj.object_name.clone())) {
@@ -2884,7 +3012,18 @@ impl MultipartOperations for SetDisks {
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
);
let (reader, w_size) = Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?; // TODO: delete temporary directory on error
let write_path = classify_small_write_path(false, data.size(), fi.erasure.block_size);
let (reader, w_size) = match write_path {
SmallWritePath::SingleBlockNonInline => {
Arc::new(erasure)
.encode_single_block_non_inline(stream, &mut writers, write_quorum)
.await?
}
SmallWritePath::Inline | SmallWritePath::Pipeline => {
Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?
}
}; // TODO: delete temporary directory on error
let _ = mem::replace(&mut data.stream, reader);
@@ -2897,7 +3036,7 @@ impl MultipartOperations for SetDisks {
)));
}
let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec());
let index_op = data.stream.try_get_index().map(crate::rio::compression_index_storage_bytes);
let mut etag = data.stream.try_resolve_etag().unwrap_or_default();
@@ -3760,7 +3899,7 @@ impl MultipartOperations for SetDisks {
self.cleanup_multipart_path(&parts).await;
let (online_disks, versions, op_old_dir) = Self::rename_data(
let (online_disks, versions, op_old_dir, cleanup_disks) = Self::rename_data(
&shuffle_disks,
RUSTFS_META_MULTIPART_BUCKET,
&upload_id_path,
@@ -3772,7 +3911,7 @@ impl MultipartOperations for SetDisks {
.await?;
if let Some(old_dir) = op_old_dir {
self.commit_rename_data_dir(&online_disks, bucket, object, &old_dir.to_string(), write_quorum)
self.commit_rename_data_dir(&cleanup_disks, bucket, object, &old_dir.to_string(), write_quorum)
.await?;
}
@@ -3856,10 +3995,7 @@ impl HealOperations for SetDisks {
let disks = disks.clone();
let (_, errs) = Self::read_all_fileinfo(&disks, "", bucket, object, version_id, false, false, false).await?;
if DiskError::is_all_not_found(&errs) {
warn!(
"heal_object failed, all obj part not found, bucket: {}, obj: {}, version_id: {}",
bucket, object, version_id
);
debug!(bucket, object, version_id, "heal_object skipped missing object");
let err = if !version_id.is_empty() {
Error::FileVersionNotFound
} else {
@@ -4647,6 +4783,7 @@ pub fn is_infrequent_access_class(storage_class: &str) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use crate::bucket::lifecycle::bucket_lifecycle_ops::TransitionedObject;
use crate::disk::CHECK_PART_UNKNOWN;
use crate::disk::CHECK_PART_VOLUME_NOT_FOUND;
use crate::disk::RUSTFS_META_BUCKET;
@@ -6187,6 +6324,10 @@ mod tests {
let data_dirs = vec![Some(uuid1), Some(uuid2), None];
let result = SetDisks::reduce_common_data_dir(&data_dirs, 2);
assert_eq!(result, None); // No UUID meets quorum of 2
let data_dirs = vec![Some(uuid1), Some(uuid1), None, None];
let result = SetDisks::reduce_common_data_dir(&data_dirs, 2);
assert_eq!(result, Some(uuid1)); // Ignore None votes; uuid1 should still meet quorum
}
#[test]
@@ -6624,6 +6765,47 @@ mod tests {
assert_eq!(delete_file_info_version_id(None), None);
}
#[test]
fn put_object_fast_path_selection_prefers_inline_only_when_inline_buffer_and_single_block() {
assert!(should_use_inline_small_fast_path(true, 1024, 4096));
assert!(!should_use_single_block_non_inline_fast_path(true, 1024, 4096));
assert!(matches!(classify_small_write_path(true, 1024, 4096), SmallWritePath::Inline));
assert!(!should_use_inline_small_fast_path(false, 1024, 4096));
assert!(should_use_single_block_non_inline_fast_path(false, 1024, 4096));
assert!(matches!(
classify_small_write_path(false, 1024, 4096),
SmallWritePath::SingleBlockNonInline
));
}
#[test]
fn put_object_fast_path_selection_rejects_zero_and_multi_block_payloads() {
assert!(!should_use_inline_small_fast_path(true, 0, 4096));
assert!(!should_use_single_block_non_inline_fast_path(false, 0, 4096));
assert!(matches!(classify_small_write_path(true, 0, 4096), SmallWritePath::Pipeline));
assert!(!should_use_inline_small_fast_path(true, -1, 4096));
assert!(!should_use_single_block_non_inline_fast_path(false, -1, 4096));
assert!(matches!(classify_small_write_path(false, -1, 4096), SmallWritePath::Pipeline));
assert!(!should_use_inline_small_fast_path(true, 8192, 4096));
assert!(!should_use_single_block_non_inline_fast_path(false, 8192, 4096));
assert!(matches!(classify_small_write_path(false, 8192, 4096), SmallWritePath::Pipeline));
}
#[test]
fn put_object_part_fast_path_selection_matches_single_block_non_inline_rules() {
assert!(should_use_single_block_non_inline_fast_path(false, 4096, 4096));
assert!(should_use_single_block_non_inline_fast_path(false, 2048, 4096));
assert!(!should_use_single_block_non_inline_fast_path(false, 4097, 4096));
assert!(!should_use_single_block_non_inline_fast_path(false, 0, 4096));
assert!(matches!(
classify_small_write_path(false, 4096, 4096),
SmallWritePath::SingleBlockNonInline
));
}
#[test]
fn test_is_cold_storage_class() {
// Test cold storage classes
@@ -6651,4 +6833,59 @@ mod tests {
assert!(!is_infrequent_access_class(storageclass::DEEP_ARCHIVE));
assert!(!is_infrequent_access_class(storageclass::EXPRESS_ONEZONE));
}
// Regression test: `mc cp --storage-class STANDARD` on a tiered object (self-copy) must not
// return NotImplemented. When the source object is tiered (transitioned_object.tier is
// non-empty) the usecase layer in object_usecase.rs intentionally leaves metadata_only=false
// so that the full copy path is taken. SetDisks::copy_object must therefore accept a
// same-bucket/same-key call even when metadata_only=false.
//
// Currently this test FAILS because the guard at set_disk.rs:1579 unconditionally rejects
// !metadata_only with StorageError::NotImplemented. Once the fix is applied the test will
// pass (or progress further through the copy path before failing on missing disk data).
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn copy_object_tiered_self_copy_does_not_return_not_implemented() {
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::Erasure).await;
let set_disks = make_test_set_disks(vec![Arc::new(LocalClient::with_manager(Arc::new(
rustfs_lock::GlobalLockManager::new(),
)))])
.await;
// Simulate a tiered object: metadata_only is false (set_disk must handle the full copy),
// and transitioned_object.tier is non-empty (the object lives on a remote tier).
let mut src_info = ObjectInfo {
metadata_only: false,
transitioned_object: TransitionedObject {
tier: "NEXTCLOUD".to_string(),
..Default::default()
},
..Default::default()
};
let result = set_disks
.copy_object(
"bucket",
"object",
"bucket",
"object",
&mut src_info,
&ObjectOptions::default(),
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await;
// The copy must not be rejected with NotImplemented. Any other outcome (Ok or a
// different error such as missing-disk / quorum) is acceptable here.
if let Err(ref err) = result {
assert!(
!matches!(err, StorageError::NotImplemented),
"tiered self-copy returned NotImplemented — copy_object must handle \
metadata_only=false for same-key copies of tiered objects, got: {err}"
);
}
}
}
+1 -4
View File
@@ -69,10 +69,7 @@ impl SetDisks {
"File info read complete"
);
if DiskError::is_all_not_found(&errs) {
warn!(
"heal_object failed, all obj part not found, bucket: {}, obj: {}, version_id: {}",
bucket, object, version_id
);
debug!(bucket, object, version_id, "heal_object skipped missing object");
let err = if !version_id.is_empty() {
DiskError::FileVersionNotFound
} else {
+1 -4
View File
@@ -67,10 +67,7 @@ impl SetDisks {
achieved,
}
}
other => StorageError::other(format!(
"Failed to acquire {mode} lock: {}",
self.format_lock_error_from_error(bucket, object, mode, &other)
)),
other => StorageError::Lock(other),
}
}
+3 -3
View File
@@ -37,10 +37,10 @@ impl SetDisks {
})
}
pub(super) fn reduce_common_data_dir(data_dirs: &Vec<Option<Uuid>>, write_quorum: usize) -> Option<Uuid> {
pub(super) fn reduce_common_data_dir(data_dirs: &[Option<Uuid>], write_quorum: usize) -> Option<Uuid> {
let mut data_dirs_count = HashMap::new();
for ddir in data_dirs {
for ddir in data_dirs.iter().flatten().copied() {
*data_dirs_count.entry(ddir).or_insert(0) += 1;
}
@@ -49,7 +49,7 @@ impl SetDisks {
for (ddir, count) in data_dirs_count {
if count > max {
max = count;
data_dir = *ddir;
data_dir = Some(ddir);
}
}
+62 -2
View File
@@ -38,7 +38,7 @@ impl SetDisks {
dst_bucket: &str,
dst_object: &str,
write_quorum: usize,
) -> disk::error::Result<(Vec<Option<DiskStore>>, Option<Vec<u8>>, Option<Uuid>)> {
) -> disk::error::Result<(Vec<Option<DiskStore>>, Option<Vec<u8>>, Option<Uuid>, Vec<Option<DiskStore>>)> {
let mut futures = Vec::with_capacity(disks.len());
let mut errs = Vec::with_capacity(disks.len());
@@ -92,6 +92,34 @@ impl SetDisks {
}
}
if issue3031_diag_enabled() {
let success_count = errs.iter().filter(|err| err.is_none()).count();
let failure_count = errs.len().saturating_sub(success_count);
let ignored_failure_count = errs
.iter()
.filter(|err| err.as_ref().is_some_and(|err| OBJECT_OP_IGNORED_ERRS.contains(err)))
.count();
let data_dir_vote_count = data_dirs.iter().filter(|data_dir| data_dir.is_some()).count();
let reduced_data_dir = Self::reduce_common_data_dir(&data_dirs, write_quorum);
warn!(
target: "rustfs_ecstore::set_disk",
src_bucket = %src_bucket,
src_object = %src_object,
dst_bucket = %dst_bucket,
dst_object = %dst_object,
write_quorum,
disk_count = errs.len(),
success_count,
failure_count,
ignored_failure_count,
data_dir_vote_count,
reduced_data_dir = ?reduced_data_dir,
errs = ?errs,
data_dirs = ?data_dirs,
"issue3031_rename_data_quorum_context"
);
}
let mut futures = Vec::with_capacity(disks.len());
if let Some(ret_err) = reduce_write_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, write_quorum) {
// TODO: add concurrency
@@ -128,6 +156,21 @@ impl SetDisks {
}
}
if issue3031_diag_enabled() {
warn!(
target: "rustfs_ecstore::set_disk",
src_bucket = %src_bucket,
src_object = %src_object,
dst_bucket = %dst_bucket,
dst_object = %dst_object,
write_quorum,
ret_err = %ret_err,
errs = ?errs,
data_dirs = ?data_dirs,
"issue3031_rename_data_quorum_failed"
);
}
let _ = join_all(futures).await;
return Err(ret_err);
}
@@ -136,6 +179,23 @@ impl SetDisks {
// TODO: reduceCommonVersions
let data_dir = Self::reduce_common_data_dir(&data_dirs, write_quorum);
let online_disks = Self::eval_disks(disks, &errs);
let cleanup_disks = if let Some(data_dir) = data_dir {
disks
.iter()
.zip(errs.iter())
.zip(data_dirs.iter())
.map(|((disk, err), old_data_dir)| {
if err.is_none() && *old_data_dir == Some(data_dir) {
disk.clone()
} else {
None
}
})
.collect()
} else {
vec![None; disks.len()]
};
// // TODO: reduce_common_data_dir
// if let Some(old_dir) = rename_ress
@@ -150,7 +210,7 @@ impl SetDisks {
// self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?;
Ok((Self::eval_disks(disks, &errs), versions, data_dir))
Ok((online_disks, versions, data_dir, cleanup_disks))
}
#[allow(dead_code)]
+122 -6
View File
@@ -14,10 +14,12 @@
#![allow(clippy::map_entry)]
use crate::bucket::bandwidth::monitor::Monitor;
use crate::bucket::lifecycle::bucket_lifecycle_audit::LcEventSrc;
use crate::bucket::lifecycle::bucket_lifecycle_ops::{
enqueue_immediate_expiry, enqueue_transition_immediate, init_background_expiry,
};
use crate::bucket::metadata_sys::get_global_bucket_metadata_sys;
use crate::bucket::metadata_sys::{self, set_bucket_metadata};
use crate::bucket::utils::check_abort_multipart_args;
use crate::bucket::utils::check_complete_multipart_args;
@@ -32,6 +34,7 @@ use crate::bucket::utils::check_put_object_args;
use crate::bucket::utils::check_put_object_part_args;
use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
use crate::config::storageclass;
use crate::config::{get_global_server_config, get_global_storage_class};
use crate::disk::endpoint::{Endpoint, EndpointType};
use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
use crate::error::{Error, Result};
@@ -39,11 +42,12 @@ use crate::error::{
StorageError, is_err_bucket_exists, is_err_bucket_not_found, is_err_invalid_upload_id, is_err_object_not_found,
is_err_read_quorum, is_err_version_not_found, to_object_err,
};
use crate::event_notification::EventNotifier;
use crate::global::{
DISK_ASSUME_UNKNOWN_SIZE, DISK_FILL_FRACTION, DISK_MIN_INODES, DISK_RESERVE_FRACTION, GLOBAL_BOOT_TIME,
GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_TierConfigMgr, get_global_bucket_monitor,
get_global_deployment_id, get_global_endpoints, init_global_bucket_monitor, is_dist_erasure, is_erasure_sd,
set_global_deployment_id, set_object_layer,
GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, TypeLocalDiskSetDrives, get_global_deployment_id, get_global_endpoints,
get_global_region, get_global_tier_config_mgr, init_global_bucket_monitor, is_erasure_sd, set_global_deployment_id,
set_object_layer,
};
use crate::notification_sys::get_global_notification_sys;
use crate::pools::PoolMeta;
@@ -53,6 +57,7 @@ use crate::store_api::{
ListMultipartsInfo, ListObjectVersionsInfo, ListPartsInfo, MultipartInfo, ObjectIO, ObjectInfoOrErr, WalkOptions,
};
use crate::store_init::{check_disk_fatal_errs, ec_drives_no_config};
use crate::tier::tier::TierConfigMgr;
use crate::{
bucket::{lifecycle::bucket_lifecycle_ops::TransitionState, metadata::BucketMetadata},
disk::{BUCKET_META_PREFIX, DiskOption, DiskStore, RUSTFS_META_BUCKET, new_disk},
@@ -71,7 +76,7 @@ use http::HeaderMap;
use lazy_static::lazy_static;
use rand::RngExt as _;
use rustfs_common::heal_channel::{HealItemType, HealOpts};
use rustfs_common::{GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_HOST, GLOBAL_RUSTFS_PORT};
use rustfs_common::{GLOBAL_LOCAL_NODE_NAME, GLOBAL_RUSTFS_ADDR, GLOBAL_RUSTFS_HOST, GLOBAL_RUSTFS_PORT};
use rustfs_filemeta::FileInfo;
use rustfs_lock::{LocalClient, LockClient, NamespaceLockWrapper};
use rustfs_madmin::heal_commands::HealResultItem;
@@ -82,7 +87,11 @@ use std::net::SocketAddr;
use std::process::exit;
use std::slice::Iter;
use std::time::SystemTime;
use std::{collections::HashMap, sync::Arc, time::Duration};
use std::{
collections::HashMap,
sync::{Arc, OnceLock},
time::Duration,
};
use time::OffsetDateTime;
use tokio::select;
use tokio::sync::RwLock;
@@ -165,7 +174,6 @@ pub use peer::{
has_space_for, init_local_disks, init_lock_clients, prewarm_local_disk_id_map,
};
#[derive(Debug)]
pub struct ECStore {
pub id: Uuid,
// pub disks: Vec<DiskStore>,
@@ -176,6 +184,114 @@ pub struct ECStore {
pub pool_meta: RwLock<PoolMeta>,
pub rebalance_meta: RwLock<Option<RebalanceMeta>>,
pub decommission_cancelers: RwLock<Vec<Option<CancellationToken>>>,
// Phase 2 migration pending - do not use directly.
/// Local disk maps (migrated from GLOBAL_LOCAL_DISK_MAP/ID_MAP/SET_DRIVES)
pub(crate) local_disk_map: Arc<RwLock<HashMap<String, Option<DiskStore>>>>,
pub(crate) local_disk_id_map: Arc<RwLock<HashMap<Uuid, String>>>,
pub(crate) local_disk_set_drives: Arc<RwLock<TypeLocalDiskSetDrives>>,
/// Tier config manager (migrated from GLOBAL_TierConfigMgr)
pub(crate) tier_config_mgr: Arc<RwLock<TierConfigMgr>>,
/// Event notifier (migrated from GLOBAL_EventNotifier)
pub(crate) event_notifier: Arc<RwLock<EventNotifier>>,
/// Bucket monitor (migrated from GLOBAL_BUCKET_MONITOR)
pub(crate) bucket_monitor: OnceLock<Arc<Monitor>>,
}
impl std::fmt::Debug for ECStore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ECStore")
.field("id", &self.id)
.field("disk_map", &self.disk_map)
.field("pools", &self.pools)
.field("pool_meta", &self.pool_meta)
.finish_non_exhaustive()
}
}
/// Phase 2: Accessor methods for config globals
/// These delegate to the process-global statics. No local state — the globals
/// remain the single source of truth until the migration is complete.
impl ECStore {
/// Get server configuration (delegates to global)
pub fn get_server_config(&self) -> Option<crate::config::Config> {
crate::config::get_global_server_config()
}
/// Set server configuration (delegates to global)
pub fn set_server_config(&self, cfg: crate::config::Config) {
crate::config::set_global_server_config(cfg);
}
/// Get storage class configuration (delegates to global)
pub fn get_storage_class(&self) -> Option<crate::config::storageclass::Config> {
crate::config::get_global_storage_class()
}
/// Set storage class configuration (delegates to global)
pub fn set_storage_class(&self, cfg: crate::config::storageclass::Config) {
crate::config::set_global_storage_class(cfg);
}
}
/// Phase 3: Accessor methods for service globals
/// These provide a unified API through ECStore for accessing cross-cutting
/// service singletons. The globals remain the source of truth.
impl ECStore {
/// Get the notification system
pub fn notification_system(&self) -> Option<&'static crate::notification_sys::NotificationSys> {
get_global_notification_sys()
}
/// Get the bucket metadata system
pub fn bucket_metadata_sys(&self) -> Option<Arc<tokio::sync::RwLock<crate::bucket::metadata_sys::BucketMetadataSys>>> {
get_global_bucket_metadata_sys()
}
/// Get the global endpoints
pub fn endpoints(&self) -> EndpointServerPools {
get_global_endpoints()
}
/// Get the global region
pub fn region(&self) -> Option<s3s::region::Region> {
get_global_region()
}
/// Get the tier config manager
pub fn tier_config_mgr(&self) -> Arc<tokio::sync::RwLock<crate::tier::tier::TierConfigMgr>> {
get_global_tier_config_mgr()
}
/// Get the server configuration
pub fn server_config(&self) -> Option<crate::config::Config> {
get_global_server_config()
}
/// Get the storage class configuration
pub fn storage_class(&self) -> Option<crate::config::storageclass::Config> {
get_global_storage_class()
}
}
/// Phase 4: Server address accessors
/// These provide a unified API through ECStore for accessing server-level
/// configuration globals. The globals remain the source of truth.
impl ECStore {
/// Get the server port
pub fn port(&self) -> u16 {
crate::global::global_rustfs_port()
}
/// Get the server host
pub async fn host(&self) -> String {
GLOBAL_RUSTFS_HOST.read().await.clone()
}
/// Get the server address (host:port)
pub async fn addr(&self) -> String {
GLOBAL_RUSTFS_ADDR.read().await.clone()
}
}
// impl Clone for ECStore {
+31 -7
View File
@@ -13,7 +13,11 @@
// limitations under the License.
use super::*;
use crate::bucket::{metadata::BUCKET_TABLE_RESERVED_PREFIX, utils::is_meta_bucketname};
use crate::bucket::{
metadata::{BUCKET_TABLE_RESERVED_PREFIX, table_bucket_catalog_metadata_prefix},
utils::is_meta_bucketname,
};
use crate::global::get_global_bucket_monitor;
use crate::set_disk::get_lock_acquire_timeout;
fn should_override_created_from_metadata(created: OffsetDateTime) -> bool {
@@ -55,6 +59,13 @@ async fn validate_table_bucket_delete_guard(bucket: &str) -> Result<()> {
Ok(())
}
fn bucket_delete_metadata_cleanup_prefixes(bucket: &str) -> [String; 2] {
[
table_bucket_catalog_metadata_prefix(bucket),
format!("{BUCKET_META_PREFIX}/{bucket}"),
]
}
impl ECStore {
#[instrument(skip(self))]
pub(super) async fn handle_make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
@@ -80,7 +91,7 @@ impl ECStore {
achieved,
}
}
other => StorageError::other(format!("make_bucket: failed to acquire write lock on {bucket}: {other}")),
other => StorageError::Lock(other),
})?,
)
} else {
@@ -184,7 +195,7 @@ impl ECStore {
achieved,
}
}
other => StorageError::other(format!("delete_bucket: failed to acquire write lock on {bucket}: {other}")),
other => StorageError::Lock(other),
})?,
)
} else {
@@ -228,9 +239,11 @@ impl ECStore {
// TODO: replication opts.srdelete_op
// Delete the metadata
self.delete_all(RUSTFS_META_BUCKET, format!("{BUCKET_META_PREFIX}/{bucket}").as_str())
.await?;
// Delete internal metadata after the bucket is gone so stale catalog records cannot be reused
// if the same bucket name is created again.
for prefix in bucket_delete_metadata_cleanup_prefixes(bucket) {
self.delete_all(RUSTFS_META_BUCKET, prefix.as_str()).await?;
}
if let Some(monitor) = get_global_bucket_monitor() {
monitor.delete_bucket(bucket);
}
@@ -240,7 +253,10 @@ impl ECStore {
#[cfg(test)]
mod tests {
use super::{should_override_created_from_metadata, validate_table_bucket_delete_allowed};
use super::{
bucket_delete_metadata_cleanup_prefixes, should_override_created_from_metadata, validate_table_bucket_delete_allowed,
};
use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
use crate::error::StorageError;
use time::OffsetDateTime;
@@ -263,4 +279,12 @@ mod tests {
assert!(validate_table_bucket_delete_allowed("table-bucket", true, false).is_ok());
assert!(validate_table_bucket_delete_allowed("regular-bucket", false, true).is_ok());
}
#[test]
fn bucket_delete_metadata_cleanup_removes_internal_table_catalog_prefix() {
let prefixes = bucket_delete_metadata_cleanup_prefixes("analytics");
assert!(prefixes.contains(&table_bucket_catalog_metadata_prefix("analytics")));
assert!(prefixes.contains(&"buckets/analytics".to_string()));
}
}
+15 -1
View File
@@ -14,7 +14,10 @@
use super::*;
use crate::error::is_err_decommission_running;
use crate::global::is_first_cluster_node_local;
use crate::global::{
GLOBAL_EventNotifier, GLOBAL_LOCAL_DISK_ID_MAP, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES, GLOBAL_TierConfigMgr,
get_global_bucket_monitor, is_dist_erasure, is_first_cluster_node_local,
};
fn pool_first_endpoint_is_local(pool: &crate::endpoints::PoolEndpoints) -> bool {
pool.endpoints.as_ref().first().is_some_and(|endpoint| endpoint.is_local)
@@ -244,6 +247,13 @@ impl ECStore {
pool_meta: RwLock::new(pool_meta),
rebalance_meta: RwLock::new(None),
decommission_cancelers,
local_disk_map: GLOBAL_LOCAL_DISK_MAP.clone(),
local_disk_id_map: GLOBAL_LOCAL_DISK_ID_MAP.clone(),
local_disk_set_drives: GLOBAL_LOCAL_DISK_SET_DRIVES.clone(),
tier_config_mgr: GLOBAL_TierConfigMgr.clone(),
event_notifier: GLOBAL_EventNotifier.clone(),
bucket_monitor: OnceLock::new(),
});
// Only set it when the global deployment ID is not yet configured
@@ -275,6 +285,10 @@ impl ECStore {
set_object_layer(ec.clone()).await;
if let Some(monitor) = get_global_bucket_monitor() {
let _ = ec.bucket_monitor.set(monitor);
}
Ok(ec)
}
+19 -3
View File
@@ -115,10 +115,24 @@ impl ECStore {
object: &str,
opts: &ObjectOptions,
) -> Result<MultipartUploadResult> {
self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts)
.await
.map(|(res, _)| res)
}
pub(crate) async fn handle_new_multipart_upload_with_pool_idx(
&self,
bucket: &str,
object: &str,
opts: &ObjectOptions,
) -> Result<(MultipartUploadResult, usize)> {
check_new_multipart_args(bucket, object)?;
if self.single_pool() {
return self.pools[0].new_multipart_upload(bucket, object, opts).await;
return self.pools[0]
.new_multipart_upload(bucket, object, opts)
.await
.map(|res| (res, 0));
}
for (idx, pool) in self.pools.iter().enumerate() {
@@ -130,7 +144,8 @@ impl ECStore {
.await?;
if !res.uploads.is_empty() {
return self.pools[idx].new_multipart_upload(bucket, object, opts).await;
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
return Ok((res, idx));
}
}
let idx = self.get_pool_idx(bucket, object, -1).await?;
@@ -142,7 +157,8 @@ impl ECStore {
));
}
self.pools[idx].new_multipart_upload(bucket, object, opts).await
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
Ok((res, idx))
}
#[instrument(skip(self))]
+450 -11
View File
@@ -233,6 +233,89 @@ fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> Object
lookup_opts
}
fn effective_object_actual_size(info: &ObjectInfo) -> Option<i64> {
info.get_actual_size().ok()
}
fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &ObjectInfo) -> bool {
is_data_movement_delete_marker(source)
&& is_data_movement_delete_marker(target)
&& source.version_id == target.version_id
&& source.mod_time == target.mod_time
}
fn is_data_movement_delete_marker(info: &ObjectInfo) -> bool {
info.delete_marker
}
fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, target: &ObjectInfo) -> bool {
source.version_id == target.version_id
&& !target.delete_marker
&& source.size == target.size
&& source.get_etag() == target.etag
&& source.checksum == target.checksum
&& source.mod_time == target.mod_time
&& source.transition_status == target.transitioned_object.status
&& source.transitioned_objname == target.transitioned_object.name
&& source.transition_tier == target.transitioned_object.tier
&& source
.transition_version_id
.map(|version_id| version_id.to_string())
.unwrap_or_default()
== target.transitioned_object.version_id
&& effective_object_actual_size(target) == Some(source.size)
}
fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
target_pool_idx != src_pool_idx
}
fn resolve_data_movement_resume_target_pool(
selected_target_pool_idx: usize,
resume_target_pool_idx: Option<usize>,
src_pool_idx: usize,
) -> usize {
if should_check_data_movement_resume_target(src_pool_idx, selected_target_pool_idx) {
selected_target_pool_idx
} else {
resume_target_pool_idx.unwrap_or(selected_target_pool_idx)
}
}
fn resolve_data_movement_delete_marker_resume_result(
target_result: Result<Option<ObjectInfo>>,
source: &ObjectInfo,
src_pool_idx: usize,
target_pool_idx: usize,
) -> Result<bool> {
if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
return Ok(false);
}
let Some(target) = target_result? else {
return Ok(false);
};
Ok(is_equivalent_data_movement_delete_marker(source, &target))
}
fn resolve_data_movement_tiered_resume_result(
target_result: Result<Option<ObjectInfo>>,
source: &rustfs_filemeta::FileInfo,
src_pool_idx: usize,
target_pool_idx: usize,
) -> Result<bool> {
if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
return Ok(false);
}
let Some(target) = target_result? else {
return Ok(false);
};
Ok(is_equivalent_data_movement_tiered_object(source, &target))
}
impl ECStore {
fn map_namespace_lock_error(bucket: &str, object: &str, mode: &'static str, err: rustfs_lock::LockError) -> StorageError {
match err {
@@ -243,7 +326,7 @@ impl ECStore {
required,
achieved,
},
other => StorageError::other(format!("Failed to acquire {mode} lock on {bucket}/{object}: {other}")),
other => StorageError::Lock(other),
}
}
@@ -377,6 +460,62 @@ impl ECStore {
}
}
async fn find_data_movement_target_info(
&self,
bucket: &str,
object: &str,
target_pool_idx: usize,
opts: &ObjectOptions,
) -> Result<Option<ObjectInfo>> {
let lookup_opts = version_aware_lookup_opts(opts, true);
let Some(pool) = self.pools.get(target_pool_idx) else {
return Err(Error::other(format!(
"data movement resume target pool {target_pool_idx} is out of range for {bucket}/{object}"
)));
};
match pool.get_object_info(bucket, object, &lookup_opts).await {
Ok(info) => Ok(Some(info)),
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
Err(err) => Err(err),
}
}
async fn has_equivalent_data_movement_delete_marker(
&self,
bucket: &str,
object: &str,
source: &ObjectInfo,
opts: &ObjectOptions,
target_pool_idx: usize,
) -> Result<bool> {
resolve_data_movement_delete_marker_resume_result(
self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
.await,
source,
opts.src_pool_idx,
target_pool_idx,
)
}
async fn has_equivalent_data_movement_tiered_object(
&self,
bucket: &str,
object: &str,
source: &rustfs_filemeta::FileInfo,
opts: &ObjectOptions,
target_pool_idx: usize,
) -> Result<bool> {
resolve_data_movement_tiered_resume_result(
self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
.await,
source,
opts.src_pool_idx,
target_pool_idx,
)
}
fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> {
result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}")))
}
@@ -419,6 +558,17 @@ impl ECStore {
)?
};
if opts.data_movement && idx == opts.src_pool_idx {
let resume_target_pool_idx = self
.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
.await;
let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
if self
.has_equivalent_data_movement_tiered_object(bucket, &object, fi, opts, target_pool_idx)
.await?
{
return Ok(());
}
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
@@ -570,9 +720,32 @@ impl ECStore {
}
if !dst_opts.versioned && src_opts.version_id.is_none() {
return self.pools[pool_idx]
.copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &dst_opts)
.await;
if src_info.metadata_only {
return self.pools[pool_idx]
.copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &dst_opts)
.await;
}
// Transitioned object self-copy: restore from tier into the same pool.
let put_opts = ObjectOptions {
user_defined: (*src_info.user_defined).clone(),
versioned: dst_opts.versioned,
version_id: dst_opts.version_id.clone(),
no_lock: dst_opts.no_lock,
mod_time: dst_opts.mod_time,
http_preconditions: dst_opts.http_preconditions.clone(),
..Default::default()
};
return if let Some(reader) = src_info.put_object_reader.as_mut() {
self.pools[pool_idx]
.put_object(dst_bucket, &dst_object, reader, &put_opts)
.await
} else {
Err(StorageError::InvalidArgument(
src_bucket.to_owned(),
src_object.to_owned(),
"put_object_reader is none".to_owned(),
))
};
}
if dst_opts.versioned && src_opts.version_id != dst_opts.version_id {
@@ -643,17 +816,44 @@ impl ECStore {
let gopts = version_aware_lookup_opts(&opts, true);
if opts.data_movement {
let existing_pool_idx = self
.get_pool_info_existing_with_opts(bucket, object, &gopts)
.await
.map(|(pinfo, _)| pinfo.index);
let target_pool_idx =
let existing_pool_info = self.get_pool_info_existing_with_opts(bucket, object, &gopts).await;
let existing_pool_idx = existing_pool_info
.as_ref()
.map(|(pinfo, _)| pinfo.index)
.map_err(Clone::clone);
let selected_target_pool_idx =
match select_data_movement_target_pool(existing_pool_idx, opts.src_pool_idx, opts.delete_marker)? {
Some(pool_idx) => pool_idx,
None => self.get_pool_idx_no_lock(bucket, object, 0).await?,
};
let resume_target_pool_idx = if selected_target_pool_idx == opts.src_pool_idx {
self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx)
.await
} else {
None
};
let target_pool_idx =
resolve_data_movement_resume_target_pool(selected_target_pool_idx, resume_target_pool_idx, opts.src_pool_idx);
if opts.src_pool_idx == selected_target_pool_idx {
if let Ok((source_pool_info, _)) = existing_pool_info
&& opts.delete_marker
&& is_data_movement_delete_marker(&source_pool_info.object_info)
&& self
.has_equivalent_data_movement_delete_marker(
bucket,
object,
&source_pool_info.object_info,
&opts,
target_pool_idx,
)
.await?
{
let mut obj = source_pool_info.object_info;
obj.name = decode_dir_object(object);
return Ok(obj);
}
if opts.src_pool_idx == target_pool_idx {
return Err(StorageError::DataMovementOverwriteErr(
bucket.to_owned(),
object.to_owned(),
@@ -661,7 +861,9 @@ impl ECStore {
));
}
let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?;
let mut obj = self.pools[selected_target_pool_idx]
.delete_object(bucket, object, opts)
.await?;
obj.name = decode_dir_object(obj.name.as_str());
return Ok(obj);
}
@@ -1063,6 +1265,9 @@ impl ECStore {
#[cfg(test)]
mod tests {
use super::*;
use crate::bucket::lifecycle::bucket_lifecycle_ops::TransitionedObject;
use crate::bucket::lifecycle::core::TRANSITION_COMPLETE;
use bytes::Bytes;
use std::io::Cursor;
use tokio::io::AsyncReadExt;
@@ -1085,6 +1290,240 @@ mod tests {
assert_eq!(target, Some(0));
}
#[test]
fn equivalent_data_movement_delete_marker_requires_same_version_and_mod_time() {
let version_id = Uuid::nil();
let mod_time = OffsetDateTime::UNIX_EPOCH;
let source = ObjectInfo {
version_id: Some(version_id),
delete_marker: true,
mod_time: Some(mod_time),
..Default::default()
};
let target = source.clone();
assert!(is_equivalent_data_movement_delete_marker(&source, &target));
let mismatched = ObjectInfo {
mod_time: Some(mod_time + Duration::from_secs(1)),
..target
};
assert!(!is_equivalent_data_movement_delete_marker(&source, &mismatched));
}
#[test]
fn equivalent_data_movement_delete_marker_rejects_live_object() {
let source = ObjectInfo {
delete_marker: true,
..Default::default()
};
let target = ObjectInfo {
delete_marker: false,
..source.clone()
};
assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
}
#[test]
fn data_movement_delete_marker_resume_accepts_equivalent_target() {
let source = ObjectInfo {
version_id: Some(Uuid::nil()),
delete_marker: true,
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 1)
.expect("equivalent delete marker target should be evaluated");
assert!(should_resume);
}
#[test]
fn data_movement_delete_marker_resume_rejects_source_pool_target() {
let source = ObjectInfo {
version_id: Some(Uuid::nil()),
delete_marker: true,
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
..Default::default()
};
let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 0)
.expect("source-pool target should be rejected before target lookup");
assert!(!should_resume);
}
#[test]
fn data_movement_resume_target_prefers_selected_non_source_pool() {
let target_pool_idx = resolve_data_movement_resume_target_pool(2, Some(3), 1);
assert_eq!(target_pool_idx, 2);
}
#[test]
fn data_movement_resume_target_uses_resolved_non_source_pool_when_selected_is_source() {
let target_pool_idx = resolve_data_movement_resume_target_pool(1, Some(3), 1);
assert_eq!(target_pool_idx, 3);
}
#[test]
fn data_movement_resume_target_keeps_source_when_no_other_pool_is_available() {
let target_pool_idx = resolve_data_movement_resume_target_pool(1, None, 1);
assert_eq!(target_pool_idx, 1);
}
#[test]
fn data_movement_delete_marker_resume_propagates_target_lookup_error() {
let source = ObjectInfo {
delete_marker: true,
..Default::default()
};
let result = resolve_data_movement_delete_marker_resume_result(Err(Error::SlowDown), &source, 0, 1);
assert!(matches!(result, Err(Error::SlowDown)));
}
#[test]
fn equivalent_data_movement_tiered_object_accepts_matching_transition_metadata() {
let version_id = Uuid::nil();
let transition_version_id = Uuid::new_v4();
let mod_time = OffsetDateTime::UNIX_EPOCH;
let source = FileInfo {
version_id: Some(version_id),
size: 1024,
mod_time: Some(mod_time),
checksum: Some(Bytes::from_static(b"checksum")),
transition_status: TRANSITION_COMPLETE.to_string(),
transitioned_objname: "remote/object".to_string(),
transition_tier: "WARM".to_string(),
transition_version_id: Some(transition_version_id),
metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]),
..Default::default()
};
let target = ObjectInfo {
version_id: Some(version_id),
size: 1024,
mod_time: Some(mod_time),
checksum: Some(Bytes::from_static(b"checksum")),
etag: Some("etag-value".to_string()),
transitioned_object: TransitionedObject {
name: "remote/object".to_string(),
version_id: transition_version_id.to_string(),
tier: "WARM".to_string(),
status: TRANSITION_COMPLETE.to_string(),
..Default::default()
},
..Default::default()
};
assert!(is_equivalent_data_movement_tiered_object(&source, &target));
}
#[test]
fn equivalent_data_movement_tiered_object_rejects_transition_mismatch() {
let source = FileInfo {
version_id: Some(Uuid::nil()),
size: 1024,
transition_status: TRANSITION_COMPLETE.to_string(),
transitioned_objname: "remote/source".to_string(),
transition_tier: "WARM".to_string(),
..Default::default()
};
let target = ObjectInfo {
version_id: source.version_id,
size: 1024,
transitioned_object: TransitionedObject {
name: "remote/target".to_string(),
tier: "WARM".to_string(),
status: TRANSITION_COMPLETE.to_string(),
..Default::default()
},
..Default::default()
};
assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
}
#[test]
fn data_movement_tiered_resume_accepts_equivalent_target() {
let version_id = Uuid::nil();
let transition_version_id = Uuid::new_v4();
let source = FileInfo {
version_id: Some(version_id),
size: 1024,
transition_status: TRANSITION_COMPLETE.to_string(),
transitioned_objname: "remote/object".to_string(),
transition_tier: "WARM".to_string(),
transition_version_id: Some(transition_version_id),
metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]),
..Default::default()
};
let target = ObjectInfo {
version_id: Some(version_id),
size: 1024,
etag: Some("etag-value".to_string()),
transitioned_object: TransitionedObject {
name: "remote/object".to_string(),
version_id: transition_version_id.to_string(),
tier: "WARM".to_string(),
status: TRANSITION_COMPLETE.to_string(),
..Default::default()
},
..Default::default()
};
let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 1)
.expect("equivalent tiered target should be evaluated");
assert!(should_resume);
}
#[test]
fn data_movement_tiered_resume_rejects_source_pool_target() {
let version_id = Uuid::nil();
let source = FileInfo {
version_id: Some(version_id),
size: 1024,
transition_status: TRANSITION_COMPLETE.to_string(),
transitioned_objname: "remote/object".to_string(),
transition_tier: "WARM".to_string(),
metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]),
..Default::default()
};
let target = ObjectInfo {
version_id: Some(version_id),
size: 1024,
etag: Some("etag-value".to_string()),
transitioned_object: TransitionedObject {
name: "remote/object".to_string(),
tier: "WARM".to_string(),
status: TRANSITION_COMPLETE.to_string(),
..Default::default()
},
..Default::default()
};
let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 0)
.expect("source-pool target should be rejected before target lookup");
assert!(!should_resume);
}
#[test]
fn data_movement_tiered_resume_rejects_missing_target() {
let source = FileInfo {
version_id: Some(Uuid::nil()),
size: 1024,
..Default::default()
};
let should_resume = resolve_data_movement_tiered_resume_result(Ok(None), &source, 0, 1)
.expect("missing tiered target should be evaluated");
assert!(!should_resume);
}
#[test]
fn latest_object_access_delete_marker_error_returns_none_for_live_object() {
let info = ObjectInfo::default();
+235 -20
View File
@@ -21,6 +21,11 @@ struct LatestObjectInfoCandidate {
err: Option<Error>,
}
struct RebalanceDeletePoolResult {
pool_idx: usize,
result: Result<ObjectInfo>,
}
fn pool_lookup_not_found_error(bucket: &str, object: &str, opts: &ObjectOptions) -> Error {
let object = decode_dir_object(object);
@@ -39,6 +44,52 @@ fn resolve_rebalance_delete_from_all_pools_result(result: Result<ObjectInfo>, bu
result.map_err(|err| Error::other(format!("failed to delete rebalance source object {bucket}/{object}: {err}")))
}
fn is_ignorable_rebalance_delete_error(err: &Error) -> bool {
is_err_object_not_found(err) || is_err_version_not_found(err)
}
fn rebalance_delete_pool_error(pool_idx: usize, bucket: &str, object: &str, err: Error) -> Error {
Error::other(format!("pool {pool_idx} delete failed for {bucket}/{object}: {err}"))
}
fn resolve_rebalance_delete_from_all_pools_results(
results: Vec<RebalanceDeletePoolResult>,
bucket: &str,
object: &str,
) -> Result<ObjectInfo> {
let mut deleted = None;
let mut ignored_error = None;
for pool_result in results {
let pool_idx = pool_result.pool_idx;
match pool_result.result {
Ok(info) => {
if deleted.is_none() {
deleted = Some(info);
}
}
Err(err) if is_ignorable_rebalance_delete_error(&err) => {
ignored_error = Some((pool_idx, err));
}
Err(err) => {
return Err(rebalance_delete_pool_error(pool_idx, bucket, object, err));
}
}
}
if let Some(info) = deleted {
return Ok(info);
}
if let Some((pool_idx, err)) = ignored_error {
return Err(rebalance_delete_pool_error(pool_idx, bucket, object, err));
}
Err(Error::other(format!(
"failed to delete rebalance source object {bucket}/{object}: no pools were attempted"
)))
}
fn rebalance_disk_set_lookup_error(pool_idx: usize, set_idx: usize, pool_count: usize) -> Error {
Error::other(format!(
"failed to resolve rebalance disk set: pool index {pool_idx}, set index {set_idx}, pool count {pool_count}",
@@ -220,6 +271,41 @@ impl ECStore {
None
}
pub(super) async fn get_available_pool_idx_excluding(
&self,
bucket: &str,
object: &str,
size: i64,
excluded_pool_idx: usize,
) -> Option<usize> {
let mut server_pools = self.get_server_pools_available_space(bucket, object, size).await;
server_pools.filter_max_used(100 - (100_f64 * DISK_RESERVE_FRACTION) as u64);
if let Some(pool) = server_pools.0.get_mut(excluded_pool_idx) {
pool.available = 0;
}
let total = server_pools.total_available();
if total == 0 {
return None;
}
let mut rng = rand::rng();
let random_u64: u64 = rng.random_range(0..total);
let choose = random_u64 % total;
let mut at_total = 0;
for pool in server_pools.iter() {
at_total += pool.available;
if at_total > choose && pool.available > 0 {
return Some(pool.index);
}
}
None
}
async fn get_server_pools_available_space(&self, bucket: &str, object: &str, size: i64) -> ServerPoolsAvailableSpace {
let mut n_sets = vec![0; self.pools.len()];
let mut infos = vec![Vec::new(); self.pools.len()];
@@ -507,38 +593,34 @@ impl ECStore {
opts: &ObjectOptions,
errs: Vec<PoolErr>,
) -> Result<ObjectInfo> {
let mut objs = Vec::new();
let mut derrs = Vec::new();
let mut results = Vec::with_capacity(errs.len());
for pe in errs.iter() {
if let Some(err) = &pe.err
&& err == &StorageError::ErasureWriteQuorum
{
objs.push(None);
derrs.push(Some(StorageError::ErasureWriteQuorum));
if let Some(idx) = pe.index {
results.push(RebalanceDeletePoolResult {
pool_idx: idx,
result: Err(StorageError::ErasureWriteQuorum),
});
}
continue;
}
if let Some(idx) = pe.index {
match self.pools[idx].delete_object(bucket, object, opts.clone()).await {
Ok(res) => {
objs.push(Some(res));
derrs.push(None);
}
Err(err) => {
objs.push(None);
derrs.push(Some(err));
}
}
results.push(RebalanceDeletePoolResult {
pool_idx: idx,
result: self.pools[idx].delete_object(bucket, object, opts.clone()).await,
});
}
}
if let Some(e) = &derrs[0] {
return resolve_rebalance_delete_from_all_pools_result(Err(e.clone()), bucket, object);
}
resolve_rebalance_delete_from_all_pools_result(Ok(objs[0].as_ref().unwrap().clone()), bucket, object)
resolve_rebalance_delete_from_all_pools_result(
resolve_rebalance_delete_from_all_pools_results(results, bucket, object),
bucket,
object,
)
}
pub async fn reload_pool_meta(&self) -> Result<()> {
@@ -902,6 +984,139 @@ mod tests {
assert!(rendered.contains(&Error::SlowDown.to_string()), "{rendered}");
}
#[test]
fn resolve_rebalance_delete_from_all_pools_results_fails_on_later_pool_error() {
let err = resolve_rebalance_delete_from_all_pools_results(
vec![
RebalanceDeletePoolResult {
pool_idx: 0,
result: Ok(ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
..Default::default()
}),
},
RebalanceDeletePoolResult {
pool_idx: 1,
result: Err(Error::SlowDown),
},
],
"bucket",
"object",
)
.expect_err("non-ignorable errors from later pools must not be hidden");
let rendered = err.to_string();
assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
assert!(rendered.contains(&Error::SlowDown.to_string()), "{rendered}");
}
#[test]
fn resolve_rebalance_delete_from_all_pools_results_ignores_later_not_found_after_success() {
let info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
..Default::default()
};
let resolved = resolve_rebalance_delete_from_all_pools_results(
vec![
RebalanceDeletePoolResult {
pool_idx: 0,
result: Ok(info.clone()),
},
RebalanceDeletePoolResult {
pool_idx: 1,
result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
},
],
"bucket",
"object",
)
.expect("not-found errors from other pools should be ignored when a delete succeeds");
assert_eq!(resolved.bucket, info.bucket);
assert_eq!(resolved.name, info.name);
}
#[test]
fn resolve_rebalance_delete_from_all_pools_results_accepts_success_after_not_found() {
let info = ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
..Default::default()
};
let resolved = resolve_rebalance_delete_from_all_pools_results(
vec![
RebalanceDeletePoolResult {
pool_idx: 0,
result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
},
RebalanceDeletePoolResult {
pool_idx: 1,
result: Ok(info.clone()),
},
],
"bucket",
"object",
)
.expect("a successful delete should pass even when an earlier pool reports not-found");
assert_eq!(resolved.bucket, info.bucket);
assert_eq!(resolved.name, info.name);
}
#[test]
fn resolve_rebalance_delete_from_all_pools_results_fails_when_all_results_are_ignored_errors() {
let err = resolve_rebalance_delete_from_all_pools_results(
vec![
RebalanceDeletePoolResult {
pool_idx: 0,
result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
},
RebalanceDeletePoolResult {
pool_idx: 1,
result: Err(Error::VersionNotFound("bucket".to_string(), "object".to_string(), "vid-1".to_string())),
},
],
"bucket",
"object",
)
.expect_err("all ignored errors without any successful delete should still fail");
let rendered = err.to_string();
assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
assert!(rendered.contains("Version not found"), "{rendered}");
}
#[test]
fn resolve_rebalance_delete_from_all_pools_results_fails_on_write_quorum_even_with_success() {
let err = resolve_rebalance_delete_from_all_pools_results(
vec![
RebalanceDeletePoolResult {
pool_idx: 0,
result: Ok(ObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
..Default::default()
}),
},
RebalanceDeletePoolResult {
pool_idx: 1,
result: Err(Error::ErasureWriteQuorum),
},
],
"bucket",
"object",
)
.expect_err("write quorum failures must fail the aggregate delete");
let rendered = err.to_string();
assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
assert!(rendered.contains(&Error::ErasureWriteQuorum.to_string()), "{rendered}");
}
#[test]
fn rebalance_disk_set_lookup_error_formats_pool_and_set_context() {
let err = rebalance_disk_set_lookup_error(2, 7, 3);
+1 -3
View File
@@ -17,6 +17,7 @@ use crate::bucket::versioning::VersioningApi as _;
use crate::config::storageclass;
use crate::disk::DiskStore;
use crate::error::{Error, Result};
use crate::rio::{HashReader, LimitReader};
use crate::store_utils::clean_metadata;
use crate::{
bucket::lifecycle::bucket_lifecycle_audit::LcAuditEvent,
@@ -34,17 +35,14 @@ use rustfs_filemeta::{
use rustfs_lock::NamespaceLockWrapper;
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_rio::Checksum;
use rustfs_rio::{DecompressReader, HashReader, LimitReader};
use rustfs_utils::CompressionAlgorithm;
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::{AMZ_BUCKET_REPLICATION_STATUS, AMZ_RESTORE, AMZ_STORAGE_CLASS};
use rustfs_utils::path::decode_dir_object;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fmt::Debug;
use std::io::Cursor;
use std::pin::Pin;
use std::str::FromStr as _;
use std::sync::Arc;
use std::task::{Context, Poll};
use time::OffsetDateTime;
File diff suppressed because it is too large Load Diff
+27 -48
View File
@@ -1,29 +1,7 @@
use super::*;
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct MakeBucketOptions {
pub lock_enabled: bool,
pub versioning_enabled: bool,
pub force_create: bool, // Create buckets even if they are already created.
pub created_at: Option<OffsetDateTime>, // only for site replication
pub no_lock: bool,
}
#[derive(Debug, Default, Clone, PartialEq)]
pub enum SRBucketDeleteOp {
#[default]
NoOp,
MarkDelete,
Purge,
}
#[derive(Debug, Default, Clone)]
pub struct DeleteBucketOptions {
pub no_lock: bool,
pub no_recreate: bool,
pub force: bool, // Force deletion
pub srdelete_op: SRBucketDeleteOp,
}
// RUSTFS_COMPAT_TODO(API-003): keep old ecstore::store_api bucket DTO import paths while storage API consumers migrate. Remove after all consumers import these DTOs from rustfs_storage_api.
pub use rustfs_storage_api::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};
#[derive(Debug, Default, Clone)]
pub struct HTTPPreconditions {
@@ -225,22 +203,6 @@ fn is_modified_since(mod_time: &OffsetDateTime, given_time: &OffsetDateTime) ->
mod_secs > given_secs
}
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct BucketOptions {
pub deleted: bool, // true only when site replication is enabled
pub cached: bool, // true only when we are requesting a cached response instead of hitting the disk for example ListBuckets() call.
pub no_metadata: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct BucketInfo {
pub name: String,
pub created: Option<OffsetDateTime>,
pub deleted: Option<OffsetDateTime>,
pub versioning: bool,
pub object_locking: bool,
}
#[derive(Debug, Default, Clone)]
pub struct MultipartUploadResult {
pub upload_id: String,
@@ -369,13 +331,18 @@ impl ObjectInfo {
}
pub fn is_compressed_ok(&self) -> Result<(CompressionAlgorithm, bool)> {
let (algorithm, _, compressed) = self.compression_read_plan()?;
Ok((algorithm, compressed))
}
pub fn compression_read_plan(&self) -> Result<(CompressionAlgorithm, crate::rio::ReadCompressionBackend, bool)> {
let scheme = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
if let Some(scheme) = scheme {
let algorithm = CompressionAlgorithm::from_str(&scheme)?;
Ok((algorithm, true))
let (algorithm, backend) = crate::rio::compression_scheme_to_read_plan(&scheme)?;
Ok((algorithm, backend, true))
} else {
Ok((CompressionAlgorithm::None, false))
Ok((CompressionAlgorithm::None, crate::rio::ReadCompressionBackend::Legacy, false))
}
}
@@ -388,13 +355,18 @@ impl ObjectInfo {
use rustfs_utils::http::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
self.user_defined.keys().any(|key| {
let key = key.to_lowercase();
key.starts_with("x-minio-encryption-")
let lower = key.to_ascii_lowercase();
lower.starts_with("x-minio-encryption-")
|| lower.starts_with("x-minio-internal-server-side-encryption-")
|| matches!(
key.as_str(),
"x-rustfs-encryption-key"
lower.as_str(),
"x-minio-internal-encrypted-multipart"
| "x-rustfs-encryption-key"
| "x-rustfs-encryption-algorithm"
| "x-rustfs-encryption-iv"
| "x-rustfs-encryption-key-id"
| "x-rustfs-encryption-context"
| "x-rustfs-encryption-tag"
| "x-amz-server-side-encryption-aws-kms-key-id"
| SSEC_ALGORITHM_HEADER
| SSEC_KEY_HEADER
@@ -405,10 +377,17 @@ impl ObjectInfo {
}
pub fn encryption_original_size(&self) -> std::io::Result<Option<i64>> {
let actual_size = rustfs_utils::http::get_str(&self.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE);
if let Some(size_str) = self
.user_defined
.get("x-rustfs-encryption-original-size")
.or_else(|| self.user_defined.get("x-amz-server-side-encryption-customer-original-size"))
.map(String::as_str)
.or_else(|| {
self.user_defined
.get("x-amz-server-side-encryption-customer-original-size")
.map(String::as_str)
})
.or(actual_size.as_deref())
&& !size_str.is_empty()
{
let size = size_str
+1
View File
@@ -87,6 +87,7 @@ fn parse_http_timestamp(value: &str) -> Option<OffsetDateTime> {
pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap<String, String>) -> PutObjectOptions {
let mut opts = PutObjectOptions {
storage_class,
send_content_md5: true,
legalhold: ObjectLockLegalHoldStatus::from_static(""),
internal: AdvancedPutOptions {
replication_status: ReplicationStatus::from_static(""),
+22 -17
View File
@@ -147,15 +147,22 @@ impl WarmBackend for WarmBackendS3 {
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
let client = self.client.clone();
let Ok(res) = client
.get_object()
.bucket(&self.bucket)
.key(&self.get_dest(object))
.send()
.await
else {
return Err(std::io::Error::other("get_object error"));
};
let mut req = client.get_object().bucket(&self.bucket).key(&self.get_dest(object));
if !rv.is_empty() {
req = req.version_id(rv);
}
if opts.start_offset >= 0 && opts.length > 0 {
let end = opts
.start_offset
.checked_add(opts.length)
.and_then(|v| v.checked_sub(1))
.ok_or_else(|| std::io::Error::other("invalid range: overflow"))?;
req = req.range(format!("bytes={}-{}", opts.start_offset, end));
}
let res = req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?;
Ok(ReadCloser::new(std::io::Cursor::new(
res.body.collect().await.map(|data| data.into_bytes().to_vec())?,
@@ -164,16 +171,14 @@ impl WarmBackend for WarmBackendS3 {
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
let client = self.client.clone();
if let Err(_) = client
.delete_object()
.bucket(&self.bucket)
.key(&self.get_dest(object))
.send()
.await
{
return Err(std::io::Error::other("delete_object error"));
let mut req = client.delete_object().bucket(&self.bucket).key(&self.get_dest(object));
if !rv.is_empty() {
req = req.version_id(rv);
}
req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?;
Ok(())
}
+24
View File
@@ -0,0 +1,24 @@
# ecstore Integration Tests
## MinIO-generated encrypted fixtures
`minio_generated_read_test.rs` validates the `bitrot -> GetObjectReader` path against raw MinIO backend data captured by
`.\rustfs\scripts\minio_fixture_lab\lab.py`.
It currently covers multipart fixtures for:
- `sse-s3-multipart-8m`
- `sse-kms-multipart-8m`
Required environment variables:
- `RUSTFS_MINIO_FIXTURE_ROOT`
- `RUSTFS_MINIO_STATIC_KMS_KEY_B64`
Example:
```powershell
$env:RUSTFS_MINIO_FIXTURE_ROOT = '.\rustfs\tmp\minio-fixture-lab-local-key'
$env:RUSTFS_MINIO_STATIC_KMS_KEY_B64 = '<base64-32-byte-local-minio-kms-key>'
cargo +1.96.0 test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored
```
@@ -0,0 +1,207 @@
#![cfg(feature = "rio-v2")]
use std::fs;
use std::io::Cursor;
use std::path::{Path, PathBuf};
use rustfs_ecstore::bitrot::create_bitrot_reader;
use rustfs_ecstore::disk::endpoint::Endpoint;
use rustfs_ecstore::disk::{DiskAPI as _, DiskOption, new_disk};
use rustfs_ecstore::store_api::{GetObjectReader, ObjectInfo, ObjectOptions};
use rustfs_filemeta::{FileInfo, FileInfoOpts, get_file_info};
use serde::Deserialize;
use sha2::{Digest, Sha256};
use temp_env::async_with_vars;
use tokio::io::AsyncReadExt;
#[derive(Debug, Deserialize)]
struct ManifestRecord {
bucket: String,
object: String,
backend_files: Vec<String>,
}
fn fixture_root() -> PathBuf {
std::env::var_os("RUSTFS_MINIO_FIXTURE_ROOT")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../rio-v2/tests/fixtures/minio-generated"))
}
fn case_dir(case_id: &str) -> PathBuf {
fixture_root().join("cases").join(case_id)
}
fn read_json<T: for<'de> Deserialize<'de>>(path: &Path) -> T {
let text = fs::read_to_string(path).unwrap_or_else(|err| panic!("read {}: {err}", path.display()));
serde_json::from_str(&text).unwrap_or_else(|err| panic!("parse {}: {err}", path.display()))
}
fn require_fixture_case(case_id: &str) -> PathBuf {
let path = case_dir(case_id);
assert!(
path.is_dir(),
"fixture case missing: {}. Run scripts/minio_fixture_lab/lab.py capture-matrix first.",
path.display()
);
path
}
fn read_plaintext_sha256(case_dir: &Path) -> String {
fs::read_to_string(case_dir.join("plaintext.sha256"))
.unwrap_or_else(|err| panic!("read plaintext.sha256 under {}: {err}", case_dir.display()))
.trim()
.to_string()
}
fn minio_static_kms_key_b64() -> String {
std::env::var("RUSTFS_MINIO_STATIC_KMS_KEY_B64")
.unwrap_or_else(|_| panic!("RUSTFS_MINIO_STATIC_KMS_KEY_B64 must point to the 32-byte static MinIO KMS key"))
}
fn object_xl_meta_path(case_dir: &Path, manifest: &ManifestRecord) -> PathBuf {
let expected = format!("disk1/{}/{}/xl.meta", manifest.bucket, manifest.object);
let relative = manifest
.backend_files
.iter()
.find(|entry| entry.as_str() == expected)
.unwrap_or_else(|| panic!("object xl.meta missing from manifest backend_files: {expected}"));
case_dir.join("backend").join(relative)
}
fn load_file_info(case_dir: &Path, manifest: &ManifestRecord) -> FileInfo {
let xl_meta_path = object_xl_meta_path(case_dir, manifest);
let xl_meta = fs::read(&xl_meta_path).unwrap_or_else(|err| panic!("read {}: {err}", xl_meta_path.display()));
get_file_info(
&xl_meta,
&manifest.bucket,
&manifest.object,
"",
FileInfoOpts {
data: true,
include_free_versions: true,
},
)
.unwrap_or_else(|err| panic!("decode {}: {err}", xl_meta_path.display()))
}
fn load_object_info(file_info: &FileInfo, manifest: &ManifestRecord) -> ObjectInfo {
ObjectInfo::from_file_info(file_info, &manifest.bucket, &manifest.object, false)
}
fn sha256_hex(bytes: &[u8]) -> String {
hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower)
}
async fn encrypted_fixture_bytes(case_dir: &Path, manifest: &ManifestRecord, file_info: &FileInfo) -> Vec<u8> {
let disk_root = case_dir.join("backend").join("disk1");
let disk_root_str = disk_root
.to_str()
.unwrap_or_else(|| panic!("non-utf8 disk root {}", disk_root.display()));
let mut endpoint = Endpoint::try_from(disk_root_str).expect("fixture disk endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("open fixture disk");
let data_dir = file_info
.data_dir
.as_ref()
.unwrap_or_else(|| panic!("fixture {} is missing data_dir", manifest.object));
let mut encrypted = Vec::new();
for part in &file_info.parts {
let checksum_info = file_info.erasure.get_checksum_info(part.number);
let path = format!("{}/{}/part.{}", manifest.object, data_dir, part.number);
let mut reader = create_bitrot_reader(
None,
Some(&disk),
&manifest.bucket,
&path,
0,
part.size,
file_info.erasure.shard_size(),
checksum_info.algorithm.clone(),
false,
false,
)
.await
.unwrap_or_else(|err| panic!("create bitrot reader for {path}: {err:?}"))
.unwrap_or_else(|| panic!("missing bitrot reader for {path}"));
let mut block = vec![0u8; file_info.erasure.shard_size()];
loop {
let n = reader
.read(&mut block)
.await
.unwrap_or_else(|err| panic!("read decoded encrypted bytes from {path}: {err}"));
if n == 0 {
break;
}
encrypted.extend_from_slice(&block[..n]);
if n < block.len() {
break;
}
}
}
disk.close().await.expect("close fixture disk");
encrypted
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn reads_minio_generated_sse_s3_multipart_fixture() {
assert_fixture_round_trip("sse-s3-multipart-8m", 8 * 1024 * 1024).await;
}
#[tokio::test]
#[ignore = "requires generated MinIO fixture data and a local static KMS key"]
async fn reads_minio_generated_sse_kms_multipart_fixture() {
assert_fixture_round_trip("sse-kms-multipart-8m", 8 * 1024 * 1024).await;
}
async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) {
let case_dir = require_fixture_case(case_id);
let manifest: ManifestRecord = read_json(&case_dir.join("manifest.json"));
let expected_sha256 = read_plaintext_sha256(&case_dir);
let file_info = load_file_info(&case_dir, &manifest);
let encrypted = encrypted_fixture_bytes(&case_dir, &manifest, &file_info).await;
let object_info = load_object_info(&file_info, &manifest);
let kms_key_b64 = minio_static_kms_key_b64();
async_with_vars(
[
("__RUSTFS_SSE_SIMPLE_CMK", Some(kms_key_b64)),
("RUSTFS_SSE_S3_MASTER_KEY", None::<String>),
],
async {
let (mut reader, offset, length) = GetObjectReader::new(
Box::new(Cursor::new(encrypted)),
None,
&object_info,
&ObjectOptions::default(),
&http::HeaderMap::new(),
)
.await
.expect("construct GetObjectReader from MinIO raw fixture");
let mut plaintext = Vec::new();
reader
.read_to_end(&mut plaintext)
.await
.expect("read plaintext from MinIO raw fixture");
assert_eq!(offset, 0);
assert_eq!(length, object_info.size);
assert_eq!(reader.object_info.size, expected_size);
assert_eq!(plaintext.len(), expected_size as usize);
assert_eq!(sha256_hex(&plaintext), expected_sha256);
},
)
.await;
}
@@ -0,0 +1,38 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use rustfs_ecstore::store_api::{
BucketInfo as EcstoreBucketInfo, BucketOptions as EcstoreBucketOptions, MakeBucketOptions as EcstoreMakeBucketOptions,
};
use rustfs_storage_api::{
BucketInfo as ApiBucketInfo, BucketOptions as ApiBucketOptions, MakeBucketOptions as ApiMakeBucketOptions,
};
#[test]
fn old_store_api_bucket_dto_path_reexports_storage_api_types() {
let ecstore_bucket: EcstoreBucketInfo = ApiBucketInfo {
name: "photos".to_owned(),
versioning: true,
..Default::default()
};
let api_bucket: ApiBucketInfo = ecstore_bucket;
let ecstore_make: EcstoreMakeBucketOptions = ApiMakeBucketOptions::default();
let api_options: ApiBucketOptions = EcstoreBucketOptions::default();
assert_eq!(api_bucket.name, "photos");
assert!(api_bucket.versioning);
assert!(!ecstore_make.lock_enabled);
assert!(!api_options.no_metadata);
}
+260
View File
@@ -14,6 +14,30 @@
use rustfs_filemeta::{FileInfoOpts, get_file_info};
use std::{env, fs, path::PathBuf};
const S2_INDEX_HEADER: &[u8] = b"s2idx\x00";
const S2_INDEX_TRAILER: &[u8] = b"\x00xdi2s";
const CHUNK_TYPE_LEGACY_INDEX: u8 = 0x50;
const CHUNK_TYPE_MINIO_INDEX: u8 = 0x99;
const SKIPPABLE_FRAME_HEADER: usize = 4;
const MAX_INDEX_ENTRIES: i64 = 1 << 16;
#[derive(Debug)]
struct CompressionIndex {
format: &'static str,
storage: &'static str,
total_uncompressed: i64,
total_compressed: i64,
est_block_uncompressed: i64,
offsets: Vec<CompressionIndexOffset>,
}
#[derive(Debug)]
struct CompressionIndexOffset {
compressed: i64,
uncompressed: i64,
}
fn main() {
let path = env::args()
.nth(1)
@@ -40,7 +64,24 @@ fn main() {
"part#{idx}: number={} size={} actual_size={} etag={}",
part.number, part.size, part.actual_size, part.etag
);
match part.index.as_deref() {
Some(index_bytes) => print_compression_index(idx, index_bytes),
None => println!("part#{idx}.index: none"),
}
}
// Tier / transition fields
if !fi.transition_status.is_empty() {
println!("transition_status: {}", fi.transition_status);
println!("transition_tier: {}", fi.transition_tier);
println!("transitioned_obj: {}", fi.transitioned_objname);
println!(
"transition_ver_id: {}",
fi.transition_version_id
.map(|u| u.to_string())
.unwrap_or_else(|| "<none>".into())
);
}
println!("metadata entries: {}", fi.metadata.len());
let mut keys = fi.metadata.keys().cloned().collect::<Vec<_>>();
keys.sort();
@@ -48,3 +89,222 @@ fn main() {
println!("meta[{key}]={}", fi.metadata.get(&key).unwrap());
}
}
fn print_compression_index(part_idx: usize, bytes: &[u8]) {
match decode_compression_index(bytes) {
Ok(index) => {
println!(
"part#{part_idx}.index: bytes={} format={} storage={} entries={} total_uncompressed={} total_compressed={} est_block_uncompressed={}",
bytes.len(),
index.format,
index.storage,
index.offsets.len(),
index.total_uncompressed,
index.total_compressed,
index.est_block_uncompressed
);
for (idx, offset) in index.offsets.iter().take(5).enumerate() {
println!(
"part#{part_idx}.index.offset#{idx}: compressed={} uncompressed={}",
offset.compressed, offset.uncompressed
);
}
if index.offsets.len() > 5
&& let Some(offset) = index.offsets.last()
{
println!(
"part#{part_idx}.index.offset#last: compressed={} uncompressed={}",
offset.compressed, offset.uncompressed
);
}
}
Err(err) => {
println!("part#{part_idx}.index: bytes={} decode_error={err}", bytes.len());
}
}
}
fn decode_compression_index(bytes: &[u8]) -> Result<CompressionIndex, String> {
if bytes.is_empty() {
return Err("empty index".to_string());
}
let mut errors = Vec::new();
for candidate in index_candidates(bytes) {
match parse_index(&candidate.bytes, candidate.format, candidate.storage, candidate.signed_varint) {
Ok(index) => return Ok(index),
Err(err) => errors.push(format!("{} {}: {err}", candidate.format, candidate.storage)),
}
}
Err(errors.join("; "))
}
struct IndexCandidate {
bytes: Vec<u8>,
format: &'static str,
storage: &'static str,
signed_varint: bool,
}
fn index_candidates(bytes: &[u8]) -> Vec<IndexCandidate> {
let mut candidates = Vec::new();
match bytes[0] {
CHUNK_TYPE_MINIO_INDEX => candidates.push(IndexCandidate {
bytes: bytes.to_vec(),
format: "minio-s2",
storage: "full",
signed_varint: true,
}),
CHUNK_TYPE_LEGACY_INDEX => candidates.push(IndexCandidate {
bytes: bytes.to_vec(),
format: "legacy-rustfs",
storage: "full",
signed_varint: false,
}),
_ => {
candidates.push(IndexCandidate {
bytes: restore_index_headers(bytes, CHUNK_TYPE_MINIO_INDEX),
format: "minio-s2",
storage: "headerless",
signed_varint: true,
});
candidates.push(IndexCandidate {
bytes: restore_index_headers(bytes, CHUNK_TYPE_LEGACY_INDEX),
format: "legacy-rustfs",
storage: "headerless",
signed_varint: false,
});
}
}
candidates
}
fn parse_index(
bytes: &[u8],
format: &'static str,
storage: &'static str,
signed_varint: bool,
) -> Result<CompressionIndex, String> {
if bytes.len() <= SKIPPABLE_FRAME_HEADER + S2_INDEX_HEADER.len() + S2_INDEX_TRAILER.len() {
return Err("buffer too small".to_string());
}
if bytes[0] != CHUNK_TYPE_MINIO_INDEX && bytes[0] != CHUNK_TYPE_LEGACY_INDEX {
return Err(format!("invalid chunk type 0x{:02x}", bytes[0]));
}
let chunk_len = bytes[1] as usize | ((bytes[2] as usize) << 8) | ((bytes[3] as usize) << 16);
let chunk = bytes
.get(SKIPPABLE_FRAME_HEADER..SKIPPABLE_FRAME_HEADER + chunk_len)
.ok_or_else(|| "chunk length exceeds buffer".to_string())?;
let mut body = chunk
.strip_prefix(S2_INDEX_HEADER)
.ok_or_else(|| "invalid index header".to_string())?;
let (total_uncompressed, used) = read_varint(body, signed_varint)?;
body = &body[used..];
let (total_compressed, used) = read_varint(body, signed_varint)?;
body = &body[used..];
let (est_block_uncompressed, used) = read_varint(body, signed_varint)?;
body = &body[used..];
let (entries, used) = read_varint(body, signed_varint)?;
body = &body[used..];
if total_uncompressed < 0 || est_block_uncompressed < 0 || !(0..=MAX_INDEX_ENTRIES).contains(&entries) {
return Err("invalid index totals".to_string());
}
let has_uncompressed = *body.first().ok_or_else(|| "missing uncompressed-offset flag".to_string())?;
if has_uncompressed & 1 != has_uncompressed {
return Err("invalid uncompressed-offset flag".to_string());
}
body = &body[1..];
let mut offsets = (0..entries)
.map(|_| CompressionIndexOffset {
compressed: 0,
uncompressed: 0,
})
.collect::<Vec<_>>();
for idx in 0..offsets.len() {
let mut uncompressed = 0;
if has_uncompressed != 0 {
let (value, used) = read_varint(body, signed_varint)?;
uncompressed = value;
body = &body[used..];
}
if idx > 0 {
let prev = offsets[idx - 1].uncompressed;
uncompressed += prev + est_block_uncompressed;
}
offsets[idx].uncompressed = uncompressed;
}
let mut compressed_predict = est_block_uncompressed / 2;
for idx in 0..offsets.len() {
let (mut compressed, used) = read_varint(body, signed_varint)?;
body = &body[used..];
if idx > 0 {
let next_compressed_predict = compressed_predict + compressed / 2;
compressed += offsets[idx - 1].compressed + compressed_predict;
compressed_predict = next_compressed_predict;
}
offsets[idx].compressed = compressed;
}
if body.len() < 4 + S2_INDEX_TRAILER.len() {
return Err("missing index trailer".to_string());
}
body = &body[4..];
if !body.starts_with(S2_INDEX_TRAILER) {
return Err("invalid index trailer".to_string());
}
Ok(CompressionIndex {
format,
storage,
total_uncompressed,
total_compressed,
est_block_uncompressed,
offsets,
})
}
fn restore_index_headers(input: &[u8], chunk_type: u8) -> Vec<u8> {
let mut bytes = Vec::with_capacity(SKIPPABLE_FRAME_HEADER + S2_INDEX_HEADER.len() + input.len() + 4 + S2_INDEX_TRAILER.len());
bytes.extend_from_slice(&[chunk_type, 0, 0, 0]);
bytes.extend_from_slice(S2_INDEX_HEADER);
bytes.extend_from_slice(input);
bytes.extend_from_slice(&((bytes.len() + 4 + S2_INDEX_TRAILER.len()) as u32).to_le_bytes());
bytes.extend_from_slice(S2_INDEX_TRAILER);
let chunk_len = bytes.len() - SKIPPABLE_FRAME_HEADER;
bytes[1] = chunk_len as u8;
bytes[2] = (chunk_len >> 8) as u8;
bytes[3] = (chunk_len >> 16) as u8;
bytes
}
fn read_varint(bytes: &[u8], signed: bool) -> Result<(i64, usize), String> {
let mut value = 0_u64;
let mut shift = 0_u32;
for (idx, byte) in bytes.iter().copied().enumerate() {
value |= u64::from(byte & 0x7f) << shift;
if byte < 0x80 {
let value = if signed {
((value >> 1) as i64) ^ (-((value & 1) as i64))
} else {
value as i64
};
return Ok((value, idx + 1));
}
shift += 7;
if shift >= 64 {
return Err("varint overflow".to_string());
}
}
Err("unexpected EOF while reading varint".to_string())
}
+80 -3
View File
@@ -2052,8 +2052,9 @@ impl MetaObject {
let transitioned_objname = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_OBJECTNAME)
.map(|v| String::from_utf8_lossy(&v).to_string())
.unwrap_or_default();
let transition_version_id =
get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID).map(|v| Uuid::from_slice(v.as_slice()).unwrap_or_default());
let transition_version_id = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID)
.and_then(|v| Uuid::from_slice(v.as_slice()).ok())
.filter(|u| !u.is_nil());
let transition_tier = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER)
.map(|v| String::from_utf8_lossy(&v).to_string())
.unwrap_or_default();
@@ -2359,7 +2360,8 @@ impl MetaDeleteMarker {
.unwrap_or_default();
fi.transition_version_id = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID)
.map(|v| Uuid::from_slice(v.as_slice()).unwrap_or_default());
.and_then(|v| Uuid::from_slice(v.as_slice()).ok())
.filter(|u| !u.is_nil());
}
fi
@@ -3314,4 +3316,79 @@ mod tests {
let decoded = FileMetaVersion::decode_data_dir_from_meta(&encoded).expect("decode data_dir");
assert_eq!(decoded, Some(data_dir));
}
fn make_meta_object_with_sys(meta_sys: HashMap<String, Vec<u8>>) -> MetaObject {
MetaObject {
erasure_algorithm: ErasureAlgo::ReedSolomon,
erasure_m: 2,
erasure_n: 4,
erasure_block_size: 1_048_576,
erasure_index: 1,
erasure_dist: vec![1, 2, 3, 4, 5, 6],
bitrot_checksum_algo: ChecksumAlgo::HighwayHash,
meta_sys,
..Default::default()
}
}
#[test]
fn meta_object_transition_version_id_absent_yields_none() {
let fi = make_meta_object_with_sys(HashMap::new()).into_fileinfo("b", "k", false);
assert_eq!(fi.transition_version_id, None);
}
#[test]
fn meta_object_transition_version_id_empty_bytes_yields_none() {
let mut sys = HashMap::new();
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, vec![]);
let fi = make_meta_object_with_sys(sys).into_fileinfo("b", "k", false);
assert_eq!(fi.transition_version_id, None);
}
#[test]
fn meta_object_transition_version_id_nil_uuid_yields_none() {
// Regression: old code used unwrap_or_default() which turned nil bytes into Some(Uuid::nil())
let mut sys = HashMap::new();
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, Uuid::nil().as_bytes().to_vec());
let fi = make_meta_object_with_sys(sys).into_fileinfo("b", "k", false);
assert_eq!(fi.transition_version_id, None);
}
#[test]
fn meta_object_transition_version_id_valid_uuid_round_trips() {
let id = sample_version_id();
let mut sys = HashMap::new();
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, id.as_bytes().to_vec());
let fi = make_meta_object_with_sys(sys).into_fileinfo("b", "k", false);
assert_eq!(fi.transition_version_id, Some(id));
}
#[test]
fn delete_marker_free_version_transition_version_id_nil_uuid_yields_none() {
let mut sys = HashMap::new();
insert_bytes(&mut sys, SUFFIX_FREE_VERSION, vec![]);
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, Uuid::nil().as_bytes().to_vec());
let fi = MetaDeleteMarker {
version_id: None,
mod_time: None,
meta_sys: sys,
}
.into_fileinfo("b", "k", false);
assert_eq!(fi.transition_version_id, None);
}
#[test]
fn delete_marker_free_version_transition_version_id_valid_uuid_round_trips() {
let id = sample_version_id();
let mut sys = HashMap::new();
insert_bytes(&mut sys, SUFFIX_FREE_VERSION, vec![]);
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, id.as_bytes().to_vec());
let fi = MetaDeleteMarker {
version_id: None,
mod_time: None,
meta_sys: sys,
}
.into_fileinfo("b", "k", false);
assert_eq!(fi.transition_version_id, Some(id));
}
}
+51 -1
View File
@@ -32,6 +32,7 @@ use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive};
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use std::collections::{HashMap, VecDeque};
use std::fmt;
use std::future::Future;
use std::net::IpAddr;
use std::pin::Pin;
@@ -275,8 +276,14 @@ impl<'c> AsyncHttpClient<'c> for ReqwestHttpClient {
// ---- Public types (unchanged API) ----
const REDACTED_SECRET: &str = "***redacted***";
fn redacted_optional_secret(value: Option<&str>) -> &'static str {
value.filter(|secret| !secret.is_empty()).map_or("", |_| REDACTED_SECRET)
}
/// Parsed configuration for a single OIDC provider.
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Clone, PartialEq, Eq)]
pub struct OidcProviderConfig {
pub id: String,
pub enabled: bool,
@@ -298,6 +305,31 @@ pub struct OidcProviderConfig {
pub hide_from_ui: bool,
}
impl fmt::Debug for OidcProviderConfig {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("OidcProviderConfig")
.field("id", &self.id)
.field("enabled", &self.enabled)
.field("config_url", &self.config_url)
.field("client_id", &self.client_id)
.field("client_secret", &redacted_optional_secret(self.client_secret.as_deref()))
.field("scopes", &self.scopes)
.field("other_audiences", &self.other_audiences)
.field("redirect_uri", &self.redirect_uri)
.field("redirect_uri_dynamic", &self.redirect_uri_dynamic)
.field("claim_name", &self.claim_name)
.field("claim_prefix", &self.claim_prefix)
.field("role_policy", &self.role_policy)
.field("display_name", &self.display_name)
.field("groups_claim", &self.groups_claim)
.field("roles_claim", &self.roles_claim)
.field("email_claim", &self.email_claim)
.field("username_claim", &self.username_claim)
.field("hide_from_ui", &self.hide_from_ui)
.finish()
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum OidcProviderConfigSource {
@@ -1978,6 +2010,24 @@ mod tests {
}
}
#[test]
fn test_oidc_provider_config_debug_redacts_client_secret() {
let config = OidcProviderConfig {
client_secret: Some("oidc-client-secret".to_string()),
..test_config("default")
};
let sourced = SourcedOidcProviderConfig {
config,
source: OidcProviderConfigSource::Persisted,
};
let rendered = format!("{sourced:?}");
assert!(!rendered.contains("oidc-client-secret"));
assert!(rendered.contains(REDACTED_SECRET));
assert!(rendered.contains("client-id"));
}
#[test]
fn test_parse_enable_state_on() {
assert!(OidcSys::parse_enable_state("on", false, false));
+19
View File
@@ -167,6 +167,10 @@ Operation-level metrics use the same low-cardinality label set:
| `rustfs_system_network_internode_operation_requests_outgoing_total` | `operation`, `backend` | Outgoing request attempts for an internode operation. |
| `rustfs_system_network_internode_operation_requests_incoming_total` | `operation`, `backend` | Incoming request attempts for an internode operation. |
| `rustfs_system_network_internode_operation_errors_total` | `operation`, `backend` | Failed internode operation attempts. |
| `rustfs_system_network_internode_operation_classified_errors_total` | `operation`, `backend`, `classification` | Classified internode transport failures. |
| `rustfs_system_network_internode_operation_retries_total` | `operation`, `backend`, `classification` | Retry attempts for retryable internode transport failures. |
| `rustfs_system_network_internode_operation_retry_successes_total` | `operation`, `backend`, `classification` | Successful recoveries after retryable internode transport failures. |
| `rustfs_system_storage_erasure_write_quorum_failures_total` | `stage`, `dominant_error` | Erasure write quorum failures grouped by failure stage and dominant error class. |
Current `operation` values are `read_file_stream`, `put_file_stream`,
`walk_dir`, `grpc_read_all`, and `grpc_write_all`. Current `backend` values are
@@ -180,6 +184,21 @@ successful completions are not emitted as a dedicated result-labeled metric.
Adding completion/result labels is a follow-up once stream completion semantics
are defined consistently for request setup, body transfer, and shutdown.
Current low-cardinality `classification` values come from the TCP/HTTP internode
path and include:
- `connect_timeout`
- `connection_refused`
- `dns_resolution_failed`
- `connection_reset`
- `body_stream_aborted`
- `http_429`
- `http_502`
- `http_503`
- `http_504`
- `http_status_other`
- `unknown`
`scripts/run_internode_transport_baseline.sh --metrics-url ...` records metric
deltas with `operation` and `backend` columns, so the TCP baseline can attribute
bytes and request/error counts to `tcp-http` transport operations.
+127 -25
View File
@@ -30,11 +30,18 @@ pub const INTERNODE_TRANSPORT_BACKEND_UNKNOWN: &str = "unknown";
const OPERATION_LABEL: &str = "operation";
const BACKEND_LABEL: &str = "backend";
const CLASSIFICATION_LABEL: &str = "classification";
const STAGE_LABEL: &str = "stage";
const DOMINANT_ERROR_LABEL: &str = "dominant_error";
const INTERNODE_OPERATION_SENT_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_sent_bytes_total";
const INTERNODE_OPERATION_RECV_BYTES_TOTAL: &str = "rustfs_system_network_internode_operation_recv_bytes_total";
const INTERNODE_OPERATION_REQUESTS_OUTGOING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_outgoing_total";
const INTERNODE_OPERATION_REQUESTS_INCOMING_TOTAL: &str = "rustfs_system_network_internode_operation_requests_incoming_total";
const INTERNODE_OPERATION_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_errors_total";
const INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_classified_errors_total";
const INTERNODE_OPERATION_RETRIES_TOTAL: &str = "rustfs_system_network_internode_operation_retries_total";
const INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL: &str = "rustfs_system_network_internode_operation_retry_successes_total";
const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct InternodeOperationMetricDescriptor {
@@ -43,6 +50,8 @@ pub struct InternodeOperationMetricDescriptor {
}
const OPERATION_BACKEND_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL];
const OPERATION_BACKEND_CLASSIFICATION_LABELS: &[&str] = &[OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL];
const QUORUM_FAILURE_LABELS: &[&str] = &[STAGE_LABEL, DOMINANT_ERROR_LABEL];
pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &[
InternodeOperationMetricDescriptor {
@@ -65,6 +74,22 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
name: INTERNODE_OPERATION_ERRORS_TOTAL,
labels: OPERATION_BACKEND_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL,
labels: OPERATION_BACKEND_CLASSIFICATION_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_RETRIES_TOTAL,
labels: OPERATION_BACKEND_CLASSIFICATION_LABELS,
},
InternodeOperationMetricDescriptor {
name: INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL,
labels: OPERATION_BACKEND_CLASSIFICATION_LABELS,
},
InternodeOperationMetricDescriptor {
name: ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
labels: QUORUM_FAILURE_LABELS,
},
];
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
@@ -183,6 +208,60 @@ impl InternodeMetrics {
counter!(INTERNODE_OPERATION_ERRORS_TOTAL, OPERATION_LABEL => operation, BACKEND_LABEL => backend).increment(1);
}
pub fn record_classified_error_for_operation_and_backend(
&self,
operation: &'static str,
backend: &'static str,
classification: &'static str,
) {
counter!(
INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL,
OPERATION_LABEL => operation,
BACKEND_LABEL => backend,
CLASSIFICATION_LABEL => classification
)
.increment(1);
}
pub fn record_retry_for_operation_and_backend(
&self,
operation: &'static str,
backend: &'static str,
classification: &'static str,
) {
counter!(
INTERNODE_OPERATION_RETRIES_TOTAL,
OPERATION_LABEL => operation,
BACKEND_LABEL => backend,
CLASSIFICATION_LABEL => classification
)
.increment(1);
}
pub fn record_retry_success_for_operation_and_backend(
&self,
operation: &'static str,
backend: &'static str,
classification: &'static str,
) {
counter!(
INTERNODE_OPERATION_RETRY_SUCCESSES_TOTAL,
OPERATION_LABEL => operation,
BACKEND_LABEL => backend,
CLASSIFICATION_LABEL => classification
)
.increment(1);
}
pub fn record_erasure_write_quorum_failure(&self, stage: &'static str, dominant_error: &'static str) {
counter!(
ERASURE_WRITE_QUORUM_FAILURES_TOTAL,
STAGE_LABEL => stage,
DOMINANT_ERROR_LABEL => dominant_error
)
.increment(1);
}
pub fn record_dial_result(&self, duration: Duration, success: bool) {
let elapsed_nanos = duration.as_nanos().min(u128::from(u64::MAX)) as u64;
self.dial_total_time_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed);
@@ -303,10 +382,14 @@ mod tests {
#[test]
fn operation_metric_descriptors_include_backend_and_operation_labels() {
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 5);
for metric in INTERNODE_OPERATION_METRICS {
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 9);
for metric in &INTERNODE_OPERATION_METRICS[..5] {
assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL]);
}
for metric in &INTERNODE_OPERATION_METRICS[5..8] {
assert_eq!(metric.labels, &[OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL]);
}
assert_eq!(INTERNODE_OPERATION_METRICS[8].labels, &[STAGE_LABEL, DOMINANT_ERROR_LABEL]);
}
#[test]
@@ -322,29 +405,48 @@ mod tests {
assert_eq!(INTERNODE_TRANSPORT_BACKEND_UNKNOWN, "unknown");
assert_eq!(
INTERNODE_OPERATION_METRICS,
&[
InternodeOperationMetricDescriptor {
name: "rustfs_system_network_internode_operation_sent_bytes_total",
labels: &[OPERATION_LABEL, BACKEND_LABEL],
},
InternodeOperationMetricDescriptor {
name: "rustfs_system_network_internode_operation_recv_bytes_total",
labels: &[OPERATION_LABEL, BACKEND_LABEL],
},
InternodeOperationMetricDescriptor {
name: "rustfs_system_network_internode_operation_requests_outgoing_total",
labels: &[OPERATION_LABEL, BACKEND_LABEL],
},
InternodeOperationMetricDescriptor {
name: "rustfs_system_network_internode_operation_requests_incoming_total",
labels: &[OPERATION_LABEL, BACKEND_LABEL],
},
InternodeOperationMetricDescriptor {
name: "rustfs_system_network_internode_operation_errors_total",
labels: &[OPERATION_LABEL, BACKEND_LABEL],
},
]
INTERNODE_OPERATION_METRICS[5].name,
"rustfs_system_network_internode_operation_classified_errors_total"
);
assert_eq!(
INTERNODE_OPERATION_METRICS[6].name,
"rustfs_system_network_internode_operation_retries_total"
);
assert_eq!(
INTERNODE_OPERATION_METRICS[7].name,
"rustfs_system_network_internode_operation_retry_successes_total"
);
assert_eq!(
INTERNODE_OPERATION_METRICS[8].name,
"rustfs_system_storage_erasure_write_quorum_failures_total"
);
}
#[test]
fn classified_and_retry_metrics_update_counters() {
let metrics = InternodeMetrics::default();
metrics.record_classified_error_for_operation_and_backend(
INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
"connection_reset",
);
metrics.record_retry_for_operation_and_backend(
INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
"connection_reset",
);
metrics.record_retry_success_for_operation_and_backend(
INTERNODE_OPERATION_PUT_FILE_STREAM,
INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
"connection_reset",
);
metrics.record_erasure_write_quorum_failure("write", "connection_reset");
let snapshot = metrics.snapshot();
assert_eq!(snapshot.sent_bytes_total, 0);
assert_eq!(snapshot.recv_bytes_total, 0);
assert_eq!(snapshot.outgoing_requests_total, 0);
assert_eq!(snapshot.incoming_requests_total, 0);
}
}
+1
View File
@@ -57,6 +57,7 @@ moka = { workspace = true, features = ["future"] }
md5 = { workspace = true }
arc-swap = { workspace = true }
rustfs-utils = { workspace = true }
rustfs-security-governance = { workspace = true }
# HTTP client for Vault
reqwest = { workspace = true }
+119 -1
View File
@@ -16,16 +16,18 @@
use crate::config::{
BackendConfig, CacheConfig, KmsBackend, KmsConfig, LocalConfig, TlsConfig, VaultAuthMethod, VaultConfig, VaultTransitConfig,
redacted_secret_option,
};
use crate::service_manager::KmsServiceStatus;
use crate::types::{KeyMetadata, KeyUsage};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fmt;
use std::path::PathBuf;
use std::time::Duration;
/// Request to configure KMS with Local backend
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Clone, Serialize, Deserialize)]
pub struct ConfigureLocalKmsRequest {
/// Directory to store key files
pub key_dir: PathBuf,
@@ -47,6 +49,23 @@ pub struct ConfigureLocalKmsRequest {
pub cache_ttl_seconds: Option<u64>,
}
impl fmt::Debug for ConfigureLocalKmsRequest {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let master_key = redacted_secret_option(self.master_key.as_deref());
f.debug_struct("ConfigureLocalKmsRequest")
.field("key_dir", &self.key_dir)
.field("master_key", &master_key)
.field("file_permissions", &self.file_permissions)
.field("default_key_id", &self.default_key_id)
.field("timeout_seconds", &self.timeout_seconds)
.field("retry_attempts", &self.retry_attempts)
.field("enable_cache", &self.enable_cache)
.field("max_cached_keys", &self.max_cached_keys)
.field("cache_ttl_seconds", &self.cache_ttl_seconds)
.finish()
}
}
/// Request to configure KMS with Vault KV v2 + Transit backend
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ConfigureVaultKmsRequest {
@@ -428,6 +447,7 @@ impl ConfigureKmsRequest {
#[cfg(test)]
mod tests {
use super::*;
use crate::config::REDACTED_SECRET;
#[test]
fn test_deserialize_vault_kv2_configure_request_accepts_type_aliases() {
@@ -536,6 +556,104 @@ mod tests {
other => panic!("expected vault-transit summary, got {other:?}"),
}
}
#[test]
fn test_configure_request_debug_redacts_kms_secret_fields() {
let local = ConfigureKmsRequest::Local(ConfigureLocalKmsRequest {
key_dir: PathBuf::from("/tmp/kms"),
master_key: Some("local-configure-master-secret".to_string()),
file_permissions: Some(0o600),
default_key_id: Some("default-key".to_string()),
timeout_seconds: Some(30),
retry_attempts: Some(3),
enable_cache: Some(true),
max_cached_keys: Some(16),
cache_ttl_seconds: Some(60),
});
let vault = ConfigureKmsRequest::VaultTransit(ConfigureVaultTransitKmsRequest {
address: "https://vault.example.com:8200".to_string(),
auth_method: VaultAuthMethod::Token {
token: "configure-vault-token-secret".to_string(),
},
namespace: None,
mount_path: Some("transit".to_string()),
skip_tls_verify: Some(false),
default_key_id: None,
timeout_seconds: None,
retry_attempts: None,
enable_cache: None,
max_cached_keys: None,
cache_ttl_seconds: None,
});
let approle = ConfigureKmsRequest::VaultKv2(ConfigureVaultKmsRequest {
address: "https://vault.example.com:8200".to_string(),
auth_method: VaultAuthMethod::AppRole {
role_id: "configure-role-id".to_string(),
secret_id: "configure-approle-secret-id".to_string(),
},
namespace: None,
mount_path: Some("transit".to_string()),
kv_mount: Some("secret".to_string()),
key_path_prefix: Some("rustfs/kms/keys".to_string()),
skip_tls_verify: Some(false),
default_key_id: None,
timeout_seconds: None,
retry_attempts: None,
enable_cache: None,
max_cached_keys: None,
cache_ttl_seconds: None,
});
let rendered = format!("{local:?}\n{vault:?}\n{approle:?}");
assert!(!rendered.contains("local-configure-master-secret"));
assert!(!rendered.contains("configure-vault-token-secret"));
assert!(!rendered.contains("configure-approle-secret-id"));
assert!(rendered.contains("configure-role-id"));
assert!(rendered.contains(REDACTED_SECRET));
}
#[test]
fn test_kms_status_response_omits_secret_values_from_json_and_debug() {
let configs = [
KmsConfig {
backend: KmsBackend::Local,
backend_config: BackendConfig::Local(LocalConfig {
key_dir: PathBuf::from("/tmp/kms"),
master_key: Some("local-summary-master-secret".to_string()),
file_permissions: Some(0o600),
}),
..Default::default()
},
KmsConfig::vault(
url::Url::parse("https://vault.example.com:8200").expect("vault URL"),
"summary-vault-token-secret".to_string(),
),
KmsConfig::vault_approle(
url::Url::parse("https://vault.example.com:8200").expect("vault URL"),
"summary-role-id".to_string(),
"summary-approle-secret-id".to_string(),
),
];
for config in configs {
let summary = KmsConfigSummary::from(&config);
let response = KmsStatusResponse {
status: KmsServiceStatus::Configured,
backend_type: Some(config.backend.clone()),
healthy: None,
config_summary: Some(summary),
};
let json = serde_json::to_string(&response).expect("kms status response should serialize");
let debug = format!("{response:?}");
let rendered = format!("{json}\n{debug}");
assert!(!rendered.contains("local-summary-master-secret"));
assert!(!rendered.contains("summary-vault-token-secret"));
assert!(!rendered.contains("summary-approle-secret-id"));
assert!(rendered.contains("has_master_key") || rendered.contains("has_stored_credentials"));
}
}
}
// ========================================
+8 -88
View File
@@ -14,22 +14,13 @@
//! Caching layer for KMS operations to improve performance
use crate::types::{KeyMetadata, KeySpec};
use crate::types::KeyMetadata;
use moka::future::Cache;
use std::time::Duration;
/// Cached data key entry
#[derive(Clone, Debug)]
pub struct CachedDataKey {
pub plaintext: Vec<u8>,
pub ciphertext: Vec<u8>,
pub key_spec: KeySpec,
}
/// KMS cache for storing frequently accessed keys and metadata
pub struct KmsCache {
key_metadata_cache: Cache<String, KeyMetadata>,
data_key_cache: Cache<String, CachedDataKey>,
}
impl KmsCache {
@@ -44,13 +35,9 @@ impl KmsCache {
pub fn new(capacity: u64) -> Self {
Self {
key_metadata_cache: Cache::builder()
.max_capacity(capacity / 2)
.max_capacity(capacity)
.time_to_live(Duration::from_secs(300)) // 5 minutes default TTL
.build(),
data_key_cache: Cache::builder()
.max_capacity(capacity / 2)
.time_to_live(Duration::from_secs(60)) // 1 minute for data keys (shorter for security)
.build(),
}
}
@@ -77,35 +64,6 @@ impl KmsCache {
self.key_metadata_cache.run_pending_tasks().await;
}
/// Get data key from cache
///
/// # Arguments
/// * `key_id` - The ID of the key to retrieve the data key for
///
/// # Returns
/// An `Option` containing the `CachedDataKey` if found, or `None` if not found
///
pub async fn get_data_key(&self, key_id: &str) -> Option<CachedDataKey> {
self.data_key_cache.get(key_id).await
}
/// Put data key into cache
///
/// # Arguments
/// * `key_id` - The ID of the key to store the data key for
/// * `plaintext` - The plaintext data key bytes
/// * `ciphertext` - The ciphertext data key bytes
///
pub async fn put_data_key(&mut self, key_id: &str, plaintext: &[u8], ciphertext: &[u8]) {
let cached_key = CachedDataKey {
plaintext: plaintext.to_vec(),
ciphertext: ciphertext.to_vec(),
key_spec: KeySpec::Aes256, // Default to AES-256
};
self.data_key_cache.insert(key_id.to_string(), cached_key).await;
self.data_key_cache.run_pending_tasks().await;
}
/// Remove key metadata from cache
///
/// # Arguments
@@ -115,23 +73,12 @@ impl KmsCache {
self.key_metadata_cache.remove(key_id).await;
}
/// Remove data key from cache
///
/// # Arguments
/// * `key_id` - The ID of the key to remove the data key for
///
pub async fn remove_data_key(&mut self, key_id: &str) {
self.data_key_cache.remove(key_id).await;
}
/// Clear all cached entries
pub async fn clear(&mut self) {
self.key_metadata_cache.invalidate_all();
self.data_key_cache.invalidate_all();
// Wait for invalidation to complete
self.key_metadata_cache.run_pending_tasks().await;
self.data_key_cache.run_pending_tasks().await;
}
/// Get cache statistics (hit count, miss count)
@@ -140,13 +87,10 @@ impl KmsCache {
/// A tuple containing total entries and total misses
///
pub fn stats(&self) -> (u64, u64) {
let metadata_stats = (
(
self.key_metadata_cache.entry_count(),
0u64, // moka doesn't provide miss count directly
);
let data_key_stats = (self.data_key_cache.entry_count(), 0u64);
(metadata_stats.0 + data_key_stats.0, metadata_stats.1 + data_key_stats.1)
)
}
}
@@ -160,37 +104,30 @@ mod tests {
#[derive(Debug, Clone)]
struct CacheInfo {
key_metadata_count: u64,
data_key_count: u64,
}
impl CacheInfo {
fn total_entries(&self) -> u64 {
self.key_metadata_count + self.data_key_count
self.key_metadata_count
}
}
impl KmsCache {
fn with_ttl_for_tests(capacity: u64, metadata_ttl: Duration, data_key_ttl: Duration) -> Self {
fn with_ttl_for_tests(capacity: u64, metadata_ttl: Duration) -> Self {
Self {
key_metadata_cache: Cache::builder().max_capacity(capacity / 2).time_to_live(metadata_ttl).build(),
data_key_cache: Cache::builder().max_capacity(capacity / 2).time_to_live(data_key_ttl).build(),
key_metadata_cache: Cache::builder().max_capacity(capacity).time_to_live(metadata_ttl).build(),
}
}
fn info_for_tests(&self) -> CacheInfo {
CacheInfo {
key_metadata_count: self.key_metadata_cache.entry_count(),
data_key_count: self.data_key_cache.entry_count(),
}
}
fn contains_key_metadata_for_tests(&self, key_id: &str) -> bool {
self.key_metadata_cache.contains_key(key_id)
}
fn contains_data_key_for_tests(&self, key_id: &str) -> bool {
self.data_key_cache.contains_key(key_id)
}
}
#[tokio::test]
@@ -216,23 +153,10 @@ mod tests {
assert!(retrieved.is_some());
assert_eq!(retrieved.expect("metadata should be cached").key_id, "test-key-1");
// Test data key caching
let plaintext = vec![1, 2, 3, 4];
let ciphertext = vec![5, 6, 7, 8];
cache.put_data_key("test-key-1", &plaintext, &ciphertext).await;
let cached_data_key = cache.get_data_key("test-key-1").await;
assert!(cached_data_key.is_some());
let cached_data_key = cached_data_key.expect("data key should be cached");
assert_eq!(cached_data_key.plaintext, plaintext);
assert_eq!(cached_data_key.ciphertext, ciphertext);
assert_eq!(cached_data_key.key_spec, KeySpec::Aes256);
// Test cache info
let info = cache.info_for_tests();
assert_eq!(info.key_metadata_count, 1);
assert_eq!(info.data_key_count, 1);
assert_eq!(info.total_entries(), 2);
assert_eq!(info.total_entries(), 1);
// Test cache clearing
cache.clear().await;
@@ -245,7 +169,6 @@ mod tests {
let mut cache = KmsCache::with_ttl_for_tests(
100,
Duration::from_millis(100), // Short TTL for testing
Duration::from_millis(50),
);
let metadata = KeyMetadata {
@@ -277,7 +200,6 @@ mod tests {
let mut cache = KmsCache::new(100);
assert!(!cache.contains_key_metadata_for_tests("nonexistent"));
assert!(!cache.contains_data_key_for_tests("nonexistent"));
let metadata = KeyMetadata {
key_id: "contains-test".to_string(),
@@ -292,9 +214,7 @@ mod tests {
};
cache.put_key_metadata("contains-test", &metadata).await;
cache.put_data_key("contains-test", &[1, 2, 3], &[4, 5, 6]).await;
assert!(cache.contains_key_metadata_for_tests("contains-test"));
assert!(cache.contains_data_key_for_tests("contains-test"));
}
}
+162 -5
View File
@@ -15,12 +15,61 @@
//! KMS configuration management
use crate::error::{KmsError, Result};
use rustfs_security_governance::{RedactionLevel, RedactionRule};
use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_str};
use serde::{Deserialize, Serialize};
use std::fmt;
use std::path::PathBuf;
use std::time::Duration;
use url::Url;
pub const KMS_CONFIG_REDACTION_RULES: &[RedactionRule] = &[
RedactionRule::new("kms.local.master_key", RedactionLevel::Secret, "local backend key encryption material"),
RedactionRule::new("kms.vault.token", RedactionLevel::Secret, "vault authentication token"),
RedactionRule::new("kms.vault.approle.secret_id", RedactionLevel::Secret, "vault approle secret"),
RedactionRule::new("kms.vault_transit.token", RedactionLevel::Secret, "vault transit authentication token"),
RedactionRule::new(
"kms.vault_transit.approle.secret_id",
RedactionLevel::Secret,
"vault transit approle secret",
),
RedactionRule::new(
"kms.configure.local.master_key",
RedactionLevel::Secret,
"admin configure request local master key",
),
RedactionRule::new(
"kms.configure.vault.token",
RedactionLevel::Secret,
"admin configure request vault authentication token",
),
RedactionRule::new(
"kms.configure.vault.approle.secret_id",
RedactionLevel::Secret,
"admin configure request vault approle secret",
),
RedactionRule::new(
"kms.configure.vault_transit.token",
RedactionLevel::Secret,
"admin configure request vault transit authentication token",
),
RedactionRule::new(
"kms.configure.vault_transit.approle.secret_id",
RedactionLevel::Secret,
"admin configure request vault transit approle secret",
),
];
pub(crate) const REDACTED_SECRET: &str = "***redacted***";
pub(crate) fn redacted_secret(value: &str) -> &'static str {
if value.is_empty() { "" } else { REDACTED_SECRET }
}
pub(crate) fn redacted_secret_option(value: Option<&str>) -> Option<&'static str> {
value.map(redacted_secret)
}
/// KMS backend types
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub enum KmsBackend {
@@ -69,7 +118,7 @@ impl Default for KmsConfig {
}
/// Backend-specific configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Clone, Serialize, Deserialize)]
pub enum BackendConfig {
/// Local backend configuration
Local(LocalConfig),
@@ -86,8 +135,18 @@ impl Default for BackendConfig {
}
}
impl fmt::Debug for BackendConfig {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Local(config) => f.debug_tuple("Local").field(config).finish(),
Self::VaultKv2(config) => f.debug_tuple("VaultKv2").field(config).finish(),
Self::VaultTransit(config) => f.debug_tuple("VaultTransit").field(config).finish(),
}
}
}
/// Local KMS backend configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Clone, Serialize, Deserialize)]
pub struct LocalConfig {
/// Directory to store key files
pub key_dir: PathBuf,
@@ -97,6 +156,17 @@ pub struct LocalConfig {
pub file_permissions: Option<u32>,
}
impl fmt::Debug for LocalConfig {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let master_key = redacted_secret_option(self.master_key.as_deref());
f.debug_struct("LocalConfig")
.field("key_dir", &self.key_dir)
.field("master_key", &master_key)
.field("file_permissions", &self.file_permissions)
.finish()
}
}
impl Default for LocalConfig {
fn default() -> Self {
Self {
@@ -108,7 +178,7 @@ impl Default for LocalConfig {
}
/// Vault KV v2 + Transit backend configuration (metadata in KV, key wrapping via Transit)
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Clone, Serialize, Deserialize)]
pub struct VaultConfig {
/// Vault server URL
pub address: String,
@@ -126,6 +196,20 @@ pub struct VaultConfig {
pub tls: Option<TlsConfig>,
}
impl fmt::Debug for VaultConfig {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("VaultConfig")
.field("address", &self.address)
.field("auth_method", &self.auth_method)
.field("namespace", &self.namespace)
.field("mount_path", &self.mount_path)
.field("kv_mount", &self.kv_mount)
.field("key_path_prefix", &self.key_path_prefix)
.field("tls", &self.tls)
.finish()
}
}
impl Default for VaultConfig {
fn default() -> Self {
Self {
@@ -143,7 +227,7 @@ impl Default for VaultConfig {
}
/// Vault Transit backend configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Clone, Serialize, Deserialize)]
pub struct VaultTransitConfig {
/// Vault server URL
pub address: String,
@@ -157,6 +241,18 @@ pub struct VaultTransitConfig {
pub tls: Option<TlsConfig>,
}
impl fmt::Debug for VaultTransitConfig {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("VaultTransitConfig")
.field("address", &self.address)
.field("auth_method", &self.auth_method)
.field("namespace", &self.namespace)
.field("mount_path", &self.mount_path)
.field("tls", &self.tls)
.finish()
}
}
impl Default for VaultTransitConfig {
fn default() -> Self {
Self {
@@ -172,7 +268,7 @@ impl Default for VaultTransitConfig {
}
/// Vault authentication methods
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Clone, Serialize, Deserialize)]
pub enum VaultAuthMethod {
/// Token authentication
Token { token: String },
@@ -180,6 +276,19 @@ pub enum VaultAuthMethod {
AppRole { role_id: String, secret_id: String },
}
impl fmt::Debug for VaultAuthMethod {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Token { token } => f.debug_struct("Token").field("token", &redacted_secret(token)).finish(),
Self::AppRole { role_id, secret_id } => f
.debug_struct("AppRole")
.field("role_id", role_id)
.field("secret_id", &redacted_secret(secret_id))
.finish(),
}
}
}
/// TLS configuration for Vault
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TlsConfig {
@@ -460,6 +569,7 @@ impl KmsConfig {
#[cfg(test)]
mod tests {
use super::*;
use rustfs_security_governance::validate_redaction_rules;
use temp_env::with_vars;
use tempfile::TempDir;
@@ -551,6 +661,53 @@ mod tests {
assert_eq!(serialized, "\"VaultTransit\"");
}
#[test]
fn test_kms_redaction_rules_are_valid() {
assert!(validate_redaction_rules(KMS_CONFIG_REDACTION_RULES).is_ok());
}
#[test]
fn test_kms_config_debug_redacts_secret_fields() {
let local = KmsConfig {
backend: KmsBackend::Local,
backend_config: BackendConfig::Local(LocalConfig {
key_dir: PathBuf::from("/tmp/kms"),
master_key: Some("local-master-secret".to_string()),
file_permissions: Some(0o600),
}),
..Default::default()
};
let vault = KmsConfig::vault(
Url::parse("https://vault.example.com:8200").expect("vault URL"),
"vault-token-secret".to_string(),
);
let approle = KmsConfig::vault_approle(
Url::parse("https://vault.example.com:8200").expect("vault URL"),
"role-id-visible".to_string(),
"approle-secret-id".to_string(),
);
let rendered = format!("{local:?}\n{vault:?}\n{approle:?}");
assert!(!rendered.contains("local-master-secret"));
assert!(!rendered.contains("vault-token-secret"));
assert!(!rendered.contains("approle-secret-id"));
assert!(rendered.contains("role-id-visible"));
assert!(rendered.contains(REDACTED_SECRET));
}
#[test]
fn test_kms_config_serialization_preserves_secret_fields_for_persistence() {
let config = KmsConfig::vault(
Url::parse("https://vault.example.com:8200").expect("vault URL"),
"persisted-token-secret".to_string(),
);
let serialized = serde_json::to_string(&config).expect("kms config should serialize for persistence");
assert!(serialized.contains("persisted-token-secret"));
}
#[test]
fn test_config_validation() {
let mut config = KmsConfig::default();
+8
View File
@@ -33,6 +33,14 @@
//! - **Data Encryption Keys (DEK)**: Generated per object, encrypted by master keys
//! - **Object Data**: Encrypted using DEKs with AES-256-GCM or ChaCha20-Poly1305
//!
//! ## Caching Discipline
//!
//! KMS may cache stable master-key metadata, but it must not cache or reuse generated
//! data encryption keys by master key id alone. A generated DEK and its encrypted
//! ciphertext can be bound to the object encryption context, such as the bucket and
//! object path. Reusing it for another object can break context validation and would
//! also violate the expected per-object DEK model for SSE-S3 and SSE-KMS.
//!
//! ## Example
//!
//! ```rust,no_run
+60 -27
View File
@@ -71,32 +71,7 @@ impl KmsManager {
/// Generate a data encryption key
pub async fn generate_data_key(&self, request: GenerateDataKeyRequest) -> Result<GenerateDataKeyResponse> {
// Check cache first if enabled
if self.config.enable_cache {
let cache = self.cache.read().await;
if let Some(cached_key) = cache.get_data_key(&request.key_id).await
&& cached_key.key_spec == request.key_spec
{
return Ok(GenerateDataKeyResponse {
key_id: request.key_id.clone(),
plaintext_key: cached_key.plaintext.clone(),
ciphertext_blob: cached_key.ciphertext,
});
}
}
// Generate new data key from backend
let response = self.backend.generate_data_key(request).await?;
// Cache the data key if enabled
if self.config.enable_cache {
let mut cache = self.cache.write().await;
cache
.put_data_key(&response.key_id, &response.plaintext_key, &response.ciphertext_blob)
.await;
}
Ok(response)
self.backend.generate_data_key(request).await
}
/// Describe a key
@@ -156,7 +131,6 @@ impl KmsManager {
if self.config.enable_cache {
let mut cache = self.cache.write().await;
cache.remove_key_metadata(&response.key_id).await;
cache.remove_data_key(&response.key_id).await;
}
Ok(response)
@@ -186,6 +160,7 @@ mod tests {
use super::*;
use crate::backends::local::LocalKmsBackend;
use crate::types::{KeySpec, KeyState, KeyUsage};
use std::collections::HashMap;
use tempfile::tempdir;
#[tokio::test]
@@ -237,4 +212,62 @@ mod tests {
let health = manager.health_check().await.expect("Health check failed");
assert!(health);
}
#[tokio::test]
async fn generate_data_key_does_not_reuse_context_bound_ciphertext() {
let temp_dir = tempdir().expect("Failed to create temp dir");
let config = KmsConfig::local(temp_dir.path().to_path_buf());
let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
let manager = KmsManager::new(backend, config);
let create_response = manager
.create_key(CreateKeyRequest {
key_usage: KeyUsage::EncryptDecrypt,
description: Some("Context-bound data key test".to_string()),
..Default::default()
})
.await
.expect("Failed to create key");
let first_context = HashMap::from([
("bucket".to_string(), "sse-smoke".to_string()),
("object".to_string(), "first.bin".to_string()),
]);
let second_context = HashMap::from([
("bucket".to_string(), "sse-smoke".to_string()),
("object".to_string(), "second.bin".to_string()),
]);
let first = manager
.generate_data_key(GenerateDataKeyRequest {
key_id: create_response.key_id.clone(),
key_spec: KeySpec::Aes256,
encryption_context: first_context.clone(),
})
.await
.expect("Failed to generate first data key");
let second = manager
.generate_data_key(GenerateDataKeyRequest {
key_id: create_response.key_id.clone(),
key_spec: KeySpec::Aes256,
encryption_context: second_context.clone(),
})
.await
.expect("Failed to generate second data key");
assert_ne!(
first.ciphertext_blob, second.ciphertext_blob,
"data keys must not be cached only by KMS key id because ciphertext is bound to object context"
);
manager
.decrypt(DecryptRequest {
ciphertext: second.ciphertext_blob,
encryption_context: second_context,
grant_tokens: Vec::new(),
})
.await
.expect("second data key should decrypt with its own context");
}
}
+59 -8
View File
@@ -49,6 +49,26 @@ pub struct ObjectEncryptionService {
kms_manager: KmsManager,
}
fn canonical_bucket_path(bucket: &str, object_key: &str) -> String {
let bucket = bucket.trim_matches('/');
let object_key = object_key.trim_matches('/');
if object_key.is_empty() {
bucket.to_string()
} else if bucket.is_empty() {
object_key.to_string()
} else {
format!("{bucket}/{object_key}")
}
}
fn request_encryption_context(context: &ObjectEncryptionContext) -> HashMap<String, String> {
let mut enc_context = context.encryption_context.clone();
enc_context
.entry(context.bucket.clone())
.or_insert_with(|| canonical_bucket_path(&context.bucket, &context.object_key));
enc_context
}
const INTERNAL_ENCRYPTION_KEY_ID_HEADER: &str = "x-rustfs-encryption-key-id";
/// Result of object encryption
@@ -178,15 +198,10 @@ impl ObjectEncryptionService {
.or_else(|| self.kms_manager.get_default_key_id().map(|s| s.as_str()))
.ok_or_else(|| KmsError::configuration_error("No KMS key ID specified and no default configured"))?;
// Build encryption context
let mut enc_context = context.encryption_context.clone();
enc_context.insert("bucket".to_string(), context.bucket.clone());
enc_context.insert("object_key".to_string(), context.object_key.clone());
let request = GenerateDataKeyRequest {
key_id: actual_key_id.to_string(),
key_spec: KeySpec::Aes256,
encryption_context: enc_context,
encryption_context: request_encryption_context(context),
};
let data_key_response = self.kms_manager.generate_data_key(request).await?;
@@ -216,10 +231,10 @@ impl ObjectEncryptionService {
/// # Returns
/// DataKey with decrypted key
///
pub async fn decrypt_data_key(&self, encrypted_key: &[u8], _context: &ObjectEncryptionContext) -> Result<DataKey> {
pub async fn decrypt_data_key(&self, encrypted_key: &[u8], context: &ObjectEncryptionContext) -> Result<DataKey> {
let decrypt_request = DecryptRequest {
ciphertext: encrypted_key.to_vec(),
encryption_context: HashMap::new(),
encryption_context: request_encryption_context(context),
grant_tokens: Vec::new(),
};
@@ -864,4 +879,40 @@ mod tests {
.is_err()
);
}
#[tokio::test]
async fn test_decrypt_data_key_uses_object_encryption_context() {
let (service, _temp_dir) = create_test_service().await;
service
.create_key(CreateKeyRequest {
key_name: Some("test-key".to_string()),
key_usage: KeyUsage::EncryptDecrypt,
description: None,
policy: None,
tags: HashMap::new(),
origin: None,
})
.await
.expect("test key should be created");
let create_context = ObjectEncryptionContext::new("bucket".to_string(), "dir/object".to_string())
.with_encryption_context("tenant".to_string(), "alpha".to_string());
let kms_key = Some("test-key".to_string());
let (_data_key, encrypted_key) = service
.create_data_key(&kms_key, &create_context)
.await
.expect("create data key should succeed");
let wrong_context = ObjectEncryptionContext::new("bucket".to_string(), "dir/object".to_string())
.with_encryption_context("tenant".to_string(), "beta".to_string());
assert!(
service.decrypt_data_key(&encrypted_key, &wrong_context).await.is_err(),
"decrypt should reject mismatched KMS context"
);
let decrypted = service
.decrypt_data_key(&encrypted_key, &create_context)
.await
.expect("decrypt should accept matching KMS context");
assert_ne!(decrypted.plaintext_key, [0u8; 32]);
}
}
+7 -6
View File
@@ -800,6 +800,7 @@ mod tests {
should_warn_lock_failure,
};
use crate::{LockError, LockId, LockInfo, LockRequest, LockResponse, LockStats, LockType, ObjectKey, client::LockClient};
use std::assert_matches;
use std::{
collections::{HashMap, VecDeque},
sync::{Arc, Mutex},
@@ -1105,7 +1106,7 @@ mod tests {
let result = lock.acquire_guard(&request).await;
let elapsed = started.elapsed();
assert!(matches!(result, Ok(None)), "unexpected result: {result:?}");
assert_matches!(result, Ok(None));
assert!(
elapsed >= Duration::from_millis(250),
"remote RPC timeouts should be retried within the caller timeout, got {elapsed:?}"
@@ -1131,7 +1132,7 @@ mod tests {
let started = tokio::time::Instant::now();
let result = lock.acquire_guard(&request).await;
assert!(matches!(result, Ok(None)), "unexpected result: {result:?}");
assert_matches!(result, Ok(None));
assert!(
started.elapsed() < Duration::from_secs(1),
"acquire should fail this attempt before waiting for delayed impossible-quorum tasks"
@@ -1210,7 +1211,7 @@ mod tests {
let result = lock.acquire_lock_quorum_with_retry(&request).await.unwrap();
assert!(matches!(result.failure_kind, Some(LockAcquireFailureKind::NonRetryable)));
assert_matches!(result.failure_kind, Some(LockAcquireFailureKind::NonRetryable));
assert!(
result
.response
@@ -1254,7 +1255,7 @@ mod tests {
let result = lock.acquire_guard(&request).await;
let elapsed = started.elapsed();
assert!(matches!(result, Ok(None)), "unexpected result: {result:?}");
assert_matches!(result, Ok(None));
assert!(
elapsed >= Duration::from_millis(250),
"expected at least one retry attempt for transient timeout, got {elapsed:?}"
@@ -1273,7 +1274,7 @@ mod tests {
let result = lock.acquire_guard(&request).await;
assert!(matches!(result, Ok(None)), "unexpected result: {result:?}");
assert_matches!(result, Ok(None));
let seen_timeouts = seen_timeouts.lock().unwrap();
assert!(!seen_timeouts.is_empty(), "expected lock clients to observe acquire timeouts");
assert!(
@@ -1297,7 +1298,7 @@ mod tests {
let result = tokio::time::timeout(Duration::from_millis(800), lock.acquire_guard(&request)).await;
assert!(matches!(result, Ok(Ok(None))), "unexpected result: {result:?}");
assert_matches!(result, Ok(Ok(None)));
}
#[tokio::test]
+4 -3
View File
@@ -290,17 +290,18 @@ impl From<tonic::Status> for LockError {
#[cfg(test)]
mod tests {
use super::*;
use std::assert_matches;
#[test]
fn test_error_creation() {
let timeout_err = LockError::timeout("test-resource", Duration::from_secs(5));
assert!(matches!(timeout_err, LockError::Timeout { .. }));
assert_matches!(timeout_err, LockError::Timeout { .. });
let not_found_err = LockError::resource_not_found("missing-resource");
assert!(matches!(not_found_err, LockError::ResourceNotFound { .. }));
assert_matches!(not_found_err, LockError::ResourceNotFound { .. });
let permission_err = LockError::permission_denied("insufficient privileges");
assert!(matches!(permission_err, LockError::PermissionDenied { .. }));
assert_matches!(permission_err, LockError::PermissionDenied { .. });
}
#[test]
+6 -6
View File
@@ -135,7 +135,7 @@ pub struct HealingDisk {
pub finished: bool,
}
#[derive(Debug, Default, Serialize, Deserialize)]
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub enum BackendByte {
#[default]
Unknown,
@@ -143,13 +143,13 @@ pub enum BackendByte {
Erasure,
}
#[derive(Debug, Default, Serialize, Deserialize)]
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct StorageInfo {
pub disks: Vec<Disk>,
pub backend: BackendInfo,
}
#[derive(Debug, Default, Serialize, Deserialize)]
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct BackendDisks(pub HashMap<String, usize>);
impl BackendDisks {
@@ -161,7 +161,7 @@ impl BackendDisks {
}
}
#[derive(Debug, Default, Serialize, Deserialize)]
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(rename_all = "PascalCase", default)]
pub struct BackendInfo {
pub backend_type: BackendByte,
@@ -187,7 +187,7 @@ pub const ITEM_OFFLINE: &str = "offline";
pub const ITEM_INITIALIZING: &str = "initializing";
pub const ITEM_ONLINE: &str = "online";
#[derive(Debug, Default, Serialize, Deserialize)]
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct MemStats {
pub alloc: u64,
pub total_alloc: u64,
@@ -196,7 +196,7 @@ pub struct MemStats {
pub heap_alloc: u64,
}
#[derive(Debug, Default, Serialize, Deserialize)]
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct ServerProperties {
pub state: String,
pub endpoint: String,
+227 -1
View File
@@ -120,6 +120,89 @@ pub struct LastMinute {
pub ilm: HashMap<String, TimedAction>,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ScannerSourceCycleSnapshot {
#[serde(rename = "source", default)]
pub source: String,
#[serde(rename = "cycles", default)]
pub cycles: u64,
}
const SCANNER_PRIMARY_PRESSURE_NONE: &str = "none";
fn default_scanner_primary_pressure() -> String {
SCANNER_PRIMARY_PRESSURE_NONE.to_string()
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub struct ScannerPacingPressureSnapshot {
#[serde(rename = "primary_pressure", default = "default_scanner_primary_pressure")]
pub primary_pressure: String,
#[serde(rename = "current_queued_scans", default)]
pub current_queued_scans: u64,
#[serde(rename = "current_active_scans", default)]
pub current_active_scans: u64,
#[serde(rename = "last_cycle_budget_limited", default)]
pub last_cycle_budget_limited: bool,
#[serde(rename = "last_cycle_pause_observed", default)]
pub last_cycle_pause_observed: bool,
#[serde(rename = "last_cycle_throttle_sleep_ratio", default)]
pub last_cycle_throttle_sleep_ratio: f64,
#[serde(rename = "last_cycle_yield_ratio", default)]
pub last_cycle_yield_ratio: f64,
#[serde(rename = "last_cycle_total_pause_ratio", default)]
pub last_cycle_total_pause_ratio: f64,
}
impl Default for ScannerPacingPressureSnapshot {
fn default() -> Self {
Self {
primary_pressure: default_scanner_primary_pressure(),
current_queued_scans: 0,
current_active_scans: 0,
last_cycle_budget_limited: false,
last_cycle_pause_observed: false,
last_cycle_throttle_sleep_ratio: 0.0,
last_cycle_yield_ratio: 0.0,
last_cycle_total_pause_ratio: 0.0,
}
}
}
impl ScannerPacingPressureSnapshot {
fn refresh_primary_pressure(&mut self) {
self.primary_pressure = if self.current_queued_scans > 0 {
"queued_scans"
} else if self.last_cycle_budget_limited {
"cycle_budget"
} else if self.last_cycle_pause_observed {
"throttle_pause"
} else if self.current_active_scans > 0 {
"active_scans"
} else {
SCANNER_PRIMARY_PRESSURE_NONE
}
.to_string();
}
fn merge(&mut self, other: &Self) {
let pause_observed = self.last_cycle_pause_observed
|| self.primary_pressure == "throttle_pause"
|| other.last_cycle_pause_observed
|| other.primary_pressure == "throttle_pause";
self.current_queued_scans = self.current_queued_scans.saturating_add(other.current_queued_scans);
self.current_active_scans = self.current_active_scans.saturating_add(other.current_active_scans);
self.last_cycle_budget_limited |= other.last_cycle_budget_limited;
self.last_cycle_pause_observed = pause_observed;
self.last_cycle_throttle_sleep_ratio = self
.last_cycle_throttle_sleep_ratio
.max(other.last_cycle_throttle_sleep_ratio);
self.last_cycle_yield_ratio = self.last_cycle_yield_ratio.max(other.last_cycle_yield_ratio);
self.last_cycle_total_pause_ratio = self.last_cycle_total_pause_ratio.max(other.last_cycle_total_pause_ratio);
self.refresh_primary_pressure();
}
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct ScannerMetrics {
#[serde(rename = "collected")]
@@ -140,14 +223,27 @@ pub struct ScannerMetrics {
pub last_minute: LastMinute,
#[serde(rename = "active")]
pub active_paths: Vec<String>,
#[serde(rename = "last_cycle_partial_source", default)]
pub last_cycle_partial_source: String,
#[serde(rename = "last_cycle_partial_source_code", default)]
pub last_cycle_partial_source_code: u64,
#[serde(rename = "partial_cycles_by_source", default)]
pub partial_cycles_by_source: Vec<ScannerSourceCycleSnapshot>,
#[serde(rename = "pacing_pressure", default)]
pub pacing_pressure: ScannerPacingPressureSnapshot,
}
impl ScannerMetrics {
pub fn merge(&mut self, other: &Self) {
if self.collected_at < other.collected_at {
let other_is_newer = self.collected_at < other.collected_at;
if other_is_newer {
self.collected_at = other.collected_at;
self.last_cycle_partial_source = other.last_cycle_partial_source.clone();
self.last_cycle_partial_source_code = other.last_cycle_partial_source_code;
}
self.pacing_pressure.merge(&other.pacing_pressure);
if self.ongoing_buckets < other.ongoing_buckets {
self.ongoing_buckets = other.ongoing_buckets;
}
@@ -182,6 +278,20 @@ impl ScannerMetrics {
self.last_minute.ilm.entry(k.clone()).or_default().merge(v);
}
for source in other.partial_cycles_by_source.iter() {
if let Some(existing) = self
.partial_cycles_by_source
.iter_mut()
.find(|existing| existing.source == source.source)
{
existing.cycles += source.cycles;
} else {
self.partial_cycles_by_source.push(source.clone());
}
}
self.partial_cycles_by_source
.sort_by(|left, right| left.source.cmp(&right.source));
self.active_paths.extend(other.active_paths.clone());
self.active_paths.sort();
@@ -668,3 +778,119 @@ pub struct Operations {
#[serde(rename = "operations")]
pub operations: HashMap<String, TimedAction>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn scanner_metrics_merge_aggregates_partial_cycles_by_source() {
let collected_at = Utc::now();
let mut scanner = ScannerMetrics {
collected_at,
last_cycle_partial_source: "usage".to_string(),
last_cycle_partial_source_code: 1,
pacing_pressure: ScannerPacingPressureSnapshot {
primary_pressure: "cycle_budget".to_string(),
current_active_scans: 1,
last_cycle_budget_limited: true,
last_cycle_pause_observed: true,
last_cycle_throttle_sleep_ratio: 0.1,
last_cycle_yield_ratio: 0.2,
last_cycle_total_pause_ratio: 0.3,
..Default::default()
},
partial_cycles_by_source: vec![ScannerSourceCycleSnapshot {
source: "usage".to_string(),
cycles: 1,
}],
..Default::default()
};
scanner.merge(&ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
last_cycle_partial_source: "lifecycle".to_string(),
last_cycle_partial_source_code: 2,
pacing_pressure: ScannerPacingPressureSnapshot {
primary_pressure: "queued_scans".to_string(),
current_queued_scans: 4,
current_active_scans: 2,
last_cycle_throttle_sleep_ratio: 0.4,
last_cycle_yield_ratio: 0.1,
last_cycle_total_pause_ratio: 0.5,
..Default::default()
},
partial_cycles_by_source: vec![
ScannerSourceCycleSnapshot {
source: "usage".to_string(),
cycles: 2,
},
ScannerSourceCycleSnapshot {
source: "lifecycle".to_string(),
cycles: 1,
},
],
..Default::default()
});
assert_eq!(scanner.last_cycle_partial_source, "lifecycle");
assert_eq!(scanner.last_cycle_partial_source_code, 2);
assert_eq!(scanner.pacing_pressure.primary_pressure, "queued_scans");
assert_eq!(scanner.pacing_pressure.current_queued_scans, 4);
assert_eq!(scanner.pacing_pressure.current_active_scans, 3);
assert!(scanner.pacing_pressure.last_cycle_budget_limited);
assert!(scanner.pacing_pressure.last_cycle_pause_observed);
assert_eq!(scanner.pacing_pressure.last_cycle_throttle_sleep_ratio, 0.4);
assert_eq!(scanner.pacing_pressure.last_cycle_yield_ratio, 0.2);
assert_eq!(scanner.pacing_pressure.last_cycle_total_pause_ratio, 0.5);
assert_eq!(
scanner.partial_cycles_by_source,
vec![
ScannerSourceCycleSnapshot {
source: "lifecycle".to_string(),
cycles: 1,
},
ScannerSourceCycleSnapshot {
source: "usage".to_string(),
cycles: 3,
},
]
);
}
#[test]
fn scanner_metrics_merge_preserves_pause_pressure_without_duration() {
let collected_at = Utc::now();
let mut scanner = ScannerMetrics {
collected_at,
pacing_pressure: ScannerPacingPressureSnapshot {
primary_pressure: "throttle_pause".to_string(),
..Default::default()
},
..Default::default()
};
scanner.merge(&ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
pacing_pressure: ScannerPacingPressureSnapshot::default(),
..Default::default()
});
assert_eq!(scanner.pacing_pressure.primary_pressure, "throttle_pause");
assert!(scanner.pacing_pressure.last_cycle_pause_observed);
assert_eq!(scanner.pacing_pressure.last_cycle_total_pause_ratio, 0.0);
}
#[test]
fn scanner_metrics_deserializes_missing_primary_pressure_as_none() {
let pacing_pressure: ScannerPacingPressureSnapshot = serde_json::from_str(
r#"{
"current_active_scans": 1
}"#,
)
.expect("deserialize partial scanner pacing pressure");
assert_eq!(pacing_pressure.primary_pressure, "none");
assert_eq!(pacing_pressure.current_active_scans, 1);
}
}
+87
View File
@@ -0,0 +1,87 @@
# rustfs-object-capacity Agent Guide
This document is a lightweight maintainer-oriented guide for collaborators working in `crates/object-capacity`.
## Purpose
`rustfs-object-capacity` is responsible for estimating and refreshing RustFS object-data usage, not general filesystem capacity. Changes in this crate can affect:
- admin `used_capacity` responses
- refresh latency and background work
- dirty-disk propagation after writes, heal, and data movement
- capacity-related metrics and operational observability
Keep changes narrow and source-driven.
## Source of Truth
Before changing behavior, read these files first:
- `src/scan.rs`
- `src/capacity_manager.rs`
- `src/capacity_scope.rs`
- `src/types.rs`
- `../../rustfs/src/capacity/service.rs`
- `../config/src/constants/capacity.rs`
Do not treat shell scripts or old docs as the authoritative behavior definition when the Rust code says otherwise.
## Change Priorities
Prefer the following order:
1. correctness of returned capacity values
2. safety of degraded behavior under timeout, stall, or partial failure
3. consistency of dirty-subset refresh and per-disk cache state
4. bounded scan cost and runtime overhead
5. clarity of metrics and logs
## Important Behavioral Constraints
- This crate intentionally prefers a usable degraded result over hard failure when partial progress is available.
- Dirty-subset refresh is safe only after a complete per-disk cache has been established.
- Subset refresh failures with partial errors must not silently corrupt aggregate cache state.
- Environment-derived configuration is cached behind `OnceLock` in non-test builds, so runtime env changes usually require a restart.
- Symlink following is disabled by default for safety and determinism.
- Multi-disk scans are concurrent, so avoid changes that introduce hidden shared-state coupling.
## Editing Guidance
- Keep scan-path changes local and easy to audit.
- Avoid refactoring the control flow unless it is necessary for correctness.
- Preserve the current exact-vs-estimated semantics unless the task explicitly changes them.
- Reuse existing configuration constants from `rustfs-config` instead of introducing duplicate literals.
- Treat metric names and labels as compatibility-sensitive unless the task explicitly allows a metric contract change.
## Validation Guidance
For code changes, prefer focused verification close to the changed behavior:
- scan correctness tests in `src/scan.rs`
- manager/refresh-state tests in `src/capacity_manager.rs`
- scope propagation tests in `src/capacity_scope.rs`
- integration behavior in `../../rustfs/src/capacity/service.rs` or related app tests when needed
Useful commands:
```bash
cargo test -p rustfs-object-capacity
cargo test -p rustfs-object-capacity --doc
cargo bench -p rustfs-object-capacity --bench capacity_scan
```
If the change is documentation-only, lightweight verification is enough.
## Documentation Expectations
- Keep `README.md` in English.
- Keep `README_ZH.md` aligned when behavior or public usage changes.
- Prefer describing real behavior from code instead of aspirational architecture.
## Common Pitfalls
- Assuming `used_capacity` means total disk usage instead of object-data usage.
- Forgetting that the initial full refresh is required before safe dirty-subset refresh.
- Forgetting that partial errors can still produce a returned result.
- Overlooking the fallback path that stores externally supplied disk-used capacity as `DataSource::Fallback`.
- Changing timeout or sampling behavior without reviewing the observable effects on degraded results and metrics.
+353
View File
@@ -0,0 +1,353 @@
# rustfs-object-capacity
`rustfs-object-capacity` is the core object-capacity statistics component in RustFS. It scans local data directories, maintains a capacity cache, triggers incremental refreshes after writes, and provides the admin layer with a used-capacity result that is as inexpensive and resilient as possible.
This crate is not meant to measure total filesystem capacity. Its job is to answer: "How many bytes are currently occupied by RustFS object data?" It makes practical tradeoffs between accuracy, freshness, and scan cost.
## Core Responsibilities
- Scan one or more local data-disk roots and aggregate used bytes and file counts.
- Reduce scan cost on large directories with an "exact prefix + sampled overflow" strategy.
- Return usable degraded results when scans time out, traversal stalls, or some directories fail, instead of failing the entire request immediately.
- Maintain a global `HybridCapacityManager` cache with scheduled refresh, write-triggered refresh, foreground blocking refresh, and background refresh.
- Track which disks were affected by writes so the system can refresh only the dirty subset after a complete per-disk cache is available.
- Emit capacity-related metrics for observability and benchmarks.
## Module Layout
- `src/lib.rs`
Re-exports `scan_used_capacity_disks`, `CapacityDiskRef`, and `CapacityScanSummary`.
- `src/types.rs`
Defines scan input/output types, including `CapacityDiskRef`, the internal `CapacityScanResult`, and the public `CapacityScanSummary`.
- `src/scan.rs`
Implements directory traversal, sampled estimation, timeout/stall detection, multi-disk concurrent scans, and conversion into `CapacityUpdate`.
- `src/capacity_manager.rs`
Owns caching, write-frequency tracking, singleflight refresh coordination, background tasks, dirty-subset merge logic, and the global singleton manager.
- `src/capacity_scope.rs`
Tracks "which disks were touched by a write", including token-bound local scopes and the global dirty-scope registry.
- `benches/capacity_scan.rs`
Exercises the public scan API with benchmark scenarios for exact, sampled, and multi-disk scans.
## Data Model
### `CapacityDiskRef`
```rust
pub struct CapacityDiskRef {
pub endpoint: String,
pub drive_path: String,
}
```
This is the minimal unit required for a scan:
- `endpoint` is used to distinguish metrics and logs.
- `drive_path` is the local disk root path.
### `CapacityScanSummary`
```rust
pub struct CapacityScanSummary {
pub used_bytes: u64,
pub file_count: usize,
pub sampled_count: usize,
pub is_estimated: bool,
pub had_partial_errors: bool,
pub scan_duration: Duration,
}
```
Field meanings:
- `used_bytes`: the computed or estimated used capacity.
- `file_count`: the number of regular files traversed.
- `sampled_count`: the number of overflow files sampled after crossing the threshold.
- `is_estimated`: whether the result is estimated instead of exact.
- `had_partial_errors`: whether traversal encountered local errors while still producing a result.
- `scan_duration`: total scan duration.
## Scan Algorithm
The directory scan lives in `scan.rs::get_dir_size_async` and works as follows:
1. Wrap blocking directory traversal in `tokio::task::spawn_blocking` so the async runtime is not blocked.
2. Walk the directory tree with `WalkDir` and count only regular files.
3. If the file count stays below `DEFAULT_MAX_FILES_THRESHOLD` (default `200_000`), add every file size exactly.
4. After crossing the threshold:
- Keep the first `max_files_threshold` files as an exact prefix.
- Sample every `sample_rate` file after that and estimate the overflow portion from sampled bytes.
5. Periodically perform progress checks:
- If total elapsed time exceeds the timeout, attempt to fall back to a sampled estimate.
- If no file progress is observed within `stall_timeout`, treat the traversal as stalled.
6. If some directory entries or metadata reads fail:
- As long as at least one disk scan succeeds, return a partial-success result.
- Mark the result with `had_partial_errors = true`.
### Scan Concurrency
- Multi-disk scans run concurrently through `buffer_unordered`.
- The current hard-coded maximum concurrency is `4` disks.
- A failure on one disk does not immediately stop scans for the others.
### Timeout and Estimation Fallback
This crate is intentionally not "timeout means hard failure":
- If enough sampled data has already been collected, a timeout or stall produces an estimated result.
- Only when no usable estimate is available does the scan return an error.
- This keeps capacity queries useful for large directories, slow disks, and temporary I/O stalls.
### Symlink Handling
- Symlinks are not followed by default: `RUSTFS_CAPACITY_FOLLOW_SYMLINKS=false`.
- If enabled, the scan applies circular-reference detection and a maximum follow depth.
- The default maximum depth is `3`.
## Capacity Cache and Refresh Strategy
`HybridCapacityManager` is the state center of this crate.
### Cached State
- Latest total capacity value `total_used`
- Last refresh time `last_update`
- File count `file_count`
- Estimated/exact flag `is_estimated`
- Data source `DataSource`
- Per-disk cache `disk_cache`
- Dirty-disk set
- Recent 60-second write buckets
### `DataSource`
- `RealTime`
Foreground real-time refresh when no cache exists yet.
- `Scheduled`
Background refresh triggered by the scheduled task.
- `WriteTriggered`
Refresh triggered when write frequency is high and the cache is old enough.
- `Fallback`
Fallback to externally supplied disk-used capacity when all scans fail.
### Refresh Entry Points
- `refresh_or_join`
A singleflight foreground refresh. If another refresh is already running, callers join and wait for the shared result.
- `spawn_refresh_if_needed`
A background refresh. If another refresh is already running, it is skipped.
- `start_background_task`
Starts two background tasks:
- the scheduled capacity refresh task
- the runtime summary logging task
### Singleflight Semantics
`refresh_or_join` and `spawn_refresh_if_needed` use a `watch` channel to coordinate refresh cycles:
- Only one leader performs the actual refresh at a time.
- Joiners share the same published result after the leader completes.
- Panics inside the refresh function are caught and converted into errors so callers do not crash with the leader.
## Dirty Scope and Subset Refresh
One of the main optimizations in this crate is "refresh only the disks dirtied by writes".
### Scope Propagation
`capacity_scope.rs` provides two ways to propagate dirty disks:
- token scope
- The caller first binds a write operation to a disk set with `record_capacity_scope(token, scope)`.
- Later, `record_write_operation_with_scope_token(Some(token))` consumes that scope and marks the disks dirty.
- global dirty scope
- `record_global_dirty_scope(scope)` records dirty disks directly in the global registry.
- The manager drains and merges them during `get_dirty_disks()`.
### When Dirty-Subset Refresh Is Allowed
Refreshing only dirty disks is safe only when:
- `disk_cache_complete == true`
- which means the system has already completed at least one full refresh without partial errors
- and the per-disk cache is fully populated
If the per-disk cache is incomplete, or there are no dirty disks, the system falls back to a full refresh.
### Merge Rules After a Subset Refresh
- On a successful full refresh, `per_disk` replaces the entire `disk_cache`.
- On a successful dirty-subset refresh, only the affected per-disk entries are updated.
- The total capacity is recomputed from the updated `disk_cache` instead of trusting the subset sum directly.
- If a dirty-subset refresh reports partial errors, that cycle fails and the caller falls back to a full refresh to recover consistency.
## Relationship to the RustFS Main Flow
This crate provides capacity primitives only. The actual RustFS integration lives in `rustfs/src/capacity/service.rs`.
The high-level flow is:
1. Startup calls `init_capacity_management_for_local_disks()`.
2. It collects all local disks and calls `capacity_manager::start_background_task(...)`.
3. Admin used-capacity queries first try the `HybridCapacityManager` cache.
4. If the cache is fresh enough, the cached value is returned directly.
5. If the cache is stale but still acceptable, the stale value is served and a background refresh is triggered.
6. If the cache is very stale and the write rate is high, the request blocks on a foreground refresh.
7. If the initial real-time scan fails, the service falls back to externally supplied disk-used capacity and stores it as `Fallback`.
`crates/ecstore/src/set_disk.rs` is responsible for recording capacity scopes during object writes, heal operations, data movement, and related flows, so this crate can learn which disks were affected.
## Public API
### 1. Direct Scan
This is useful for benchmarks, operational tooling, or isolated validation.
```rust
use rustfs_object_capacity::{CapacityDiskRef, scan_used_capacity_disks};
let disks = vec![
CapacityDiskRef {
endpoint: "node-a".to_string(),
drive_path: "/data/disk1".to_string(),
},
];
let summary = scan_used_capacity_disks(&disks).await?;
println!(
"used={} files={} estimated={}",
summary.used_bytes, summary.file_count, summary.is_estimated
);
# Ok::<(), Box<dyn std::error::Error>>(())
```
### 2. Use the Global Manager
This is useful for in-service caching and refresh orchestration.
```rust
use rustfs_object_capacity::capacity_manager::{DataSource, get_capacity_manager};
let manager = get_capacity_manager();
if let Some(cached) = manager.get_capacity().await {
println!("cached bytes={}", cached.total_used);
}
manager.record_write_operation().await;
let _ = manager
.refresh_or_join(DataSource::Scheduled, || async {
rustfs_object_capacity::scan::refresh_capacity_with_scope(
vec![rustfs_object_capacity::CapacityDiskRef {
endpoint: "node-a".to_string(),
drive_path: "/data/disk1".to_string(),
}],
false,
)
.await
})
.await;
```
### 3. Propagate a Dirty Scope
```rust
use rustfs_object_capacity::capacity_scope::{
CapacityScope, CapacityScopeDisk, record_capacity_scope,
};
use rustfs_object_capacity::capacity_manager::get_capacity_manager;
use uuid::Uuid;
let token = Uuid::new_v4();
record_capacity_scope(
token,
CapacityScope {
disks: vec![CapacityScopeDisk {
endpoint: "node-a".to_string(),
drive_path: "/data/disk1".to_string(),
}],
},
);
get_capacity_manager()
.record_write_operation_with_scope_token(Some(token))
.await;
```
## Environment Variables and Defaults
The configuration constants are defined in `crates/config/src/constants/capacity.rs`.
| Environment Variable | Default | Description |
| --- | --- | --- |
| `RUSTFS_CAPACITY_SCHEDULED_INTERVAL` | `120s` | Scheduled refresh interval |
| `RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY` | `5s` | Debounce delay after writes |
| `RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD` | `5` | Recent 60-second write-frequency threshold |
| `RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD` | `30s` | Cache age required before fast refresh is considered |
| `RUSTFS_CAPACITY_MAX_FILES_THRESHOLD` | `200000` | Exact-count file threshold |
| `RUSTFS_CAPACITY_STAT_TIMEOUT` | `3s` | Base scan timeout |
| `RUSTFS_CAPACITY_SAMPLE_RATE` | `200` | Overflow-file sampling interval |
| `RUSTFS_CAPACITY_METRICS_INTERVAL` | `600s` | Runtime summary emission interval |
| `RUSTFS_CAPACITY_FOLLOW_SYMLINKS` | `false` | Whether to follow symlinks |
| `RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH` | `3` | Maximum symlink follow depth |
| `RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT` | `true` | Whether to enable dynamic timeout scaling |
| `RUSTFS_CAPACITY_MIN_TIMEOUT` | `2s` | Dynamic-timeout lower bound |
| `RUSTFS_CAPACITY_MAX_TIMEOUT` | `15s` | Dynamic-timeout upper bound |
| `RUSTFS_CAPACITY_STALL_TIMEOUT` | `20s` | Stall-detection threshold |
### Configuration-Caching Note
In non-test builds, configuration is cached behind `OnceLock`:
- Environment variables are effectively read once on first access.
- Updating `RUSTFS_CAPACITY_*` during runtime usually does not take effect immediately.
- A process restart is normally required to apply configuration changes reliably.
## Metrics
This crate reports multiple metric families to `rustfs-io-metrics::capacity_metrics`, including:
- cache hit / miss / served state
- refresh inflight, joiners, and success / error outcomes
- current capacity bytes
- write frequency
- dirty-disk count
- per-disk scan duration, sampling mode, timeout fallback, stall detection, and symlink statistics
So this crate is both a capacity-calculation component and an important producer of runtime observability data.
## Benchmarks
Run the benchmark suite with:
```bash
cargo bench -p rustfs-object-capacity --bench capacity_scan
```
Current benchmark scenarios:
- `capacity_scan_exact`
Single-disk exact scan over 10k files.
- `capacity_scan_sampled`
Single-disk scan over 202,048 files that triggers sampled estimation.
- `capacity_scan_multi_disk`
Four-disk exact scan with mixed directory sizes.
## Known Boundaries and Tradeoffs
- It sums file sizes under RustFS object-data directories; it is not a full replacement for filesystem-level `du`.
- Estimated mode prioritizes bounded cost and usable results over perfect per-run precision.
- Dirty-subset refresh is safe only after a complete per-disk cache has been established.
- Partial errors intentionally try to return a degraded result, which improves availability but means callers should pay attention to `had_partial_errors`.
- Symlink following is disabled by default for safety and determinism.
## Relevant Source Entry Points
- [src/lib.rs](./src/lib.rs)
- [src/scan.rs](./src/scan.rs)
- [src/capacity_manager.rs](./src/capacity_manager.rs)
- [src/capacity_scope.rs](./src/capacity_scope.rs)
- [src/types.rs](./src/types.rs)
- [benches/capacity_scan.rs](./benches/capacity_scan.rs)
- [../../rustfs/src/capacity/service.rs](../../rustfs/src/capacity/service.rs)
- [../config/src/constants/capacity.rs](../config/src/constants/capacity.rs)
+353
View File
@@ -0,0 +1,353 @@
# rustfs-object-capacity
`rustfs-object-capacity` 是 RustFS 的对象容量统计核心组件,负责扫描本地数据目录、维护容量缓存、在写入后触发增量刷新,并为上层管理接口提供尽量便宜且可恢复的 used-capacity 结果。
这个 crate 的目标不是做“磁盘总容量”探测,而是回答“RustFS 当前对象数据大约占用了多少字节”,并在精确性、实时性、扫描成本之间做工程化折中。
## 核心职责
- 扫描一个或多个本地数据盘目录,汇总对象数据占用字节数与文件数。
- 在目录规模较大时使用“前缀精确统计 + 尾部采样估算”降低扫描成本。
- 在扫描超时、遍历卡住、部分目录失败时尽量保留可用结果,而不是直接让上层完全失效。
- 维护全局 `HybridCapacityManager` 缓存,支持定时刷新、写触发刷新、前台阻塞刷新和后台异步刷新。
- 记录写入涉及的脏盘范围,在完整磁盘缓存可用时只刷新 dirty subset,而不是每次全量扫描所有盘。
- 输出容量相关 metrics,供运行时观测与基准测试使用。
## 模块划分
- `src/lib.rs`
对外导出 `scan_used_capacity_disks``CapacityDiskRef``CapacityScanSummary`
- `src/types.rs`
定义扫描输入输出类型,包括 `CapacityDiskRef`、内部 `CapacityScanResult` 和公开 `CapacityScanSummary`
- `src/scan.rs`
负责真实目录遍历、采样估算、超时/卡顿检测、多盘并发扫描,以及把扫描结果转换成 `CapacityUpdate`
- `src/capacity_manager.rs`
负责缓存、写频率统计、singleflight 刷新协调、后台定时任务、dirty subset 合并和全局单例管理。
- `src/capacity_scope.rs`
负责“写操作影响了哪些磁盘”的范围传播,包括 token 绑定的局部 scope 和全局 dirty scope 注册表。
- `benches/capacity_scan.rs`
使用公开扫描 API 做基准,覆盖单盘精确扫描、单盘采样扫描和多盘扫描。
## 数据模型
### `CapacityDiskRef`
```rust
pub struct CapacityDiskRef {
pub endpoint: String,
pub drive_path: String,
}
```
它是扫描入口的最小描述单元:
- `endpoint` 用于指标标签和日志区分。
- `drive_path` 是本地磁盘根目录。
### `CapacityScanSummary`
```rust
pub struct CapacityScanSummary {
pub used_bytes: u64,
pub file_count: usize,
pub sampled_count: usize,
pub is_estimated: bool,
pub had_partial_errors: bool,
pub scan_duration: Duration,
}
```
字段语义:
- `used_bytes`:本次扫描或估算得到的容量。
- `file_count`:遍历到的普通文件数量。
- `sampled_count`:超过阈值后被抽样统计的 overflow 文件数。
- `is_estimated`:是否为估算值。
- `had_partial_errors`:遍历中是否出现局部错误但整体仍返回了结果。
- `scan_duration`:扫描耗时。
## 扫描算法
目录扫描实现在 `scan.rs::get_dir_size_async`,核心逻辑如下:
1. 用 `tokio::task::spawn_blocking` 包裹阻塞型目录遍历,避免阻塞 async runtime。
2. 通过 `WalkDir` 遍历目录树,只统计普通文件大小。
3. 当文件数未超过 `DEFAULT_MAX_FILES_THRESHOLD`(默认 `200_000`)时,逐文件精确累加。
4. 超过阈值后:
- 前 `max_files_threshold` 个文件继续作为精确前缀保留。
- 之后每隔 `sample_rate` 个文件采样一次,基于 sampled bytes 估算 overflow 部分。
5. 周期性做进度检查:
- 若总耗时超过 timeout,则尝试退化为采样估算结果。
- 若在 `stall_timeout` 内没有任何新文件进展,则判定为 stall。
6. 若遍历中部分目录或元数据读取失败:
- 只要仍有至少一个磁盘成功,就返回部分成功结果。
- 同时设置 `had_partial_errors = true`
### 扫描并发
- 多盘扫描使用 `buffer_unordered` 并发执行。
- 当前硬编码最大并发为 `4` 个磁盘。
- 单次磁盘扫描失败不会立即中断其它磁盘。
### 超时与估算退化
crate 不是“超时就直接失败”的设计:
- 如果已经收集到足够的采样数据,超时或 stall 时会返回估算值。
- 如果还没有可用采样,才会真正报错。
- 这保证了大目录、慢盘、暂时抖动场景下,上层依然能拿到近似可用的容量值。
### 符号链接处理
- 默认不跟随符号链接:`RUSTFS_CAPACITY_FOLLOW_SYMLINKS=false`
- 开启后会做循环引用检测和最大深度限制。
- 默认最大深度是 `3`
## 容量缓存与刷新策略
`HybridCapacityManager` 是这个 crate 的状态中心。
### 缓存内容
- 最近一次容量值 `total_used`
- 更新时间 `last_update`
- 文件数 `file_count`
- 是否估算值 `is_estimated`
- 数据来源 `DataSource`
- 每盘缓存 `disk_cache`
- dirty disk 集合
- 最近 60 秒写入桶统计
### `DataSource`
- `RealTime`
首次无缓存时的前台实时刷新。
- `Scheduled`
定时后台刷新。
- `WriteTriggered`
写入频率高且缓存偏旧时触发的刷新。
- `Fallback`
全部扫描失败时,回退到外部传入的磁盘 used capacity。
### 刷新入口
- `refresh_or_join`
singleflight 前台刷新。若已有刷新进行中,调用方加入等待,不重复扫描。
- `spawn_refresh_if_needed`
后台异步刷新。若已有刷新在进行,则直接跳过。
- `start_background_task`
启动两个后台任务:
- 定时容量刷新任务
- 定时 runtime summary 日志任务
### singleflight 语义
`refresh_or_join` / `spawn_refresh_if_needed` 通过 `watch` channel 协调刷新:
- 同一时刻只允许一个 leader 真正执行 refresh。
- joiner 在 leader 完成后共享同一份结果。
- refresh panic 会被捕获并转换为错误,避免把调用者一起打崩。
## Dirty Scope 与子集刷新
这个 crate 的一个关键优化是“写后只刷新脏盘”。
### Scope 传播
`capacity_scope.rs` 提供两种脏盘传播方式:
- token scope
- 调用方先用 `record_capacity_scope(token, scope)` 把一次写操作关联到一组磁盘。
- 后续 `record_write_operation_with_scope_token(Some(token))` 会取出该 scope,并把磁盘标记为 dirty。
- global dirty scope
- 通过 `record_global_dirty_scope(scope)` 直接记录全局脏盘。
- manager 在 `get_dirty_disks()` 时会 drain 这些全局脏盘并合并。
### 何时允许 dirty subset refresh
不是所有时候都能只刷脏盘。前提是:
- `disk_cache_complete == true`
- 也就是系统已经完成过一次“无部分错误”的全盘刷新
- 并且成功拿到了每盘缓存
若当前还没有完整 per-disk cache,或者脏盘集合为空,就会回退到全盘刷新。
### 子集刷新后的合并规则
- 全盘刷新成功时,`per_disk` 会完整替换 `disk_cache`
- dirty subset 刷新成功时,只更新对应脏盘的缓存条目。
- 总容量会基于更新后的 `disk_cache` 重新求和,而不是盲信子集扫描返回的局部和。
- 若 dirty subset 刷新出现 partial errors,则当前轮次失败,并回退到全盘刷新恢复一致性。
## 与 RustFS 主流程的关系
这个 crate 本身只提供容量能力,真正把它接到 RustFS 主流程的是 `rustfs/src/capacity/service.rs`
上层使用方式大致如下:
1. 启动时调用 `init_capacity_management_for_local_disks()`
2. 它收集所有本地盘,调用 `capacity_manager::start_background_task(...)`
3. 管理接口查询 used capacity 时,优先读 `HybridCapacityManager` 缓存。
4. 缓存足够新时直接返回。
5. 缓存过旧但还可容忍时,先返回 stale cache,再后台刷新。
6. 缓存极旧且写入频率高时,前台阻塞刷新。
7. 若首次实时扫描失败,则退回外部已有的磁盘 used capacity,并写入 `Fallback` 缓存。
`crates/ecstore/src/set_disk.rs` 中则负责在对象写入、heal、data movement 等流程里记录 capacity scope,把“这次写影响了哪些盘”传播给本 crate。
## 公开 API
### 1. 直接扫描
适合 benchmark、运维工具或独立验证路径。
```rust
use rustfs_object_capacity::{CapacityDiskRef, scan_used_capacity_disks};
let disks = vec![
CapacityDiskRef {
endpoint: "node-a".to_string(),
drive_path: "/data/disk1".to_string(),
},
];
let summary = scan_used_capacity_disks(&disks).await?;
println!(
"used={} files={} estimated={}",
summary.used_bytes, summary.file_count, summary.is_estimated
);
# Ok::<(), Box<dyn std::error::Error>>(())
```
### 2. 使用全局 manager
适合服务内缓存与刷新控制。
```rust
use rustfs_object_capacity::capacity_manager::{DataSource, get_capacity_manager};
let manager = get_capacity_manager();
if let Some(cached) = manager.get_capacity().await {
println!("cached bytes={}", cached.total_used);
}
manager.record_write_operation().await;
let _ = manager
.refresh_or_join(DataSource::Scheduled, || async {
rustfs_object_capacity::scan::refresh_capacity_with_scope(
vec![rustfs_object_capacity::CapacityDiskRef {
endpoint: "node-a".to_string(),
drive_path: "/data/disk1".to_string(),
}],
false,
)
.await
})
.await;
```
### 3. 传播 dirty scope
```rust
use rustfs_object_capacity::capacity_scope::{
CapacityScope, CapacityScopeDisk, record_capacity_scope,
};
use rustfs_object_capacity::capacity_manager::get_capacity_manager;
use uuid::Uuid;
let token = Uuid::new_v4();
record_capacity_scope(
token,
CapacityScope {
disks: vec![CapacityScopeDisk {
endpoint: "node-a".to_string(),
drive_path: "/data/disk1".to_string(),
}],
},
);
get_capacity_manager()
.record_write_operation_with_scope_token(Some(token))
.await;
```
## 环境变量与默认值
配置常量定义在 `crates/config/src/constants/capacity.rs`
| 环境变量 | 默认值 | 说明 |
| --- | --- | --- |
| `RUSTFS_CAPACITY_SCHEDULED_INTERVAL` | `120s` | 定时刷新间隔 |
| `RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY` | `5s` | 写后防抖延迟 |
| `RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD` | `5` | 最近 60 秒写频率阈值 |
| `RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD` | `30s` | 缓存超过该年龄后才考虑快速刷新 |
| `RUSTFS_CAPACITY_MAX_FILES_THRESHOLD` | `200000` | 精确统计文件数阈值 |
| `RUSTFS_CAPACITY_STAT_TIMEOUT` | `3s` | 基础扫描超时 |
| `RUSTFS_CAPACITY_SAMPLE_RATE` | `200` | overflow 文件采样间隔 |
| `RUSTFS_CAPACITY_METRICS_INTERVAL` | `600s` | runtime summary 打点间隔 |
| `RUSTFS_CAPACITY_FOLLOW_SYMLINKS` | `false` | 是否跟随符号链接 |
| `RUSTFS_CAPACITY_MAX_SYMLINK_DEPTH` | `3` | 符号链接最大跟随深度 |
| `RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT` | `true` | 是否启用动态超时 |
| `RUSTFS_CAPACITY_MIN_TIMEOUT` | `2s` | 动态超时下界 |
| `RUSTFS_CAPACITY_MAX_TIMEOUT` | `15s` | 动态超时上界 |
| `RUSTFS_CAPACITY_STALL_TIMEOUT` | `20s` | 无进展 stall 判定阈值 |
### 配置缓存注意事项
在非测试构建中,配置通过 `OnceLock` 缓存:
- 环境变量只在首次读取时生效。
- 运行中修改 `RUSTFS_CAPACITY_*` 通常不会即时生效。
- 需要重启进程才能稳定应用新配置。
## Metrics
这个 crate 会向 `rustfs-io-metrics::capacity_metrics` 上报多类指标,包括但不限于:
- cache hit / miss / served 状态
- refresh inflight、joiner、success / error
- 当前容量字节数
- 写频率
- dirty disk 数量
- 单盘扫描耗时、采样模式、timeout fallback、stall、symlink 统计
因此它既是容量计算模块,也是容量观测数据的重要生产者。
## 基准测试
运行基准:
```bash
cargo bench -p rustfs-object-capacity --bench capacity_scan
```
当前 bench 场景:
- `capacity_scan_exact`
单盘 10k 文件精确扫描。
- `capacity_scan_sampled`
单盘 202,048 文件,触发采样估算。
- `capacity_scan_multi_disk`
四盘混合规模精确扫描。
## 已知边界与设计取舍
- 它统计的是对象数据目录中文件大小之和,不是文件系统 `du` 的完全等价替代。
- 估算模式优先保证成本可控和结果可用,不保证逐次完全精确。
- dirty subset refresh 只有在完整 per-disk cache 已建立后才安全。
- 部分错误会尽量返回 degraded result,这对可用性更友好,但也意味着调用方需要识别 `had_partial_errors`
- symlink 默认关闭,是出于安全性和结果确定性考虑。
## 相关源码入口
- [src/lib.rs](./src/lib.rs)
- [src/scan.rs](./src/scan.rs)
- [src/capacity_manager.rs](./src/capacity_manager.rs)
- [src/capacity_scope.rs](./src/capacity_scope.rs)
- [src/types.rs](./src/types.rs)
- [benches/capacity_scan.rs](./benches/capacity_scan.rs)
- [../../rustfs/src/capacity/service.rs](../../rustfs/src/capacity/service.rs)
- [../config/src/constants/capacity.rs](../config/src/constants/capacity.rs)
+4 -3
View File
@@ -28,6 +28,7 @@ documentation = "https://docs.rs/rustfs-obs/latest/rustfs_obs/"
[features]
default = []
gpu = ["dep:nvml-wrapper"]
pyroscope = ["dep:jemalloc_pprof", "dep:pyroscope"]
[lints]
workspace = true
@@ -71,9 +72,9 @@ zstd = { workspace = true, features = ["zstdmt"] }
sysinfo = { workspace = true }
nvml-wrapper = { workspace = true, optional = true }
[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies]
pyroscope = { workspace = true }
jemalloc_pprof = { workspace = true }
[target.'cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))'.dependencies]
pyroscope = { workspace = true, features = ["backend-pprof-rs"], optional = true }
jemalloc_pprof = { workspace = true, optional = true }
[dev-dependencies]
+26 -12
View File
@@ -28,6 +28,21 @@
use opentelemetry_sdk::{logs::SdkLoggerProvider, metrics::SdkMeterProvider, trace::SdkTracerProvider};
#[cfg(all(
feature = "pyroscope",
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
))]
pub(crate) type ProfilingAgent = pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>;
#[cfg(not(all(
feature = "pyroscope",
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
)))]
pub(crate) type ProfilingAgent = ();
#[cfg(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
pub(crate) type MemoryProfilingAgent = pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>;
#[cfg(not(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
pub(crate) type MemoryProfilingAgent = ();
/// RAII guard that owns all active OpenTelemetry providers and the
/// `tracing_appender` worker guard.
///
@@ -41,10 +56,8 @@ pub struct OtelGuard {
pub(crate) meter_provider: Option<SdkMeterProvider>,
/// Optional logger provider for OTLP log export.
pub(crate) logger_provider: Option<SdkLoggerProvider>,
#[cfg(any(target_os = "linux", target_os = "macos"))]
pub(crate) profiling_agent: Option<pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>>,
#[cfg(any(target_os = "linux", target_os = "macos"))]
pub(crate) memory_profiling_agent: Option<pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>>,
pub(crate) profiling_agent: Option<ProfilingAgent>,
pub(crate) memory_profiling_agent: Option<MemoryProfilingAgent>,
/// Handle to the background log-cleanup task; aborted on drop.
pub(crate) cleanup_handle: Option<tokio::task::JoinHandle<()>>,
/// Worker guard that keeps the non-blocking `tracing_appender` thread
@@ -59,12 +72,10 @@ impl std::fmt::Debug for OtelGuard {
let mut s = f.debug_struct("OtelGuard");
s.field("tracer_provider", &self.tracer_provider.is_some())
.field("meter_provider", &self.meter_provider.is_some())
.field("logger_provider", &self.logger_provider.is_some());
#[cfg(any(target_os = "linux", target_os = "macos"))]
s.field("profiling_agent", &self.profiling_agent.is_some());
#[cfg(any(target_os = "linux", target_os = "macos"))]
s.field("memory_profiling_agent", &self.memory_profiling_agent.is_some());
s.field("cleanup_handle", &self.cleanup_handle.is_some())
.field("logger_provider", &self.logger_provider.is_some())
.field("profiling_agent", &self.profiling_agent.is_some())
.field("memory_profiling_agent", &self.memory_profiling_agent.is_some())
.field("cleanup_handle", &self.cleanup_handle.is_some())
.field("tracing_guard", &self.tracing_guard.is_some())
.field("stdout_guard", &self.stdout_guard.is_some())
.finish()
@@ -95,7 +106,10 @@ impl Drop for OtelGuard {
eprintln!("Logger shutdown error: {err:?}");
}
#[cfg(any(target_os = "linux", target_os = "macos"))]
#[cfg(all(
feature = "pyroscope",
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
))]
if let Some(agent) = self.profiling_agent.take() {
match agent.stop() {
Err(err) => eprintln!("Profiling agent stop error: {err:?}"),
@@ -105,7 +119,7 @@ impl Drop for OtelGuard {
}
}
#[cfg(any(target_os = "linux", target_os = "macos"))]
#[cfg(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
if let Some(agent) = self.memory_profiling_agent.take() {
match agent.stop() {
Err(err) => eprintln!("Memory profiling agent stop error: {err:?}"),
-4
View File
@@ -155,9 +155,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
tracer_provider: None,
meter_provider: None,
logger_provider: None,
#[cfg(any(target_os = "linux", target_os = "macos"))]
profiling_agent: None,
#[cfg(any(target_os = "linux", target_os = "macos"))]
memory_profiling_agent: None,
tracing_guard: Some(guard),
stdout_guard: None,
@@ -291,9 +289,7 @@ fn init_file_logging_internal(
tracer_provider: None,
meter_provider: None,
logger_provider: None,
#[cfg(any(target_os = "linux", target_os = "macos"))]
profiling_agent: None,
#[cfg(any(target_os = "linux", target_os = "macos"))]
memory_profiling_agent: None,
tracing_guard: Some(guard),
stdout_guard,

Some files were not shown because too many files have changed in this diff Show More