mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-27 15:37:02 +00:00
refactor(config): migrate scanner config consumer (#3356)
This commit is contained in:
@@ -30,7 +30,7 @@ documentation = "https://docs.rs/rustfs-scanner/latest/rustfs_scanner/"
|
|||||||
workspace = true
|
workspace = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
rustfs-config = { workspace = true }
|
rustfs-config = { workspace = true, features = ["server-config-model"] }
|
||||||
rustfs-common = { workspace = true }
|
rustfs-common = { workspace = true }
|
||||||
rustfs-utils = { workspace = true }
|
rustfs-utils = { workspace = true }
|
||||||
tokio = { workspace = true, features = ["fs","sync","rt","time","io-uring","macros"] }
|
tokio = { workspace = true, features = ["fs","sync","rt","time","io-uring","macros"] }
|
||||||
|
|||||||
@@ -30,8 +30,8 @@ use rustfs_config::{
|
|||||||
SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_MAX_CONCURRENT_DISK_SCANS,
|
SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_MAX_CONCURRENT_DISK_SCANS,
|
||||||
SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_MAX_WAIT, SCANNER_SPEED, SCANNER_START_DELAY, SCANNER_SUB_SYS,
|
SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_MAX_WAIT, SCANNER_SPEED, SCANNER_START_DELAY, SCANNER_SUB_SYS,
|
||||||
SCANNER_YIELD_EVERY_N_OBJECTS, ScannerSpeed,
|
SCANNER_YIELD_EVERY_N_OBJECTS, ScannerSpeed,
|
||||||
|
server_config::{Config as ServerConfig, KVS},
|
||||||
};
|
};
|
||||||
use rustfs_ecstore::config::{Config as ServerConfig, KVS};
|
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
@@ -798,6 +798,7 @@ pub(crate) fn scanner_alert_excess_folders() -> u64 {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{ScannerRuntimeConfigSource, lookup_scanner_runtime_config, validate_scanner_runtime_config};
|
use super::{ScannerRuntimeConfigSource, lookup_scanner_runtime_config, validate_scanner_runtime_config};
|
||||||
|
use rustfs_config::server_config::{Config as ServerConfig, KVS};
|
||||||
use rustfs_config::{
|
use rustfs_config::{
|
||||||
DEFAULT_DELIMITER, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE,
|
DEFAULT_DELIMITER, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE,
|
||||||
ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, HEAL_BITROT_CYCLE,
|
ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, HEAL_BITROT_CYCLE,
|
||||||
@@ -805,7 +806,6 @@ mod tests {
|
|||||||
SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_SPEED, SCANNER_SUB_SYS,
|
SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_SPEED, SCANNER_SUB_SYS,
|
||||||
ScannerSpeed,
|
ScannerSpeed,
|
||||||
};
|
};
|
||||||
use rustfs_ecstore::config::{Config as ServerConfig, KVS};
|
|
||||||
use serial_test::serial;
|
use serial_test::serial;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|||||||
@@ -5,17 +5,17 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
## Current Context
|
## Current Context
|
||||||
|
|
||||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||||
- Branch: `overtrue/arch-config-consumers`
|
- Branch: `overtrue/arch-config-scanner-consumer`
|
||||||
- Baseline: `origin/main` at `22059911807158aae9617415a916cf1156558b7b`
|
- Baseline: `origin/main` at `704c95a22d9106b75cde26bf6fcc3988a175a19b`
|
||||||
- PR type for this branch: `consumer-migration`
|
- PR type for this branch: `consumer-migration`
|
||||||
- Runtime behavior changes: none.
|
- Runtime behavior changes: none.
|
||||||
- Rust code changes: migrate admin/runtime/audit/notify/targets/iam and
|
- Rust code changes: migrate the scanner runtime-config model consumer from
|
||||||
ECStore service/default model consumers from the temporary
|
the temporary `rustfs_ecstore::config` model path to
|
||||||
`rustfs_ecstore::config` model path to `rustfs_config::server_config`, and
|
`rustfs_config::server_config`, and enable the `server-config-model` feature
|
||||||
enable the `server-config-model` feature in affected crates.
|
in `rustfs-scanner`.
|
||||||
- CI/script changes: none.
|
- CI/script changes: none.
|
||||||
- Docs changes: record CFG-005/CFG-006 consumer-migration context, verification
|
- Docs changes: record CFG-007 consumer-migration context, verification
|
||||||
evidence, and the preserved ECStore runtime-state boundary.
|
evidence, and the preserved ECStore global server-config boundary.
|
||||||
|
|
||||||
## Phase 0 Tasks
|
## Phase 0 Tasks
|
||||||
|
|
||||||
@@ -104,6 +104,15 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
model import path except the deliberate compatibility smoke test; the old
|
model import path except the deliberate compatibility smoke test; the old
|
||||||
public re-export remains available for downstream callers until CFG-004 is
|
public re-export remains available for downstream callers until CFG-004 is
|
||||||
cleaned up.
|
cleaned up.
|
||||||
|
- [x] `CFG-007` Migrate scanner runtime-config model consumer.
|
||||||
|
- Current branch: migrate scanner runtime-config parsing and validation from
|
||||||
|
the temporary `rustfs_ecstore::config::{Config, KVS}` model path to
|
||||||
|
`rustfs_config::server_config`.
|
||||||
|
- Acceptance: scanner uses the model crate for pure server-config types while
|
||||||
|
still using ECStore for the global server-config accessor; scanner defaults,
|
||||||
|
env overrides, persisted-config validation, cycle scheduling, bitrot-cycle
|
||||||
|
compatibility, cache timeout, and alert threshold semantics remain
|
||||||
|
unchanged.
|
||||||
|
|
||||||
## Phase 1 Security Governance Tasks
|
## Phase 1 Security Governance Tasks
|
||||||
|
|
||||||
@@ -329,7 +338,8 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
## Next PRs
|
## Next PRs
|
||||||
|
|
||||||
1. `api-extraction`: remove the temporary `rustfs_ecstore::config` model
|
1. `api-extraction`: remove the temporary `rustfs_ecstore::config` model
|
||||||
re-export after downstream compatibility policy allows CFG-004 cleanup.
|
re-export after code scans prove only the deliberate CFG-004 compatibility
|
||||||
|
marker and smoke test still use the old model path.
|
||||||
2. `security-change`: make Local KMS unsafe defaults explicit development
|
2. `security-change`: make Local KMS unsafe defaults explicit development
|
||||||
opt-ins or production failures in KMSD-002.
|
opt-ins or production failures in KMSD-002.
|
||||||
3. `security-change`: make Vault unsafe defaults explicit development opt-ins
|
3. `security-change`: make Vault unsafe defaults explicit development opt-ins
|
||||||
@@ -339,34 +349,24 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
|
|
||||||
| Expert | Status | Notes |
|
| Expert | Status | Notes |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Quality/architecture | pass | Confirmed this is only a model-consumer migration and feature-gate update; ECStore persistence helpers, global state, startup wiring, config persistence, and storage hot paths stay unchanged. |
|
| Quality/architecture | pass | Confirmed the diff only moves scanner pure model imports to `rustfs_config::server_config`, enables the required feature, and keeps ECStore global-state access through ECStore. |
|
||||||
| Migration preservation | pass | Confirmed the old `rustfs_ecstore::config` model path remains available through `RUSTFS_COMPAT_TODO(CFG-004)`, with no tuple-shape, serde, default, or persisted JSON changes. |
|
| Migration preservation | pass | Confirmed scanner defaults, env/config precedence, validation, cycle scheduling, bitrot compatibility, cache timeout, alert thresholds, and active global-config access are unchanged. |
|
||||||
| Testing/verification | pass | Confirmed focused config/admin/audit/notify/targets/iam tests, compile checks, migration guards, boundary scans, and added-line risk scan are sufficient; full pre-commit is skipped under the current larger-granularity instruction. |
|
| Testing/verification | pass | Confirmed focused scanner runtime-config tests, scanner/ECStore/server compile check, migration guards, old-path scan, and added-line risk scan are sufficient; full pre-commit is skipped under the current larger-granularity instruction. |
|
||||||
|
|
||||||
## Verification Notes
|
## Verification Notes
|
||||||
|
|
||||||
Passed:
|
Passed:
|
||||||
- `cargo fmt --all --check`.
|
- `cargo fmt --all --check`.
|
||||||
- `cargo check -p rustfs-config -p rustfs-audit -p rustfs-notify -p rustfs-targets -p rustfs-iam -p rustfs-ecstore -p rustfs --lib`.
|
- `cargo test -p rustfs-scanner runtime_config --lib`; 16 passed.
|
||||||
- `cargo test -p rustfs-config --features server-config-model --lib`; 29 passed.
|
- `cargo check -p rustfs-scanner -p rustfs-ecstore -p rustfs --lib`.
|
||||||
- `cargo test -p rustfs-ecstore config --lib`; 60 passed.
|
|
||||||
- `cargo test -p rustfs-targets config --lib`; 65 passed.
|
|
||||||
- `cargo test -p rustfs-notify config_manager --lib`; 12 passed.
|
|
||||||
- `cargo test -p rustfs-audit --lib`; 5 passed.
|
|
||||||
- `cargo test -p rustfs-iam oidc --lib`; 53 passed.
|
|
||||||
- `cargo test -p rustfs admin::handlers::config_admin --lib`; 29 passed.
|
|
||||||
- `cargo test -p rustfs admin::service::config --lib`; 6 passed.
|
|
||||||
- `./scripts/check_architecture_migration_rules.sh`.
|
- `./scripts/check_architecture_migration_rules.sh`.
|
||||||
- `./scripts/check_layer_dependencies.sh`.
|
- `./scripts/check_layer_dependencies.sh`.
|
||||||
- `./scripts/check_metrics_migration_refs.sh`.
|
- `./scripts/check_metrics_migration_refs.sh`.
|
||||||
- `./scripts/check_unsafe_code_allowances.sh`.
|
- `./scripts/check_unsafe_code_allowances.sh`.
|
||||||
- `git diff --check`.
|
- `git diff --check`.
|
||||||
- External old-model path scan:
|
- Old model code-path scan:
|
||||||
`rustfs_ecstore::config::{Config, KV, KVS}` is absent from `rustfs/src`,
|
`rustfs_ecstore::config::{Config, KV, KVS, DEFAULT_KVS,
|
||||||
`crates/audit`, `crates/notify`, `crates/targets`, and `crates/iam`.
|
register_default_kvs}` is absent from `crates/**/*.rs` and `rustfs/src`.
|
||||||
- ECStore runtime helper scan: no `rustfs_config::server_config::{init,
|
|
||||||
get_global_server_config, set_global_server_config,
|
|
||||||
set_global_storage_class, com, storageclass}` usage exists.
|
|
||||||
- Added-line risk scan found no production `unwrap`/`expect`, lossy numeric
|
- Added-line risk scan found no production `unwrap`/`expect`, lossy numeric
|
||||||
casts, stringly public errors, boxed dynamic errors, stdout/stderr printing,
|
casts, stringly public errors, boxed dynamic errors, stdout/stderr printing,
|
||||||
or relaxed atomic ordering.
|
or relaxed atomic ordering.
|
||||||
@@ -374,19 +374,20 @@ Passed:
|
|||||||
Notes:
|
Notes:
|
||||||
- Full pre-commit may be skipped if focused tests, compile checks, and guards
|
- Full pre-commit may be skipped if focused tests, compile checks, and guards
|
||||||
pass, per the current instruction to increase PR granularity.
|
pass, per the current instruction to increase PR granularity.
|
||||||
- This slice migrates model consumers only. ECStore retains persistence helpers,
|
- This slice migrates the scanner model consumer only. ECStore retains
|
||||||
ConfigSys, global server-config state, storage-class global state, startup
|
persistence helpers, ConfigSys, global server-config state, storage-class
|
||||||
wiring, and all storage/config persistence logic.
|
global state, startup wiring, and all storage/config persistence logic.
|
||||||
- The old rustfs_ecstore::config model path intentionally remains as a
|
- The old rustfs_ecstore::config model path intentionally remains as a
|
||||||
temporary compatibility re-export with `RUSTFS_COMPAT_TODO(CFG-004)` and a
|
temporary compatibility re-export with `RUSTFS_COMPAT_TODO(CFG-004)` and a
|
||||||
matching cleanup-register entry.
|
matching cleanup-register entry.
|
||||||
- The remaining ECStore old-path references are limited to the deliberate
|
- The remaining old-path references are limited to architecture documentation;
|
||||||
compatibility smoke test in `crates/ecstore/src/config/mod.rs`.
|
the deliberate CFG-004 compatibility re-export remains available but is no
|
||||||
|
longer imported by Rust code in `crates` or `rustfs/src`.
|
||||||
|
|
||||||
## Handoff Notes
|
## Handoff Notes
|
||||||
|
|
||||||
- Keep this CFG-005/CFG-006 slice as a `consumer-migration` PR that only
|
- Keep this CFG-007 slice as a `consumer-migration` PR that only updates the
|
||||||
updates model type imports and affected crate feature gates.
|
scanner model type import and affected crate feature gate.
|
||||||
- Do not move `ConfigSys`, `GLOBAL_SERVER_CONFIG`, storage-class global state,
|
- Do not move `ConfigSys`, `GLOBAL_SERVER_CONFIG`, storage-class global state,
|
||||||
`read_config_without_migrate`, `save_server_config`, config-object helpers,
|
`read_config_without_migrate`, `save_server_config`, config-object helpers,
|
||||||
startup wiring, storage-class helpers, ECStore persistence helpers, or storage
|
startup wiring, storage-class helpers, ECStore persistence helpers, or storage
|
||||||
|
|||||||
Reference in New Issue
Block a user