gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-27 15:37:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(config): extract server config model (#3351)

Browse Source
This commit is contained in:
安正超
2026-06-11 16:11:17 +08:00
committed by GitHub
parent 7d38b0cf90
commit 2205991180
9 changed files with 330 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/config/Cargo.toml
+3
View File
@@ -26,6 +26,8 @@ categories = ["web-programming", "development-tools", "config"]
[dependencies]
const-str = { workspace = true, optional = true }
serde = { workspace = true, optional = true }
serde_json = { workspace = true, optional = true }
[lints]
workspace = true
@@ -37,6 +39,7 @@ constants = ["dep:const-str"]
notify = ["dep:const-str", "constants"]
observability = ["constants"]
opa = ["constants"]
server-config-model = ["constants", "dep:serde", "dep:serde_json"]
[lib]
doctest = false
crates/config/src/lib.rs
+2
View File
@@ -68,3 +68,5 @@ pub mod notify;
pub mod observability;
#[cfg(feature = "opa")]
pub mod opa;
#[cfg(feature = "server-config-model")]
pub mod server_config;
crates/config/src/server_config.rs
+229
View File
@@ -0,0 +1,229 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::{LazyLock, OnceLock};
use crate::{COMMENT_KEY, DEFAULT_DELIMITER};
pub static DEFAULT_KVS: LazyLock<OnceLock<HashMap<String, KVS>>> = LazyLock::new(OnceLock::new);
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct KV {
pub key: String,
pub value: String,
#[serde(default, alias = "hiddenIfEmpty")]
pub hidden_if_empty: bool,
}
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct KVS(pub Vec<KV>);
impl Default for KVS {
fn default() -> Self {
Self::new()
}
}
impl KVS {
pub fn new() -> Self {
KVS(Vec::new())
}
pub fn get(&self, key: &str) -> String {
if let Some(v) = self.lookup(key) { v } else { "".to_owned() }
}
pub fn lookup(&self, key: &str) -> Option<String> {
for kv in self.0.iter() {
if kv.key.as_str() == key {
return Some(kv.value.clone());
}
}
None
}
/// Check if KVS is empty.
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
/// Returns a list of all keys for the current KVS.
/// If the "comment" key does not exist, it will be added.
pub fn keys(&self) -> Vec<String> {
let mut found_comment = false;
let mut keys: Vec<String> = self
.0
.iter()
.map(|kv| {
if kv.key == COMMENT_KEY {
found_comment = true;
}
kv.key.clone()
})
.collect();
if !found_comment {
keys.push(COMMENT_KEY.to_owned());
}
keys
}
/// Insert or update a pair of key/values in KVS
pub fn insert(&mut self, key: String, value: String) {
for kv in self.0.iter_mut() {
if kv.key == key {
kv.value = value;
return;
}
}
self.0.push(KV {
key,
value,
hidden_if_empty: false,
});
}
/// Merge all entries from another KVS to the current instance
pub fn extend(&mut self, other: KVS) {
for KV { key, value, .. } in other.0.into_iter() {
self.insert(key, value);
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Config(pub HashMap<String, HashMap<String, KVS>>);
impl Default for Config {
fn default() -> Self {
Self::new()
}
}
impl Config {
pub fn new() -> Self {
let mut cfg = Config(HashMap::new());
cfg.set_defaults();
cfg
}
pub fn get_value(&self, sub_sys: &str, key: &str) -> Option<KVS> {
if let Some(m) = self.0.get(sub_sys) {
m.get(key).cloned()
} else {
None
}
}
pub fn set_defaults(&mut self) {
if let Some(defaults) = DEFAULT_KVS.get() {
for (k, v) in defaults.iter() {
if !self.0.contains_key(k) {
let mut default = HashMap::new();
default.insert(DEFAULT_DELIMITER.to_owned(), v.clone());
self.0.insert(k.clone(), default);
} else if !self.0[k].contains_key(DEFAULT_DELIMITER)
&& let Some(m) = self.0.get_mut(k)
{
m.insert(DEFAULT_DELIMITER.to_owned(), v.clone());
}
}
}
}
pub fn unmarshal(data: &[u8]) -> Result<Config, serde_json::Error> {
let m: HashMap<String, HashMap<String, KVS>> = serde_json::from_slice(data)?;
let mut cfg = Config(m);
cfg.set_defaults();
Ok(cfg)
}
pub fn marshal(&self) -> Result<Vec<u8>, serde_json::Error> {
let data = serde_json::to_vec(&self.0)?;
Ok(data)
}
pub fn merge(&self) -> Config {
// TODO: merge default
self.clone()
}
}
pub fn register_default_kvs(kvs: HashMap<String, KVS>) {
let mut p = HashMap::new();
for (k, v) in kvs {
p.insert(k, v);
}
let _ = DEFAULT_KVS.set(p);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn kvs_preserves_lookup_insert_extend_and_keys_behavior() {
let mut kvs = KVS::new();
assert!(kvs.is_empty());
kvs.insert("first".to_string(), "1".to_string());
kvs.insert("first".to_string(), "2".to_string());
kvs.extend(KVS(vec![KV {
key: "second".to_string(),
value: "3".to_string(),
hidden_if_empty: true,
}]));
assert_eq!(kvs.get("first"), "2");
assert_eq!(kvs.lookup("second"), Some("3".to_string()));
assert_eq!(kvs.get("missing"), "");
assert!(kvs.keys().contains(&COMMENT_KEY.to_string()));
}
#[test]
fn kv_hidden_if_empty_accepts_legacy_camel_case_alias() {
let kvs: KVS = serde_json::from_str(r#"[{"key":"token","value":"","hiddenIfEmpty":true}]"#)
.expect("legacy hiddenIfEmpty alias should deserialize");
assert!(kvs.0[0].hidden_if_empty);
}
#[test]
fn config_marshal_unmarshal_preserves_internal_json_shape() {
let mut kvs = KVS::new();
kvs.insert("standard".to_string(), "EC:4".to_string());
let cfg = Config(HashMap::from([(
"storage_class".to_string(),
HashMap::from([(DEFAULT_DELIMITER.to_string(), kvs)]),
)]));
let data = cfg.marshal().expect("config should marshal");
let loaded = Config::unmarshal(&data).expect("config should unmarshal");
assert_eq!(
loaded
.get_value("storage_class", DEFAULT_DELIMITER)
.expect("storage class should exist")
.get("standard"),
"EC:4"
);
assert_eq!(loaded.merge(), loaded);
}
}
crates/ecstore/Cargo.toml
+1 -1
View File
@@ -43,7 +43,7 @@ rustfs-signer.workspace = true
rustfs-storage-api.workspace = true
rustfs-tls-runtime.workspace = true
rustfs-checksums.workspace = true
rustfs-config = { workspace = true, features = ["constants", "notify", "audit"] }
rustfs-config = { workspace = true, features = ["constants", "notify", "audit", "server-config-model"] }
rustfs-credentials = { workspace = true }
rustfs-common.workspace = true
rustfs-policy.workspace = true
crates/ecstore/src/config/com.rs
+1 -1
View File
@@ -2697,7 +2697,7 @@ mod tests {
// Verify the decoded config has "storage_class" (with underscore) subsystem
let kvs = decoded
.get_value(STORAGE_CLASS_SUB_SYS, crate::config::DEFAULT_DELIMITER)
.get_value(STORAGE_CLASS_SUB_SYS, rustfs_config::DEFAULT_DELIMITER)
.expect("decoded config should have storage_class subsystem");
assert_eq!(kvs.get("standard"), "EC:4", "standard should be EC:4");
assert_eq!(kvs.get("rrs"), "EC:2", "rrs should be EC:2");
crates/ecstore/src/config/mod.rs
+23 -157
View File
@@ -24,8 +24,6 @@ pub mod storageclass;
use crate::error::Result;
use crate::store::ECStore;
use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
use rustfs_config::COMMENT_KEY;
use rustfs_config::DEFAULT_DELIMITER;
use rustfs_config::HEAL_SUB_SYS;
use rustfs_config::audit::{
AUDIT_AMQP_SUB_SYS, AUDIT_KAFKA_SUB_SYS, AUDIT_MQTT_SUB_SYS, AUDIT_MYSQL_SUB_SYS, AUDIT_NATS_SUB_SYS, AUDIT_POSTGRES_SUB_SYS,
@@ -36,14 +34,15 @@ use rustfs_config::notify::{
NOTIFY_POSTGRES_SUB_SYS, NOTIFY_PULSAR_SUB_SYS, NOTIFY_REDIS_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS,
};
use rustfs_config::oidc::IDENTITY_OPENID_SUB_SYS;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::LazyLock;
use std::sync::{Arc, OnceLock, RwLock};
use std::sync::{Arc, RwLock};
// RUSTFS_COMPAT_TODO(CFG-004): keep old rustfs_ecstore::config model import paths while server-config model consumers migrate. Remove after all consumers import Config, KV, KVS, DEFAULT_KVS, and register_default_kvs from rustfs_config::server_config.
pub use rustfs_config::server_config::{Config, DEFAULT_KVS, KV, KVS, register_default_kvs};
pub static GLOBAL_STORAGE_CLASS: LazyLock<RwLock<storageclass::Config>> =
LazyLock::new(|| RwLock::new(storageclass::Config::default()));
pub static DEFAULT_KVS: LazyLock<OnceLock<HashMap<String, KVS>>> = LazyLock::new(OnceLock::new);
pub static GLOBAL_SERVER_CONFIG: LazyLock<RwLock<Option<Config>>> = LazyLock::new(|| RwLock::new(None));
pub static GLOBAL_CONFIG_SYS: LazyLock<ConfigSys> = LazyLock::new(ConfigSys::new);
@@ -100,158 +99,6 @@ pub async fn try_migrate_server_config(api: Arc<ECStore>) {
com::try_migrate_server_config(api).await
}
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct KV {
pub key: String,
pub value: String,
#[serde(default, alias = "hiddenIfEmpty")]
pub hidden_if_empty: bool,
}
#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct KVS(pub Vec<KV>);
impl Default for KVS {
fn default() -> Self {
Self::new()
}
}
impl KVS {
pub fn new() -> Self {
KVS(Vec::new())
}
pub fn get(&self, key: &str) -> String {
if let Some(v) = self.lookup(key) { v } else { "".to_owned() }
}
pub fn lookup(&self, key: &str) -> Option<String> {
for kv in self.0.iter() {
if kv.key.as_str() == key {
return Some(kv.value.clone());
}
}
None
}
///Check if KVS is empty.
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
/// Returns a list of all keys for the current KVS.
/// If the "comment" key does not exist, it will be added.
pub fn keys(&self) -> Vec<String> {
let mut found_comment = false;
let mut keys: Vec<String> = self
.0
.iter()
.map(|kv| {
if kv.key == COMMENT_KEY {
found_comment = true;
}
kv.key.clone()
})
.collect();
if !found_comment {
keys.push(COMMENT_KEY.to_owned());
}
keys
}
/// Insert or update a pair of key/values in KVS
pub fn insert(&mut self, key: String, value: String) {
for kv in self.0.iter_mut() {
if kv.key == key {
kv.value = value;
return;
}
}
self.0.push(KV {
key,
value,
hidden_if_empty: false,
});
}
/// Merge all entries from another KVS to the current instance
pub fn extend(&mut self, other: KVS) {
for KV { key, value, .. } in other.0.into_iter() {
self.insert(key, value);
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Config(pub HashMap<String, HashMap<String, KVS>>);
impl Default for Config {
fn default() -> Self {
Self::new()
}
}
impl Config {
pub fn new() -> Self {
let mut cfg = Config(HashMap::new());
cfg.set_defaults();
cfg
}
pub fn get_value(&self, sub_sys: &str, key: &str) -> Option<KVS> {
if let Some(m) = self.0.get(sub_sys) {
m.get(key).cloned()
} else {
None
}
}
pub fn set_defaults(&mut self) {
if let Some(defaults) = DEFAULT_KVS.get() {
for (k, v) in defaults.iter() {
if !self.0.contains_key(k) {
let mut default = HashMap::new();
default.insert(DEFAULT_DELIMITER.to_owned(), v.clone());
self.0.insert(k.clone(), default);
} else if !self.0[k].contains_key(DEFAULT_DELIMITER)
&& let Some(m) = self.0.get_mut(k)
{
m.insert(DEFAULT_DELIMITER.to_owned(), v.clone());
}
}
}
}
pub fn unmarshal(data: &[u8]) -> Result<Config> {
let m: HashMap<String, HashMap<String, KVS>> = serde_json::from_slice(data)?;
let mut cfg = Config(m);
cfg.set_defaults();
Ok(cfg)
}
pub fn marshal(&self) -> Result<Vec<u8>> {
let data = serde_json::to_vec(&self.0)?;
Ok(data)
}
pub fn merge(&self) -> Config {
// TODO: merge default
self.clone()
}
}
pub fn register_default_kvs(kvs: HashMap<String, KVS>) {
let mut p = HashMap::new();
for (k, v) in kvs {
p.insert(k, v);
}
let _ = DEFAULT_KVS.set(p);
}
pub fn init() {
let mut kvs = HashMap::new();
// Load storageclass default configuration
@@ -328,4 +175,23 @@ mod tests {
assert_eq!(heal_kvs.get(HEAL_BITROT_CYCLE), DEFAULT_HEAL_BITROT_CYCLE_SECS.to_string());
}
#[test]
fn old_config_model_path_reexports_moved_types() {
let mut kvs = crate::config::KVS::new();
kvs.insert("key".to_string(), "value".to_string());
let cfg = crate::config::Config(HashMap::from([(
"subsys".to_string(),
HashMap::from([(DEFAULT_DELIMITER.to_string(), kvs)]),
)]));
let moved_cfg: rustfs_config::server_config::Config = cfg;
assert_eq!(
moved_cfg
.get_value("subsys", DEFAULT_DELIMITER)
.expect("subsys should exist")
.get("key"),
"value"
);
}
}