mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 03:46:37 +00:00
refactor: Restructure project layout and clean up dependencies (#30)
This commit introduces a significant reorganization of the project structure to improve maintainability and clarity. Key changes include: - Adjusted the directory layout for a more logical module organization. - Removed unused crate dependencies, reducing the overall project size and potentially speeding up build times. - Updated import paths and configuration files to reflect the structural changes.
This commit is contained in:
houseme
@@ -0,0 +1,230 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::{Config, GLOBAL_StorageClass, storageclass};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
|
||||
use http::HeaderMap;
|
||||
use lazy_static::lazy_static;
|
||||
use rustfs_config::DEFAULT_DELIMITER;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
use tracing::{error, warn};
|
||||
|
||||
pub const CONFIG_PREFIX: &str = "config";
|
||||
const CONFIG_FILE: &str = "config.json";
|
||||
|
||||
pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
|
||||
|
||||
lazy_static! {
|
||||
static ref CONFIG_BUCKET: String = format!("{}{}{}", RUSTFS_META_BUCKET, SLASH_SEPARATOR, CONFIG_PREFIX);
|
||||
static ref SubSystemsDynamic: HashSet<String> = {
|
||||
let mut h = HashSet::new();
|
||||
h.insert(STORAGE_CLASS_SUB_SYS.to_owned());
|
||||
h
|
||||
};
|
||||
}
|
||||
pub async fn read_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<Vec<u8>> {
|
||||
let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub async fn read_config_with_metadata<S: StorageAPI>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)> {
|
||||
let h = HeaderMap::new();
|
||||
let mut rd = api
|
||||
.get_object_reader(RUSTFS_META_BUCKET, file, None, h, opts)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) {
|
||||
Error::ConfigNotFound
|
||||
} else {
|
||||
warn!("read_config_with_metadata: err: {:?}, file: {}", err, file);
|
||||
err
|
||||
}
|
||||
})?;
|
||||
|
||||
let data = rd.read_all().await?;
|
||||
|
||||
if data.is_empty() {
|
||||
return Err(Error::ConfigNotFound);
|
||||
}
|
||||
|
||||
Ok((data, rd.object_info))
|
||||
}
|
||||
|
||||
pub async fn save_config<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>) -> Result<()> {
|
||||
save_config_with_opts(
|
||||
api,
|
||||
file,
|
||||
data,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn delete_config<S: StorageAPI>(api: Arc<S>, file: &str) -> Result<()> {
|
||||
match api
|
||||
.delete_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
file,
|
||||
ObjectOptions {
|
||||
delete_prefix: true,
|
||||
delete_prefix_object: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) => {
|
||||
if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) {
|
||||
Err(Error::ConfigNotFound)
|
||||
} else {
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn save_config_with_opts<S: StorageAPI>(api: Arc<S>, file: &str, data: Vec<u8>, opts: &ObjectOptions) -> Result<()> {
|
||||
if let Err(err) = api
|
||||
.put_object(RUSTFS_META_BUCKET, file, &mut PutObjReader::from_vec(data), opts)
|
||||
.await
|
||||
{
|
||||
error!("save_config_with_opts: err: {:?}, file: {}", err, file);
|
||||
return Err(err);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn new_server_config() -> Config {
|
||||
Config::new()
|
||||
}
|
||||
|
||||
async fn new_and_save_server_config<S: StorageAPI>(api: Arc<S>) -> Result<Config> {
|
||||
let mut cfg = new_server_config();
|
||||
lookup_configs(&mut cfg, api.clone()).await;
|
||||
save_server_config(api, &cfg).await?;
|
||||
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
fn get_config_file() -> String {
|
||||
format!("{CONFIG_PREFIX}{SLASH_SEPARATOR}{CONFIG_FILE}")
|
||||
}
|
||||
|
||||
/// Handle the situation where the configuration file does not exist, create and save a new configuration
|
||||
async fn handle_missing_config<S: StorageAPI>(api: Arc<S>, context: &str) -> Result<Config> {
|
||||
warn!("Configuration not found ({}): Start initializing new configuration", context);
|
||||
let cfg = new_and_save_server_config(api).await?;
|
||||
warn!("Configuration initialization complete ({})", context);
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
/// Handle configuration file read errors
|
||||
fn handle_config_read_error(err: Error, file_path: &str) -> Result<Config> {
|
||||
error!("Read configuration failed (path: '{}'): {:?}", file_path, err);
|
||||
Err(err)
|
||||
}
|
||||
|
||||
pub async fn read_config_without_migrate<S: StorageAPI>(api: Arc<S>) -> Result<Config> {
|
||||
let config_file = get_config_file();
|
||||
|
||||
// Try to read the configuration file
|
||||
match read_config(api.clone(), &config_file).await {
|
||||
Ok(data) => read_server_config(api, &data).await,
|
||||
Err(Error::ConfigNotFound) => handle_missing_config(api, "Read the main configuration").await,
|
||||
Err(err) => handle_config_read_error(err, &config_file),
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_server_config<S: StorageAPI>(api: Arc<S>, data: &[u8]) -> Result<Config> {
|
||||
// If the provided data is empty, try to read from the file again
|
||||
if data.is_empty() {
|
||||
let config_file = get_config_file();
|
||||
warn!("Received empty configuration data, try to reread from '{}'", config_file);
|
||||
|
||||
// Try to read the configuration again
|
||||
match read_config(api.clone(), &config_file).await {
|
||||
Ok(cfg_data) => {
|
||||
// TODO: decrypt
|
||||
let cfg = Config::unmarshal(&cfg_data)?;
|
||||
return Ok(cfg.merge());
|
||||
}
|
||||
Err(Error::ConfigNotFound) => return handle_missing_config(api, "Read alternate configuration").await,
|
||||
Err(err) => return handle_config_read_error(err, &config_file),
|
||||
}
|
||||
}
|
||||
|
||||
// Process non-empty configuration data
|
||||
let cfg = Config::unmarshal(data)?;
|
||||
Ok(cfg.merge())
|
||||
}
|
||||
|
||||
pub async fn save_server_config<S: StorageAPI>(api: Arc<S>, cfg: &Config) -> Result<()> {
|
||||
let data = cfg.marshal()?;
|
||||
|
||||
let config_file = get_config_file();
|
||||
|
||||
save_config(api, &config_file, data).await
|
||||
}
|
||||
|
||||
pub async fn lookup_configs<S: StorageAPI>(cfg: &mut Config, api: Arc<S>) {
|
||||
// TODO: from etcd
|
||||
if let Err(err) = apply_dynamic_config(cfg, api).await {
|
||||
error!("apply_dynamic_config err {:?}", &err);
|
||||
}
|
||||
}
|
||||
|
||||
async fn apply_dynamic_config<S: StorageAPI>(cfg: &mut Config, api: Arc<S>) -> Result<()> {
|
||||
for key in SubSystemsDynamic.iter() {
|
||||
apply_dynamic_config_for_sub_sys(cfg, api.clone(), key).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn apply_dynamic_config_for_sub_sys<S: StorageAPI>(cfg: &mut Config, api: Arc<S>, subsys: &str) -> Result<()> {
|
||||
let set_drive_counts = api.set_drive_counts();
|
||||
if subsys == STORAGE_CLASS_SUB_SYS {
|
||||
let kvs = cfg.get_value(STORAGE_CLASS_SUB_SYS, DEFAULT_DELIMITER).unwrap_or_default();
|
||||
|
||||
for (i, count) in set_drive_counts.iter().enumerate() {
|
||||
match storageclass::lookup_config(&kvs, *count) {
|
||||
Ok(res) => {
|
||||
if i == 0 && GLOBAL_StorageClass.get().is_none() {
|
||||
if let Err(r) = GLOBAL_StorageClass.set(res) {
|
||||
error!("GLOBAL_StorageClass.set failed {:?}", r);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
error!("init storage class err:{:?}", &err);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use rustfs_utils::string::parse_bool;
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Config {
|
||||
pub bitrot: String,
|
||||
pub sleep: Duration,
|
||||
pub io_count: usize,
|
||||
pub drive_workers: usize,
|
||||
pub cache: Duration,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn bitrot_scan_cycle(&self) -> Duration {
|
||||
self.cache
|
||||
}
|
||||
|
||||
pub fn get_workers(&self) -> usize {
|
||||
self.drive_workers
|
||||
}
|
||||
|
||||
pub fn update(&mut self, nopts: &Config) {
|
||||
self.bitrot = nopts.bitrot.clone();
|
||||
self.io_count = nopts.io_count;
|
||||
self.sleep = nopts.sleep;
|
||||
self.drive_workers = nopts.drive_workers;
|
||||
}
|
||||
}
|
||||
|
||||
const RUSTFS_BITROT_CYCLE_IN_MONTHS: u64 = 1;
|
||||
|
||||
fn parse_bitrot_config(s: &str) -> Result<Duration> {
|
||||
match parse_bool(s) {
|
||||
Ok(enabled) => {
|
||||
if enabled {
|
||||
Ok(Duration::from_secs_f64(0.0))
|
||||
} else {
|
||||
Ok(Duration::from_secs_f64(-1.0))
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
if !s.ends_with("m") {
|
||||
return Err(Error::other("unknown format"));
|
||||
}
|
||||
|
||||
match s.trim_end_matches('m').parse::<u64>() {
|
||||
Ok(months) => {
|
||||
if months < RUSTFS_BITROT_CYCLE_IN_MONTHS {
|
||||
return Err(Error::other(format!("minimum bitrot cycle is {RUSTFS_BITROT_CYCLE_IN_MONTHS} month(s)")));
|
||||
}
|
||||
|
||||
Ok(Duration::from_secs(months * 30 * 24 * 60))
|
||||
}
|
||||
Err(err) => Err(Error::other(err)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod com;
|
||||
#[allow(dead_code)]
|
||||
pub mod heal;
|
||||
mod notify;
|
||||
pub mod storageclass;
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::store::ECStore;
|
||||
use com::{STORAGE_CLASS_SUB_SYS, lookup_configs, read_config_without_migrate};
|
||||
use lazy_static::lazy_static;
|
||||
use rustfs_config::DEFAULT_DELIMITER;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
|
||||
lazy_static! {
|
||||
pub static ref GLOBAL_StorageClass: OnceLock<storageclass::Config> = OnceLock::new();
|
||||
pub static ref DefaultKVS: OnceLock<HashMap<String, KVS>> = OnceLock::new();
|
||||
pub static ref GLOBAL_ServerConfig: OnceLock<Config> = OnceLock::new();
|
||||
pub static ref GLOBAL_ConfigSys: ConfigSys = ConfigSys::new();
|
||||
}
|
||||
|
||||
/// Standard config keys and values.
|
||||
pub const ENABLE_KEY: &str = "enable";
|
||||
pub const COMMENT_KEY: &str = "comment";
|
||||
|
||||
/// Enable values
|
||||
pub const ENABLE_ON: &str = "on";
|
||||
pub const ENABLE_OFF: &str = "off";
|
||||
|
||||
pub const ENV_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
|
||||
pub const ENV_SECRET_KEY: &str = "RUSTFS_SECRET_KEY";
|
||||
pub const ENV_ROOT_USER: &str = "RUSTFS_ROOT_USER";
|
||||
pub const ENV_ROOT_PASSWORD: &str = "RUSTFS_ROOT_PASSWORD";
|
||||
|
||||
pub static RUSTFS_CONFIG_PREFIX: &str = "config";
|
||||
|
||||
pub struct ConfigSys {}
|
||||
|
||||
impl Default for ConfigSys {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl ConfigSys {
|
||||
pub fn new() -> Self {
|
||||
Self {}
|
||||
}
|
||||
pub async fn init(&self, api: Arc<ECStore>) -> Result<()> {
|
||||
let mut cfg = read_config_without_migrate(api.clone().clone()).await?;
|
||||
|
||||
lookup_configs(&mut cfg, api).await;
|
||||
|
||||
let _ = GLOBAL_ServerConfig.set(cfg);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct KV {
|
||||
pub key: String,
|
||||
pub value: String,
|
||||
pub hidden_if_empty: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct KVS(pub Vec<KV>);
|
||||
|
||||
impl Default for KVS {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl KVS {
|
||||
pub fn new() -> Self {
|
||||
KVS(Vec::new())
|
||||
}
|
||||
pub fn get(&self, key: &str) -> String {
|
||||
if let Some(v) = self.lookup(key) { v } else { "".to_owned() }
|
||||
}
|
||||
pub fn lookup(&self, key: &str) -> Option<String> {
|
||||
for kv in self.0.iter() {
|
||||
if kv.key.as_str() == key {
|
||||
return Some(kv.value.clone());
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
///Check if KVS is empty.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.0.is_empty()
|
||||
}
|
||||
|
||||
/// Returns a list of all keys for the current KVS.
|
||||
/// If the "comment" key does not exist, it will be added.
|
||||
pub fn keys(&self) -> Vec<String> {
|
||||
let mut found_comment = false;
|
||||
let mut keys: Vec<String> = self
|
||||
.0
|
||||
.iter()
|
||||
.map(|kv| {
|
||||
if kv.key == COMMENT_KEY {
|
||||
found_comment = true;
|
||||
}
|
||||
kv.key.clone()
|
||||
})
|
||||
.collect();
|
||||
|
||||
if !found_comment {
|
||||
keys.push(COMMENT_KEY.to_owned());
|
||||
}
|
||||
|
||||
keys
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Config(pub HashMap<String, HashMap<String, KVS>>);
|
||||
|
||||
impl Default for Config {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn new() -> Self {
|
||||
let mut cfg = Config(HashMap::new());
|
||||
cfg.set_defaults();
|
||||
|
||||
cfg
|
||||
}
|
||||
|
||||
pub fn get_value(&self, sub_sys: &str, key: &str) -> Option<KVS> {
|
||||
if let Some(m) = self.0.get(sub_sys) {
|
||||
m.get(key).cloned()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_defaults(&mut self) {
|
||||
if let Some(defaults) = DefaultKVS.get() {
|
||||
for (k, v) in defaults.iter() {
|
||||
if !self.0.contains_key(k) {
|
||||
let mut default = HashMap::new();
|
||||
default.insert(DEFAULT_DELIMITER.to_owned(), v.clone());
|
||||
self.0.insert(k.clone(), default);
|
||||
} else if !self.0[k].contains_key(DEFAULT_DELIMITER) {
|
||||
if let Some(m) = self.0.get_mut(k) {
|
||||
m.insert(DEFAULT_DELIMITER.to_owned(), v.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn unmarshal(data: &[u8]) -> Result<Config> {
|
||||
let m: HashMap<String, HashMap<String, KVS>> = serde_json::from_slice(data)?;
|
||||
let mut cfg = Config(m);
|
||||
cfg.set_defaults();
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
pub fn marshal(&self) -> Result<Vec<u8>> {
|
||||
let data = serde_json::to_vec(&self.0)?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub fn merge(&self) -> Config {
|
||||
// TODO: merge default
|
||||
self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn register_default_kvs(kvs: HashMap<String, KVS>) {
|
||||
let mut p = HashMap::new();
|
||||
for (k, v) in kvs {
|
||||
p.insert(k, v);
|
||||
}
|
||||
|
||||
let _ = DefaultKVS.set(p);
|
||||
}
|
||||
|
||||
pub fn init() {
|
||||
let mut kvs = HashMap::new();
|
||||
// Load storageclass default configuration
|
||||
kvs.insert(STORAGE_CLASS_SUB_SYS.to_owned(), storageclass::DefaultKVS.clone());
|
||||
// New: Loading default configurations for notify_webhook and notify_mqtt
|
||||
// Referring subsystem names through constants to improve the readability and maintainability of the code
|
||||
kvs.insert(
|
||||
rustfs_config::notify::NOTIFY_WEBHOOK_SUB_SYS.to_owned(),
|
||||
notify::DefaultWebhookKVS.clone(),
|
||||
);
|
||||
kvs.insert(rustfs_config::notify::NOTIFY_MQTT_SUB_SYS.to_owned(), notify::DefaultMqttKVS.clone());
|
||||
|
||||
// Register all default configurations
|
||||
register_default_kvs(kvs)
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::config::{ENABLE_KEY, ENABLE_OFF, KV, KVS};
|
||||
use lazy_static::lazy_static;
|
||||
use rustfs_config::notify::{
|
||||
DEFAULT_DIR, DEFAULT_LIMIT, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT,
|
||||
MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
|
||||
WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
|
||||
};
|
||||
|
||||
lazy_static! {
|
||||
/// The default configuration collection of webhooks，
|
||||
/// Use lazy_static! to ensure that these configurations are initialized only once during the program life cycle, enabling high-performance lazy loading.
|
||||
pub static ref DefaultWebhookKVS: KVS = KVS(vec![
|
||||
KV { key: ENABLE_KEY.to_owned(), value: ENABLE_OFF.to_owned(), hidden_if_empty: false },
|
||||
KV { key: WEBHOOK_ENDPOINT.to_owned(), value: "".to_owned(), hidden_if_empty: false },
|
||||
// Sensitive information such as authentication tokens is hidden when the value is empty, enhancing security
|
||||
KV { key: WEBHOOK_AUTH_TOKEN.to_owned(), value: "".to_owned(), hidden_if_empty: true },
|
||||
KV { key: WEBHOOK_QUEUE_LIMIT.to_owned(), value: DEFAULT_LIMIT.to_string().to_owned(), hidden_if_empty: false },
|
||||
KV { key: WEBHOOK_QUEUE_DIR.to_owned(), value: DEFAULT_DIR.to_owned(), hidden_if_empty: false },
|
||||
KV { key: WEBHOOK_CLIENT_CERT.to_owned(), value: "".to_owned(), hidden_if_empty: false },
|
||||
KV { key: WEBHOOK_CLIENT_KEY.to_owned(), value: "".to_owned(), hidden_if_empty: false },
|
||||
]);
|
||||
|
||||
/// MQTT's default configuration collection
|
||||
pub static ref DefaultMqttKVS: KVS = KVS(vec![
|
||||
KV { key: ENABLE_KEY.to_owned(), value: ENABLE_OFF.to_owned(), hidden_if_empty: false },
|
||||
KV { key: MQTT_BROKER.to_owned(), value: "".to_owned(), hidden_if_empty: false },
|
||||
KV { key: MQTT_TOPIC.to_owned(), value: "".to_owned(), hidden_if_empty: false },
|
||||
// Sensitive information such as passwords are hidden when the value is empty
|
||||
KV { key: MQTT_PASSWORD.to_owned(), value: "".to_owned(), hidden_if_empty: true },
|
||||
KV { key: MQTT_USERNAME.to_owned(), value: "".to_owned(), hidden_if_empty: false },
|
||||
KV { key: MQTT_QOS.to_owned(), value: "0".to_owned(), hidden_if_empty: false },
|
||||
KV { key: MQTT_KEEP_ALIVE_INTERVAL.to_owned(), value: "0s".to_owned(), hidden_if_empty: false },
|
||||
KV { key: MQTT_RECONNECT_INTERVAL.to_owned(), value: "0s".to_owned(), hidden_if_empty: false },
|
||||
KV { key: MQTT_QUEUE_DIR.to_owned(), value: DEFAULT_DIR.to_owned(), hidden_if_empty: false },
|
||||
KV { key: MQTT_QUEUE_LIMIT.to_owned(), value: DEFAULT_LIMIT.to_string().to_owned(), hidden_if_empty: false },
|
||||
]);
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::KVS;
|
||||
use crate::config::KV;
|
||||
use crate::error::{Error, Result};
|
||||
use lazy_static::lazy_static;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::env;
|
||||
use tracing::warn;
|
||||
|
||||
/// Default parity count for a given drive count
|
||||
/// The default configuration allocates the number of check disks based on the total number of disks
|
||||
pub fn default_parity_count(drive: usize) -> usize {
|
||||
match drive {
|
||||
1 => 0,
|
||||
2 | 3 => 1,
|
||||
4 | 5 => 2,
|
||||
6 | 7 => 3,
|
||||
_ => 4,
|
||||
}
|
||||
}
|
||||
|
||||
// Standard constants for all storage class
|
||||
pub const RRS: &str = "REDUCED_REDUNDANCY";
|
||||
pub const STANDARD: &str = "STANDARD";
|
||||
|
||||
// Standard constants for config info storage class
|
||||
pub const CLASS_STANDARD: &str = "standard";
|
||||
pub const CLASS_RRS: &str = "rrs";
|
||||
pub const OPTIMIZE: &str = "optimize";
|
||||
pub const INLINE_BLOCK: &str = "inline_block";
|
||||
|
||||
// Reduced redundancy storage class environment variable
|
||||
pub const RRS_ENV: &str = "RUSTFS_STORAGE_CLASS_RRS";
|
||||
// Standard storage class environment variable
|
||||
pub const STANDARD_ENV: &str = "RUSTFS_STORAGE_CLASS_STANDARD";
|
||||
// Optimize storage class environment variable
|
||||
pub const OPTIMIZE_ENV: &str = "RUSTFS_STORAGE_CLASS_OPTIMIZE";
|
||||
// Inline block indicates the size of the shard that is considered for inlining
|
||||
pub const INLINE_BLOCK_ENV: &str = "RUSTFS_STORAGE_CLASS_INLINE_BLOCK";
|
||||
|
||||
// Supported storage class scheme is EC
|
||||
pub const SCHEME_PREFIX: &str = "EC";
|
||||
|
||||
// Min parity drives
|
||||
pub const MIN_PARITY_DRIVES: usize = 0;
|
||||
|
||||
// Default RRS parity is always minimum parity.
|
||||
pub const DEFAULT_RRS_PARITY: usize = 1;
|
||||
|
||||
pub static DEFAULT_INLINE_BLOCK: usize = 128 * 1024;
|
||||
|
||||
lazy_static! {
|
||||
pub static ref DefaultKVS: KVS = {
|
||||
let kvs = vec![
|
||||
KV {
|
||||
key: CLASS_STANDARD.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: CLASS_RRS.to_owned(),
|
||||
value: "EC:1".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: OPTIMIZE.to_owned(),
|
||||
value: "availability".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: INLINE_BLOCK.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
];
|
||||
|
||||
KVS(kvs)
|
||||
};
|
||||
}
|
||||
|
||||
// StorageClass - holds storage class information
|
||||
#[derive(Serialize, Deserialize, Debug, Default)]
|
||||
pub struct StorageClass {
|
||||
parity: usize,
|
||||
}
|
||||
|
||||
// Config storage class configuration
|
||||
#[derive(Serialize, Deserialize, Debug, Default)]
|
||||
pub struct Config {
|
||||
standard: StorageClass,
|
||||
rrs: StorageClass,
|
||||
optimize: Option<String>,
|
||||
inline_block: usize,
|
||||
initialized: bool,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn get_parity_for_sc(&self, sc: &str) -> Option<usize> {
|
||||
match sc.trim() {
|
||||
RRS => {
|
||||
if self.initialized {
|
||||
Some(self.rrs.parity)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
if self.initialized {
|
||||
Some(self.standard.parity)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn should_inline(&self, shard_size: i64, versioned: bool) -> bool {
|
||||
if shard_size < 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let shard_size = shard_size as usize;
|
||||
|
||||
let mut inline_block = DEFAULT_INLINE_BLOCK;
|
||||
if self.initialized {
|
||||
inline_block = self.inline_block;
|
||||
}
|
||||
|
||||
if versioned {
|
||||
shard_size <= inline_block / 8
|
||||
} else {
|
||||
shard_size <= inline_block
|
||||
}
|
||||
}
|
||||
|
||||
pub fn inline_block(&self) -> usize {
|
||||
if !self.initialized {
|
||||
DEFAULT_INLINE_BLOCK
|
||||
} else {
|
||||
self.inline_block
|
||||
}
|
||||
}
|
||||
|
||||
pub fn capacity_optimized(&self) -> bool {
|
||||
if !self.initialized {
|
||||
false
|
||||
} else {
|
||||
self.optimize.as_ref().is_some_and(|v| v.as_str() == "capacity")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn lookup_config(kvs: &KVS, set_drive_count: usize) -> Result<Config> {
|
||||
let standard = {
|
||||
let ssc_str = {
|
||||
if let Ok(ssc_str) = env::var(STANDARD_ENV) {
|
||||
ssc_str
|
||||
} else {
|
||||
kvs.get(CLASS_STANDARD)
|
||||
}
|
||||
};
|
||||
|
||||
if !ssc_str.is_empty() {
|
||||
parse_storage_class(&ssc_str)?
|
||||
} else {
|
||||
StorageClass {
|
||||
parity: default_parity_count(set_drive_count),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let rrs = {
|
||||
let ssc_str = {
|
||||
if let Ok(ssc_str) = env::var(RRS_ENV) {
|
||||
ssc_str
|
||||
} else {
|
||||
kvs.get(RRS)
|
||||
}
|
||||
};
|
||||
|
||||
if !ssc_str.is_empty() {
|
||||
parse_storage_class(&ssc_str)?
|
||||
} else {
|
||||
StorageClass {
|
||||
parity: { if set_drive_count == 1 { 0 } else { DEFAULT_RRS_PARITY } },
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
validate_parity_inner(standard.parity, rrs.parity, set_drive_count)?;
|
||||
|
||||
let optimize = { env::var(OPTIMIZE_ENV).ok() };
|
||||
|
||||
let inline_block = {
|
||||
if let Ok(ev) = env::var(INLINE_BLOCK_ENV) {
|
||||
if let Ok(block) = ev.parse::<bytesize::ByteSize>() {
|
||||
if block.as_u64() as usize > DEFAULT_INLINE_BLOCK {
|
||||
warn!(
|
||||
"inline block value bigger than recommended max of 128KiB -> {}, performance may degrade for PUT please benchmark the changes",
|
||||
block
|
||||
);
|
||||
}
|
||||
block.as_u64() as usize
|
||||
} else {
|
||||
return Err(Error::other(format!("parse {INLINE_BLOCK_ENV} format failed")));
|
||||
}
|
||||
} else {
|
||||
DEFAULT_INLINE_BLOCK
|
||||
}
|
||||
};
|
||||
|
||||
Ok(Config {
|
||||
standard,
|
||||
rrs,
|
||||
optimize,
|
||||
inline_block,
|
||||
initialized: true,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_storage_class(env: &str) -> Result<StorageClass> {
|
||||
let s: Vec<&str> = env.split(':').collect();
|
||||
|
||||
// only two elements allowed in the string - "scheme" and "number of parity drives"
|
||||
if s.len() != 2 {
|
||||
return Err(Error::other(format!(
|
||||
"Invalid storage class format: {env}. Expected 'Scheme:Number of parity drives'."
|
||||
)));
|
||||
}
|
||||
|
||||
// only allowed scheme is "EC"
|
||||
if s[0] != SCHEME_PREFIX {
|
||||
return Err(Error::other(format!("Unsupported scheme {}. Supported scheme is EC.", s[0])));
|
||||
}
|
||||
|
||||
// Number of parity drives should be integer
|
||||
let parity_drives: usize = match s[1].parse() {
|
||||
Ok(num) => num,
|
||||
Err(_) => return Err(Error::other(format!("Failed to parse parity value: {}.", s[1]))),
|
||||
};
|
||||
|
||||
Ok(StorageClass { parity: parity_drives })
|
||||
}
|
||||
|
||||
// ValidateParity validates standard storage class parity.
|
||||
pub fn validate_parity(ss_parity: usize, set_drive_count: usize) -> Result<()> {
|
||||
// if ss_parity > 0 && ss_parity < MIN_PARITY_DRIVES {
|
||||
// return Err(Error::other(format!(
|
||||
// "parity {} should be greater than or equal to {}",
|
||||
// ss_parity, MIN_PARITY_DRIVES
|
||||
// )));
|
||||
// }
|
||||
|
||||
if ss_parity > set_drive_count / 2 {
|
||||
return Err(Error::other(format!(
|
||||
"parity {} should be less than or equal to {}",
|
||||
ss_parity,
|
||||
set_drive_count / 2
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Validates the parity drives.
|
||||
pub fn validate_parity_inner(ss_parity: usize, rrs_parity: usize, set_drive_count: usize) -> Result<()> {
|
||||
// if ss_parity > 0 && ss_parity < MIN_PARITY_DRIVES {
|
||||
// return Err(Error::other(format!(
|
||||
// "Standard storage class parity {} should be greater than or equal to {}",
|
||||
// ss_parity, MIN_PARITY_DRIVES
|
||||
// )));
|
||||
// }
|
||||
|
||||
// RRS parity drives should be greater than or equal to minParityDrives.
|
||||
// Parity below minParityDrives is not supported.
|
||||
// if rrs_parity > 0 && rrs_parity < MIN_PARITY_DRIVES {
|
||||
// return Err(Error::other(format!(
|
||||
// "Reduced redundancy storage class parity {} should be greater than or equal to {}",
|
||||
// rrs_parity, MIN_PARITY_DRIVES
|
||||
// )));
|
||||
// }
|
||||
|
||||
if set_drive_count > 2 {
|
||||
if ss_parity > set_drive_count / 2 {
|
||||
return Err(Error::other(format!(
|
||||
"Standard storage class parity {} should be less than or equal to {}",
|
||||
ss_parity,
|
||||
set_drive_count / 2
|
||||
)));
|
||||
}
|
||||
|
||||
if rrs_parity > set_drive_count / 2 {
|
||||
return Err(Error::other(format!(
|
||||
"Reduced redundancy storage class parity {} should be less than or equal to {}",
|
||||
rrs_parity,
|
||||
set_drive_count / 2
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
if ss_parity > 0 && rrs_parity > 0 && ss_parity < rrs_parity {
|
||||
return Err(Error::other(format!(
|
||||
"Standard storage class parity drives {ss_parity} should be greater than or equal to Reduced redundancy storage class parity drives {rrs_parity}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user