merge versioning, fix bug todo

ecstore/src/config/common.rs

@@ -1,13 +1,34 @@
+use std::collections::HashSet;
+
+use super::error::ConfigError;
+use super::{storageclass, Config, GLOBAL_StorageClass, KVS};
+use crate::config::error::is_not_found;
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::{Error, Result};
 use crate::store::ECStore;
-use crate::store_api::{HTTPRangeSpec, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader};
+use crate::store_api::{HTTPRangeSpec, ObjectIO, ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
+use crate::store_err::is_err_object_not_found;
+use crate::utils::path::SLASH_SEPARATOR;
 use http::HeaderMap;
+use lazy_static::lazy_static;
 use s3s::dto::StreamingBlob;
 use s3s::Body;
+use tracing::error;
 
-use super::error::ConfigError;
+const CONFIG_PREFIX: &str = "config";
+const CONFIG_FILE: &str = "config.json";
 
+pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
+pub const DEFAULT_KV_KEY: &str = "_";
+
+lazy_static! {
+    static ref CONFIG_BUCKET: String = format!("{}{}{}", RUSTFS_META_BUCKET, SLASH_SEPARATOR, CONFIG_PREFIX);
+    static ref SubSystemsDynamic: HashSet<String> = {
+        let mut h = HashSet::new();
+        h.insert(STORAGE_CLASS_SUB_SYS.to_owned());
+        h
+    };
+}
 pub async fn read_config(api: &ECStore, file: &str) -> Result<Vec<u8>> {
     let (data, _obj) = read_config_with_metadata(api, file, &ObjectOptions::default()).await?;
 
@@ -17,7 +38,16 @@ pub async fn read_config(api: &ECStore, file: &str) -> Result<Vec<u8>> {
 async fn read_config_with_metadata(api: &ECStore, file: &str, opts: &ObjectOptions) -> Result<(Vec<u8>, ObjectInfo)> {
     let range = HTTPRangeSpec::nil();
     let h = HeaderMap::new();
-    let mut rd = api.get_object_reader(RUSTFS_META_BUCKET, file, range, h, opts).await?;
+    let mut rd = api
+        .get_object_reader(RUSTFS_META_BUCKET, file, range, h, opts)
+        .await
+        .map_err(|err| {
+            if is_err_object_not_found(&err) {
+                Error::new(ConfigError::NotFound)
+            } else {
+                err
+            }
+        })?;
 
     let data = rd.read_all().await?;
 
@@ -46,9 +76,120 @@ async fn save_config_with_opts(api: &ECStore, file: &str, data: &[u8], opts: &Ob
         .put_object(
             RUSTFS_META_BUCKET,
             file,
-            PutObjReader::new(StreamingBlob::from(Body::from(data.to_vec())), data.len()),
+            &mut PutObjReader::new(StreamingBlob::from(Body::from(data.to_vec())), data.len()),
             opts,
         )
         .await?;
     Ok(())
 }
+
+fn new_server_config() -> Config {
+    Config::new()
+}
+
+async fn new_and_save_server_config(api: &ECStore) -> Result<Config> {
+    let mut cfg = new_server_config();
+    lookup_configs(&mut cfg, api).await;
+    save_server_config(api, &cfg).await?;
+
+    Ok(cfg)
+}
+
+pub async fn read_config_without_migrate(api: &ECStore) -> Result<Config> {
+    let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, CONFIG_FILE);
+    let data = match read_config(api, config_file.as_str()).await {
+        Ok(res) => res,
+        Err(err) => {
+            if is_not_found(&err) {
+                let cfg = new_and_save_server_config(api).await?;
+                return Ok(cfg);
+            } else {
+                return Err(err);
+            }
+        }
+    };
+
+    read_server_config(api, data.as_slice()).await
+}
+
+async fn read_server_config(api: &ECStore, data: &[u8]) -> Result<Config> {
+    let cfg = {
+        if data.is_empty() {
+            let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, CONFIG_FILE);
+            let cfg_data = match read_config(api, config_file.as_str()).await {
+                Ok(res) => res,
+                Err(err) => {
+                    if is_not_found(&err) {
+                        let cfg = new_and_save_server_config(api).await?;
+                        return Ok(cfg);
+                    } else {
+                        return Err(err);
+                    }
+                }
+            };
+            // TODO: decrypt
+
+            Config::unmarshal(cfg_data.as_slice())?
+        } else {
+            Config::unmarshal(data)?
+        }
+    };
+
+    Ok(cfg.merge())
+}
+
+async fn save_server_config(api: &ECStore, cfg: &Config) -> Result<()> {
+    let data = cfg.marshal()?;
+
+    let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, CONFIG_FILE);
+
+    save_config(api, &config_file, data.as_slice()).await
+}
+
+pub async fn lookup_configs(cfg: &mut Config, api: &ECStore) {
+    // TODO: from etcd
+    if let Err(err) = apply_dynamic_config(cfg, api).await {
+        error!("apply_dynamic_config err {:?}", &err);
+    }
+}
+
+async fn apply_dynamic_config(cfg: &mut Config, api: &ECStore) -> Result<()> {
+    for key in SubSystemsDynamic.iter() {
+        apply_dynamic_config_for_sub_sys(cfg, api, key).await?;
+    }
+
+    Ok(())
+}
+
+async fn apply_dynamic_config_for_sub_sys(cfg: &mut Config, api: &ECStore, subsys: &String) -> Result<()> {
+    let set_drive_counts = api.set_drive_counts();
+    match subsys.as_str() {
+        STORAGE_CLASS_SUB_SYS => {
+            let kvs = match cfg.get_value(STORAGE_CLASS_SUB_SYS, DEFAULT_KV_KEY) {
+                Some(res) => res,
+                None => KVS::new(),
+            };
+
+            for (i, count) in set_drive_counts.iter().enumerate() {
+                match storageclass::lookup_config(&kvs, *count) {
+                    Ok(res) => {
+                        if i == 0 {
+                            if GLOBAL_StorageClass.get().is_none() {
+                                if let Err(r) = GLOBAL_StorageClass.set(res) {
+                                    error!("GLOBAL_StorageClass.set failed {:?}", r);
+                                }
+                            }
+                        }
+                    }
+                    Err(err) => {
+                        error!("init storageclass err:{:?}", &err);
+                        break;
+                    }
+                }
+            }
+        }
+        _ => {}
+    }
+
+    Ok(())
+}

ecstore/src/config/mod.rs

@@ -1,2 +1,138 @@
 pub mod common;
 pub mod error;
+pub mod storageclass;
+
+use crate::error::Result;
+use crate::store::ECStore;
+use common::{lookup_configs, read_config_without_migrate, STORAGE_CLASS_SUB_SYS};
+use lazy_static::lazy_static;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::sync::OnceLock;
+
+lazy_static! {
+    pub static ref GLOBAL_StorageClass: OnceLock<storageclass::Config> = OnceLock::new();
+    pub static ref DefaultKVS: OnceLock<HashMap<String, KVS>> = OnceLock::new();
+    pub static ref GLOBAL_ServerConfig: OnceLock<Config> = OnceLock::new();
+    pub static ref GLOBAL_ConfigSys: ConfigSys = ConfigSys::new();
+}
+
+pub struct ConfigSys {}
+
+impl ConfigSys {
+    pub fn new() -> Self {
+        Self {}
+    }
+    pub async fn init(&self, api: &ECStore) -> Result<()> {
+        let mut cfg = read_config_without_migrate(api).await?;
+
+        lookup_configs(&mut cfg, api).await;
+
+        let _ = GLOBAL_ServerConfig.set(cfg);
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, Deserialize, Serialize, Clone)]
+pub struct KV {
+    pub key: String,
+    pub value: String,
+    pub hidden_if_empty: bool,
+}
+
+#[derive(Debug, Deserialize, Serialize, Clone)]
+pub struct KVS(Vec<KV>);
+
+impl KVS {
+    pub fn new() -> Self {
+        KVS(Vec::new())
+    }
+    pub fn get(&self, key: &str) -> String {
+        if let Some(v) = self.lookup(key) {
+            v
+        } else {
+            "".to_owned()
+        }
+    }
+    pub fn lookup(&self, key: &str) -> Option<String> {
+        for kv in self.0.iter() {
+            if kv.key.as_str() == key {
+                return Some(kv.value.clone());
+            }
+        }
+
+        None
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct Config(HashMap<String, HashMap<String, KVS>>);
+
+impl Config {
+    pub fn new() -> Self {
+        let mut cfg = Config(HashMap::new());
+        cfg.set_defaults();
+
+        cfg
+    }
+
+    pub fn get_value(&self, subsys: &str, key: &str) -> Option<KVS> {
+        if let Some(m) = self.0.get(subsys) {
+            m.get(key).cloned()
+        } else {
+            None
+        }
+    }
+
+    pub fn set_defaults(&mut self) {
+        if let Some(defaults) = DefaultKVS.get() {
+            for (k, v) in defaults.iter() {
+                if !self.0.contains_key(k) {
+                    let mut default = HashMap::new();
+                    default.insert("_".to_owned(), v.clone());
+                    self.0.insert(k.clone(), default);
+                } else {
+                    if !self.0[k].contains_key("_") {
+                        if let Some(m) = self.0.get_mut(k) {
+                            m.insert("_".to_owned(), v.clone());
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    pub fn unmarshal(data: &[u8]) -> Result<Config> {
+        let m: HashMap<String, HashMap<String, KVS>> = serde_json::from_slice(data)?;
+        let mut cfg = Config(m);
+        cfg.set_defaults();
+        Ok(cfg)
+    }
+
+    pub fn marshal(&self) -> Result<Vec<u8>> {
+        let data = serde_json::to_vec(&self.0)?;
+        Ok(data)
+    }
+
+    pub fn merge(&self) -> Config {
+        // TODO: merge defauls
+        self.clone()
+    }
+}
+
+pub fn register_default_kvs(kvs: HashMap<String, KVS>) {
+    let mut p = HashMap::new();
+    for (k, v) in kvs {
+        p.insert(k, v);
+    }
+
+    let _ = DefaultKVS.set(p);
+}
+
+pub fn init() {
+    let mut kvs = HashMap::new();
+    kvs.insert(STORAGE_CLASS_SUB_SYS.to_owned(), storageclass::DefaultKVS.clone());
+    // TODO: other defauls
+    register_default_kvs(kvs)
+}

ecstore/src/config/storageclass.rs

@@ -0,0 +1,316 @@
+use std::env;
+
+use crate::{
+    config::KV,
+    error::{Error, Result},
+};
+
+use super::KVS;
+use lazy_static::lazy_static;
+use serde::{Deserialize, Serialize};
+use tracing::warn;
+
+// default_partiy_count 默认配置，根据磁盘总数分配校验磁盘数量
+pub fn default_partiy_count(drive: usize) -> usize {
+    match drive {
+        1 => 0,
+        2 | 3 => 1,
+        4 | 5 => 2,
+        6 | 7 => 3,
+        _ => 4,
+    }
+}
+
+// Standard constants for all storage class
+pub const RRS: &str = "REDUCED_REDUNDANCY";
+pub const STANDARD: &str = "STANDARD";
+
+// Standard constants for config info storage class
+pub const CLASS_STANDARD: &str = "standard";
+pub const CLASS_RRS: &str = "rrs";
+pub const OPTIMIZE: &str = "optimize";
+pub const INLINE_BLOCK: &str = "inline_block";
+
+// Reduced redundancy storage class environment variable
+pub const RRS_ENV: &str = "RUSTFS_STORAGE_CLASS_RRS";
+// Standard storage class environment variable
+pub const STANDARD_ENV: &str = "RUSTFS_STORAGE_CLASS_STANDARD";
+// Optimize storage class environment variable
+pub const OPTIMIZE_ENV: &str = "RUSTFS_STORAGE_CLASS_OPTIMIZE";
+// Inline block indicates the size of the shard that is considered for inlining
+pub const INLINE_BLOCK_ENV: &str = "RUSTFS_STORAGE_CLASS_INLINE_BLOCK";
+
+// Supported storage class scheme is EC
+pub const SCHEME_PREFIX: &str = "EC";
+
+// Min parity drives
+pub const MIN_PARITY_DRIVES: usize = 0;
+
+// Default RRS parity is always minimum parity.
+pub const DEFAULT_RRS_PARITY: usize = 1;
+
+pub static DEFAULT_INLINE_BLOCK: usize = 128 * 1024;
+
+lazy_static! {
+    pub static ref DefaultKVS: KVS = {
+        let mut kvs = Vec::new();
+
+        kvs.push(KV {
+            key: CLASS_STANDARD.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: false,
+        });
+
+        kvs.push(KV {
+            key: CLASS_RRS.to_owned(),
+            value: "EC:1".to_owned(),
+            hidden_if_empty: false,
+        });
+
+        kvs.push(KV {
+            key: OPTIMIZE.to_owned(),
+            value: "availability".to_owned(),
+            hidden_if_empty: false,
+        });
+
+        kvs.push(KV {
+            key: INLINE_BLOCK.to_owned(),
+            value: "".to_owned(),
+            hidden_if_empty: true,
+        });
+
+        KVS(kvs)
+    };
+}
+
+// StorageClass - holds storage class information
+#[derive(Serialize, Deserialize, Debug, Default)]
+pub struct StorageClass {
+    parity: usize,
+}
+
+// Config storage class configuration
+#[derive(Serialize, Deserialize, Debug, Default)]
+pub struct Config {
+    standard: StorageClass,
+    rrs: StorageClass,
+    optimize: Option<String>,
+    inline_block: usize,
+    initialized: bool,
+}
+
+impl Config {
+    pub fn get_parity_for_sc(&self, sc: &str) -> Option<usize> {
+        match sc.trim() {
+            RRS => {
+                if self.initialized {
+                    Some(self.rrs.parity)
+                } else {
+                    None
+                }
+            }
+            _ => {
+                if self.initialized {
+                    Some(self.standard.parity)
+                } else {
+                    None
+                }
+            }
+        }
+    }
+
+    pub fn should_inline(&self, shard_size: usize, versioned: bool) -> bool {
+        let mut inline_block = DEFAULT_INLINE_BLOCK;
+        if self.initialized {
+            inline_block = self.inline_block;
+        }
+
+        if versioned {
+            shard_size <= inline_block / 8
+        } else {
+            shard_size <= inline_block
+        }
+    }
+
+    pub fn inline_block(&self) -> usize {
+        if !self.initialized {
+            DEFAULT_INLINE_BLOCK
+        } else {
+            self.inline_block
+        }
+    }
+
+    pub fn capacity_optimized(&self) -> bool {
+        if !self.initialized {
+            false
+        } else {
+            self.optimize.as_ref().is_some_and(|v| v.as_str() == "capacity")
+        }
+    }
+}
+
+pub fn lookup_config(kvs: &KVS, set_drive_count: usize) -> Result<Config> {
+    let standard = {
+        let ssc_str = {
+            if let Ok(ssc_str) = env::var(STANDARD_ENV) {
+                ssc_str
+            } else {
+                kvs.get(CLASS_STANDARD)
+            }
+        };
+
+        if !ssc_str.is_empty() {
+            parse_storage_class(&ssc_str)?
+        } else {
+            StorageClass {
+                parity: default_partiy_count(set_drive_count),
+            }
+        }
+    };
+
+    let rrs = {
+        let ssc_str = {
+            if let Ok(ssc_str) = env::var(RRS_ENV) {
+                ssc_str
+            } else {
+                kvs.get(RRS)
+            }
+        };
+
+        if !ssc_str.is_empty() {
+            parse_storage_class(&ssc_str)?
+        } else {
+            StorageClass {
+                parity: {
+                    if set_drive_count == 1 {
+                        0
+                    } else {
+                        DEFAULT_RRS_PARITY
+                    }
+                },
+            }
+        }
+    };
+
+    validate_parity_inner(standard.parity, rrs.parity, set_drive_count)?;
+
+    let optimize = {
+        if let Ok(ev) = env::var(OPTIMIZE_ENV) {
+            Some(ev)
+        } else {
+            None
+        }
+    };
+
+    let inline_block = {
+        if let Ok(ev) = env::var(INLINE_BLOCK_ENV) {
+            if let Ok(block) = ev.parse::<bytesize::ByteSize>() {
+                if block.as_u64() as usize > DEFAULT_INLINE_BLOCK {
+                    warn!("inline block value bigger than recommended max of 128KiB -> {}, performance may degrade for PUT please benchmark the changes",block);
+                }
+                block.as_u64() as usize
+            } else {
+                return Err(Error::msg(format!("parse {} format failed", INLINE_BLOCK_ENV)));
+            }
+        } else {
+            DEFAULT_INLINE_BLOCK
+        }
+    };
+
+    Ok(Config {
+        standard,
+        rrs,
+        optimize,
+        inline_block,
+        initialized: true,
+    })
+}
+
+pub fn parse_storage_class(env: &str) -> Result<StorageClass> {
+    let s: Vec<&str> = env.split(':').collect();
+
+    // only two elements allowed in the string - "scheme" and "number of parity drives"
+    if s.len() != 2 {
+        return Err(Error::msg(&format!(
+            "Invalid storage class format: {}. Expected 'Scheme:Number of parity drives'.",
+            env
+        )));
+    }
+
+    // only allowed scheme is "EC"
+    if s[0] != SCHEME_PREFIX {
+        return Err(Error::msg(&format!("Unsupported scheme {}. Supported scheme is EC.", s[0])));
+    }
+
+    // Number of parity drives should be integer
+    let parity_drives: usize = match s[1].parse() {
+        Ok(num) => num,
+        Err(_) => return Err(Error::msg(&format!("Failed to parse parity value: {}.", s[1]))),
+    };
+
+    Ok(StorageClass { parity: parity_drives })
+}
+
+// ValidateParity validates standard storage class parity.
+pub fn validate_parity(ss_parity: usize, set_drive_count: usize) -> Result<()> {
+    if ss_parity > 0 && ss_parity < MIN_PARITY_DRIVES {
+        return Err(Error::msg(format!(
+            "parity {} should be greater than or equal to {}",
+            ss_parity, MIN_PARITY_DRIVES
+        )));
+    }
+
+    if ss_parity > set_drive_count / 2 {
+        return Err(Error::msg(format!(
+            "parity {} should be less than or equal to {}",
+            ss_parity,
+            set_drive_count / 2
+        )));
+    }
+
+    Ok(())
+}
+
+// Validates the parity drives.
+pub fn validate_parity_inner(ss_parity: usize, rrs_parity: usize, set_drive_count: usize) -> Result<()> {
+    if ss_parity > 0 && ss_parity < MIN_PARITY_DRIVES {
+        return Err(Error::msg(format!(
+            "Standard storage class parity {} should be greater than or equal to {}",
+            ss_parity, MIN_PARITY_DRIVES
+        )));
+    }
+
+    // RRS parity drives should be greater than or equal to minParityDrives.
+    // Parity below minParityDrives is not supported.
+    if rrs_parity > 0 && rrs_parity < MIN_PARITY_DRIVES {
+        return Err(Error::msg(format!(
+            "Reduced redundancy storage class parity {} should be greater than or equal to {}",
+            rrs_parity, MIN_PARITY_DRIVES
+        )));
+    }
+
+    if set_drive_count > 2 {
+        if ss_parity > set_drive_count / 2 {
+            return Err(Error::msg(format!(
+                "Standard storage class parity {} should be less than or equal to {}",
+                ss_parity,
+                set_drive_count / 2
+            )));
+        }
+
+        if rrs_parity > set_drive_count / 2 {
+            return Err(Error::msg(format!(
+                "Reduced redundancy storage class parity {} should be less than or equal to {}",
+                rrs_parity,
+                set_drive_count / 2
+            )));
+        }
+    }
+
+    if ss_parity > 0 && rrs_parity > 0 {
+        if ss_parity < rrs_parity {
+            return Err(Error::msg(format!("Standard storage class parity drives {} should be greater than or equal to Reduced redundancy storage class parity drives {}", ss_parity, rrs_parity)));
+        }
+    }
+    Ok(())
+}