From 28f86a505ef1611ecedb0bc88df753c28140501a Mon Sep 17 00:00:00 2001 From: houseme Date: Fri, 20 Mar 2026 01:16:45 +0800 Subject: [PATCH] feat: Add --info command and refactor config module (#2234) Signed-off-by: houseme Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- crates/common/src/readiness.rs | 17 + crates/config/src/constants/app.rs | 52 ++ crates/config/src/constants/mod.rs | 1 + crates/config/src/constants/workload.rs | 70 +++ crates/config/src/lib.rs | 2 + rustfs/src/admin/console.rs | 2 +- rustfs/src/app/context.rs | 2 +- rustfs/src/app/object_usecase.rs | 2 +- rustfs/src/config/cli.rs | 287 +++++++++++ rustfs/src/config/config_struct.rs | 205 ++++++++ rustfs/src/config/info.rs | 656 ++++++++++++++++++++++++ rustfs/src/config/mod.rs | 496 ++---------------- rustfs/src/config/opt.rs | 161 ++++++ rustfs/src/config/snapshot.rs | 164 ++++++ rustfs/src/config/workload_profiles.rs | 83 ++- rustfs/src/init.rs | 61 ++- rustfs/src/main.rs | 28 +- rustfs/src/storage/ecfs_extend.rs | 20 +- rustfs/src/storage/ecfs_test.rs | 12 +- 19 files changed, 1839 insertions(+), 482 deletions(-) create mode 100644 crates/config/src/constants/workload.rs create mode 100644 rustfs/src/config/cli.rs create mode 100644 rustfs/src/config/config_struct.rs create mode 100644 rustfs/src/config/info.rs create mode 100644 rustfs/src/config/opt.rs create mode 100644 rustfs/src/config/snapshot.rs diff --git a/crates/common/src/readiness.rs b/crates/common/src/readiness.rs index 1a0b50d39..4ab848835 100644 --- a/crates/common/src/readiness.rs +++ b/crates/common/src/readiness.rs @@ -60,6 +60,23 @@ impl GlobalReadiness { pub fn is_ready(&self) -> bool { self.status.load(Ordering::SeqCst) == SystemStage::FullReady as u8 } + + /// Get the current system stage + /// # Returns + /// The current SystemStage of the service + pub fn current_stage(&self) -> SystemStage { + match self.status.load(Ordering::SeqCst) { + 0 => SystemStage::Booting, + 1 => SystemStage::StorageReady, + 2 => SystemStage::IamReady, + 3 => SystemStage::FullReady, + invalid => { + debug_assert!(false, "GlobalReadiness::current_stage: invalid status value {}", invalid); + // Fallback to the most conservative stage on invalid values + SystemStage::Booting + } + } + } } #[cfg(test)] diff --git a/crates/config/src/constants/app.rs b/crates/config/src/constants/app.rs index 627019dfb..f0dc5727b 100644 --- a/crates/config/src/constants/app.rs +++ b/crates/config/src/constants/app.rs @@ -131,9 +131,61 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS"; /// Environment variable for server volumes. pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES"; +/// Environment variable for server access key. +pub const ENV_RUSTFS_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY"; + +/// Environment variable for server access key file. +pub const ENV_RUSTFS_ACCESS_KEY_FILE: &str = "RUSTFS_ACCESS_KEY_FILE"; + +/// Environment variable for server root user. +pub const ENV_RUSTFS_ROOT_USER: &str = "RUSTFS_ROOT_USER"; + +/// Environment variable for server secret key. +pub const ENV_RUSTFS_SECRET_KEY: &str = "RUSTFS_SECRET_KEY"; + +/// Environment variable for server secret key file. +pub const ENV_RUSTFS_SECRET_KEY_FILE: &str = "RUSTFS_SECRET_KEY_FILE"; + +/// Environment variable for server root password. +pub const ENV_RUSTFS_ROOT_PASSWORD: &str = "RUSTFS_ROOT_PASSWORD"; + +/// Environment variable for server OBS endpoint. +pub const ENV_RUSTFS_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT"; + +/// Environment variable for console server enable. +pub const ENV_RUSTFS_CONSOLE_ENABLE: &str = "RUSTFS_CONSOLE_ENABLE"; + +/// Environment variable for console server address. +pub const ENV_RUSTFS_CONSOLE_ADDRESS: &str = "RUSTFS_CONSOLE_ADDRESS"; + /// Environment variable for server tls path. pub const ENV_RUSTFS_TLS_PATH: &str = "RUSTFS_TLS_PATH"; +/// Environment variable for server KMS enable. +pub const ENV_RUSTFS_KMS_ENABLE: &str = "RUSTFS_KMS_ENABLE"; + +/// Default KMS enable for server-side encryption +/// This is the default value for enabling KMS encryption for server-side encryption. +/// Default value: false +pub const DEFAULT_KMS_ENABLE: bool = false; + +/// Environment variable for server KMS backend. +pub const ENV_RUSTFS_KMS_BACKEND: &str = "RUSTFS_KMS_BACKEND"; + +/// Default KMS backend for server-side encryption +/// This is the default KMS backend for server-side encryption. +/// Default value: local +pub const DEFAULT_KMS_BACKEND: &str = "local"; + +/// Environment variable for selecting the buffer profile used for adaptive buffer sizing. +pub const ENV_RUSTFS_BUFFER_PROFILE: &str = "RUSTFS_BUFFER_PROFILE"; + +/// Default buffer profile for adaptive buffer sizing +/// This is the default buffer profile for adaptive buffer sizing. +/// It is used to identify the workload profile for adaptive buffer sizing. +/// Default value: GeneralPurpose +pub const DEFAULT_BUFFER_PROFILE: &str = "GeneralPurpose"; + /// Default value for the server TLS path if `ENV_RUSTFS_TLS_PATH` is not set. pub const DEFAULT_RUSTFS_TLS_PATH: &str = ""; diff --git a/crates/config/src/constants/mod.rs b/crates/config/src/constants/mod.rs index 6176f3e98..b5ce800fa 100644 --- a/crates/config/src/constants/mod.rs +++ b/crates/config/src/constants/mod.rs @@ -28,3 +28,4 @@ pub(crate) mod runtime; pub(crate) mod scanner; pub(crate) mod targets; pub(crate) mod tls; +pub(crate) mod workload; diff --git a/crates/config/src/constants/workload.rs b/crates/config/src/constants/workload.rs new file mode 100644 index 000000000..27d618d19 --- /dev/null +++ b/crates/config/src/constants/workload.rs @@ -0,0 +1,70 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Workload profile buffer configuration constants. +//! +//! This module defines environment variable keys and default values for +//! custom buffer profile configuration. + +use crate::{KI_B, MI_B}; + +/// Environment variable for minimum buffer size +/// Default: 64KB (65536 bytes) +pub const ENV_RUSTFS_BUFFER_MIN_SIZE: &str = "RUSTFS_BUFFER_MIN_SIZE"; + +/// Environment variable for maximum buffer size +/// Default: 1MB (1048576 bytes) +pub const ENV_RUSTFS_BUFFER_MAX_SIZE: &str = "RUSTFS_BUFFER_MAX_SIZE"; + +/// Environment variable for default buffer size (used when file size is unknown) +/// Default: 256KB (262144 bytes) +pub const ENV_RUSTFS_BUFFER_DEFAULT_SIZE: &str = "RUSTFS_BUFFER_DEFAULT_SIZE"; + +/// Default minimum buffer size: 64KB +pub const DEFAULT_BUFFER_MIN_SIZE: usize = 64 * KI_B; + +/// Default maximum buffer size: 1MB +pub const DEFAULT_BUFFER_MAX_SIZE: usize = MI_B; + +/// Default buffer size for unknown file size: 256KB +pub const DEFAULT_BUFFER_UNKNOWN_SIZE: usize = 256 * KI_B; + +#[cfg(test)] +mod tests { + use super::*; + use crate::MI_B; + + #[test] + fn test_default_values() { + assert_eq!(DEFAULT_BUFFER_MIN_SIZE, 65536); // 64KB + assert_eq!(DEFAULT_BUFFER_MAX_SIZE, 1048576); // 1MB + assert_eq!(DEFAULT_BUFFER_UNKNOWN_SIZE, 262144); // 256KB + } + + #[test] + fn test_constants() { + assert_eq!(KI_B, 1024); + assert_eq!(MI_B, 1024 * 1024); + assert_eq!(64 * KI_B, DEFAULT_BUFFER_MIN_SIZE); + assert_eq!(MI_B, DEFAULT_BUFFER_MAX_SIZE); + assert_eq!(256 * KI_B, DEFAULT_BUFFER_UNKNOWN_SIZE); + } + + #[test] + fn test_env_var_names() { + assert_eq!(ENV_RUSTFS_BUFFER_MIN_SIZE, "RUSTFS_BUFFER_MIN_SIZE"); + assert_eq!(ENV_RUSTFS_BUFFER_MAX_SIZE, "RUSTFS_BUFFER_MAX_SIZE"); + assert_eq!(ENV_RUSTFS_BUFFER_DEFAULT_SIZE, "RUSTFS_BUFFER_DEFAULT_SIZE"); + } +} diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index c46fcf104..e0c1fd068 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -45,6 +45,8 @@ pub use constants::targets::*; #[cfg(feature = "constants")] pub use constants::tls::*; #[cfg(feature = "constants")] +pub use constants::workload::*; +#[cfg(feature = "constants")] pub mod oidc { pub use super::constants::oidc::*; } diff --git a/rustfs/src/admin/console.rs b/rustfs/src/admin/console.rs index cb2a7158b..e76112909 100644 --- a/rustfs/src/admin/console.rs +++ b/rustfs/src/admin/console.rs @@ -13,9 +13,9 @@ // limitations under the License. use crate::admin::handlers::health::{HealthProbe, build_component_details, collect_dependency_readiness, health_check_state}; -use crate::config::build; use crate::license::get_license; use crate::server::{CONSOLE_PREFIX, FAVICON_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, RUSTFS_ADMIN_PREFIX}; +use crate::version::build; use axum::{ Router, body::Body, diff --git a/rustfs/src/app/context.rs b/rustfs/src/app/context.rs index 287d0df52..f18bce2c6 100644 --- a/rustfs/src/app/context.rs +++ b/rustfs/src/app/context.rs @@ -16,7 +16,7 @@ //! This module introduces explicit dependency injection entry points //! for storage, IAM, and KMS handles. -use crate::config::workload_profiles::{RustFSBufferConfig, get_global_buffer_config}; +use crate::config::{RustFSBufferConfig, get_global_buffer_config}; use async_trait::async_trait; use rustfs_ecstore::bucket::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys}; use rustfs_ecstore::config::{Config, get_global_server_config}; diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index aac013524..0a4c417cb 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -15,7 +15,7 @@ //! Object application use-case contracts. use crate::app::context::{AppContext, default_notify_interface, get_global_app_context}; -use crate::config::workload_profiles::RustFSBufferConfig; +use crate::config::RustFSBufferConfig; use crate::error::ApiError; use crate::storage::access::{authorize_request, has_bypass_governance_header, req_info_mut}; use crate::storage::concurrency::{ diff --git a/rustfs/src/config/cli.rs b/rustfs/src/config/cli.rs new file mode 100644 index 000000000..e662cf2ed --- /dev/null +++ b/rustfs/src/config/cli.rs @@ -0,0 +1,287 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! CLI definitions and parsing logic. +//! +//! This module contains the command-line interface definitions including: +//! - `Cli`: Main CLI parser +//! - `Commands`: Subcommands (Server, Info) +//! - `ServerOpts`: Server subcommand options +//! - `InfoOpts`: Info subcommand options +//! - `InfoType`: Information type enum +//! - `CommandResult`: Result of parsing command line arguments + +use crate::version::build; +use clap::builder::NonEmptyStringValueParser; +use clap::{Args, Parser, Subcommand, ValueEnum}; +use const_str::concat; +use rustfs_config::{DEFAULT_ADDRESS, DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_OBS_ENDPOINT, ENV_RUSTFS_VOLUMES}; +use std::path::PathBuf; +// build module is re-exported from crate::build + +#[allow(clippy::const_is_empty)] +pub(super) const SHORT_VERSION: &str = { + if !build::TAG.is_empty() { + build::TAG + } else if !build::SHORT_COMMIT.is_empty() { + concat!("@", build::SHORT_COMMIT) + } else { + build::PKG_VERSION + } +}; + +pub(super) const LONG_VERSION: &str = concat!( + concat!(SHORT_VERSION, "\n"), + concat!("build time : ", build::BUILD_TIME, "\n"), + concat!("build profile: ", build::BUILD_RUST_CHANNEL, "\n"), + concat!("build os : ", build::BUILD_OS, "\n"), + concat!("rust version : ", build::RUST_VERSION, "\n"), + concat!("rust channel : ", build::RUST_CHANNEL, "\n"), + concat!("git branch : ", build::BRANCH, "\n"), + concat!("git commit : ", build::COMMIT_HASH, "\n"), + concat!("git tag : ", build::TAG, "\n"), + concat!("git status :\n", build::GIT_STATUS_FILE), +); + +/// Known subcommands. When the first arg matches one of these, it is treated as a subcommand. +pub const KNOWN_SUBCOMMANDS: &[&str] = &["server", "info"]; + +/// Preprocess argv for legacy compatibility: `rustfs ` and `rustfs --address ...` are +/// treated as `rustfs server ` and `rustfs server --address ...` respectively. +/// Also: `rustfs` with no args becomes `rustfs server` (volumes from env), and `rustfs --info` +/// is treated as `rustfs info`. +pub fn preprocess_args_for_legacy(args: Vec) -> Vec { + if args.len() < 2 { + // rustfs -> rustfs server (volumes from RUSTFS_VOLUMES env) + return vec![args[0].clone(), "server".to_string()]; + } + let first = &args[1]; + // If first arg looks like a subcommand, do nothing + if KNOWN_SUBCOMMANDS.contains(&first.as_str()) { + return args; + } + // If first arg is --info, treat it as info subcommand + if first == "--info" { + let mut out = vec![args[0].clone(), "info".to_string()]; + out.extend(args[2..].iter().cloned()); + return out; + } + // If first arg is a global flag (--help, --version), do nothing + if first == "--help" || first == "-h" || first == "--version" || first == "-V" { + return args; + } + // Legacy: rustfs or rustfs --address ... -> rustfs server + let mut out = vec![args[0].clone(), "server".to_string()]; + out.extend(args[1..].iter().cloned()); + out +} + +/// Main CLI parser +#[derive(Parser, Clone)] +#[command(name = "rustfs", version = SHORT_VERSION, long_version = LONG_VERSION)] +pub struct Cli { + #[command(subcommand)] + pub command: Option, +} + +/// Available subcommands +#[derive(Subcommand, Clone)] +pub enum Commands { + /// Start the object storage server (default when no subcommand is given) + Server(Box), + /// Display system information + Info(InfoOpts), +} + +/// Information type to display +#[derive(ValueEnum, Clone, Copy, Debug, PartialEq, Eq)] +pub enum InfoType { + /// System basic information (OS, architecture, hostname, etc.) + System, + /// Runtime information (PID, memory, CPU, threads, etc.) + Runtime, + /// Build information (version, build time, git info, etc.) + Build, + /// Current configuration information + Config, + /// Dependency library versions + Deps, +} + +/// Info subcommand options +#[derive(Args, Clone)] +pub struct InfoOpts { + /// Display all information types + #[arg(long, conflicts_with = "info_type")] + pub all: bool, + + /// Type of information to display + #[arg(value_enum, conflicts_with = "all")] + pub info_type: Option, +} + +/// Server subcommand options +#[derive(Args, Clone)] +pub struct ServerOpts { + /// DIR points to a directory on a filesystem. + #[arg( + required = true, + env = "RUSTFS_VOLUMES", + value_delimiter = ' ', + value_parser = NonEmptyStringValueParser::new() + )] + pub volumes: Vec, + + /// bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname + #[arg( + long, + default_value_t = rustfs_config::DEFAULT_ADDRESS.to_string(), + env = "RUSTFS_ADDRESS" + )] + pub address: String, + + /// Domain name used for virtual-hosted-style requests. + #[arg( + long, + env = "RUSTFS_SERVER_DOMAINS", + value_delimiter = ',', + value_parser = NonEmptyStringValueParser::new() + )] + pub server_domains: Vec, + + /// Access key used for authentication. + #[arg(long, env = "RUSTFS_ACCESS_KEY", group = "access-key")] + pub access_key: Option, + + /// Access key stored in a file used for authentication. + #[arg(long, env = "RUSTFS_ACCESS_KEY_FILE", group = "access-key")] + pub access_key_file: Option, + + /// Secret key used for authentication. + #[arg(long, env = "RUSTFS_SECRET_KEY", group = "secret-key")] + pub secret_key: Option, + + /// Secret key stored in a file used for authentication. + #[arg(long, env = "RUSTFS_SECRET_KEY_FILE", group = "secret-key")] + pub secret_key_file: Option, + + /// Enable console server + #[arg( + long, + default_value_t = rustfs_config::DEFAULT_CONSOLE_ENABLE, + env = "RUSTFS_CONSOLE_ENABLE" + )] + pub console_enable: bool, + + /// Console server bind address + #[arg( + long, + default_value_t = rustfs_config::DEFAULT_CONSOLE_ADDRESS.to_string(), + env = "RUSTFS_CONSOLE_ADDRESS" + )] + pub console_address: String, + + /// Observability endpoint for trace, metrics and logs,only support grpc mode. + #[arg( + long, + default_value_t = rustfs_config::DEFAULT_OBS_ENDPOINT.to_string(), + env = "RUSTFS_OBS_ENDPOINT" + )] + pub obs_endpoint: String, + + /// tls path for rustfs API and console. + #[arg(long, env = "RUSTFS_TLS_PATH")] + pub tls_path: Option, + + #[arg(long, env = "RUSTFS_LICENSE")] + pub license: Option, + + #[arg(long, env = "RUSTFS_REGION")] + pub region: Option, + + /// Enable KMS encryption for server-side encryption + #[arg(long, default_value_t = false, env = "RUSTFS_KMS_ENABLE")] + pub kms_enable: bool, + + /// KMS backend type (local or vault) + #[arg(long, default_value_t = rustfs_config::DEFAULT_KMS_BACKEND.to_string(), env = "RUSTFS_KMS_BACKEND")] + pub kms_backend: String, + + /// KMS key directory for local backend + #[arg(long, env = "RUSTFS_KMS_KEY_DIR")] + pub kms_key_dir: Option, + + /// Vault address for vault backend + #[arg(long, env = "RUSTFS_KMS_VAULT_ADDRESS")] + pub kms_vault_address: Option, + + /// Vault token for vault backend + #[arg(long, env = "RUSTFS_KMS_VAULT_TOKEN")] + pub kms_vault_token: Option, + + /// Default KMS key ID for encryption + #[arg(long, env = "RUSTFS_KMS_DEFAULT_KEY_ID")] + pub kms_default_key_id: Option, + + /// Disable adaptive buffer sizing with workload profiles + /// Set this flag to use legacy fixed-size buffer behavior from PR #869 + #[arg(long, default_value_t = false, env = "RUSTFS_BUFFER_PROFILE_DISABLE")] + pub buffer_profile_disable: bool, + + /// Workload profile for adaptive buffer sizing + /// Options: GeneralPurpose, AiTraining, DataAnalytics, WebWorkload, IndustrialIoT, SecureStorage + #[arg(long, default_value_t = rustfs_config::DEFAULT_BUFFER_PROFILE.to_string(), env = "RUSTFS_BUFFER_PROFILE")] + pub buffer_profile: String, +} + +/// Result of parsing command line arguments +#[derive(Clone)] +pub enum CommandResult { + /// Server command with configuration + Server(Box), + /// Info command with options + Info(InfoOpts), +} + +/// Create default ServerOpts from environment variables +pub fn default_server_opts() -> ServerOpts { + ServerOpts { + volumes: std::env::var(ENV_RUSTFS_VOLUMES) + .unwrap_or_default() + .split(' ') + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .collect(), + address: DEFAULT_ADDRESS.to_string(), + server_domains: vec![], + access_key: None, + access_key_file: None, + secret_key: None, + secret_key_file: None, + console_enable: DEFAULT_CONSOLE_ENABLE, + console_address: DEFAULT_CONSOLE_ADDRESS.to_string(), + obs_endpoint: DEFAULT_OBS_ENDPOINT.to_string(), + tls_path: None, + license: None, + region: None, + kms_enable: false, + kms_backend: "local".to_string(), + kms_key_dir: None, + kms_vault_address: None, + kms_vault_token: None, + kms_default_key_id: None, + buffer_profile_disable: false, + buffer_profile: "GeneralPurpose".to_string(), + } +} diff --git a/rustfs/src/config/config_struct.rs b/rustfs/src/config/config_struct.rs new file mode 100644 index 000000000..0d4f730cd --- /dev/null +++ b/rustfs/src/config/config_struct.rs @@ -0,0 +1,205 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Server configuration. +//! +//! This module contains the `Config` struct which holds the final server configuration +//! after processing command line arguments, environment variables, and files. + +use super::Opt; +use crate::apply_external_env_compat; +use rustfs_config::{ENV_RUSTFS_ROOT_PASSWORD, ENV_RUSTFS_ROOT_USER, RUSTFS_REGION}; +use rustfs_credentials::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, Masked}; + +/// Helper function to resolve credentials from multiple sources with precedence: +/// 1. Inline value (if provided) +/// 2. File value (if provided, read the content of the file) +/// 3. Environment variable (if set) +/// 4. Default value (if none of the above are provided) +pub(crate) fn resolve_credential>( + inline_value: Option, + file_value: Option, + env_key: &str, + default_value: &str, +) -> std::io::Result { + let value = inline_value + .map(Ok) + .or_else(|| file_value.map(std::fs::read_to_string)) + .or_else(|| rustfs_utils::get_env_opt_str(env_key).map(Ok)) + .transpose()? + .unwrap_or_else(|| default_value.to_string()); + + Ok(value.trim().to_string()) +} + +/// Server configuration. +/// +/// This struct holds all configuration values needed to run the server. +/// It is created from `Opt` which is parsed from command line arguments. +#[derive(Clone)] +pub struct Config { + /// DIR points to a directory on a filesystem. + pub volumes: Vec, + + /// bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname + pub address: String, + + /// Domain name used for virtual-hosted-style requests. + pub server_domains: Vec, + + /// Access key used for authentication. + pub access_key: String, + + /// Secret key used for authentication. + pub secret_key: String, + + /// Enable console server + pub console_enable: bool, + + /// Console server bind address + pub console_address: String, + + /// Observability endpoint for trace, metrics and logs,only support grpc mode. + pub obs_endpoint: String, + + /// tls path for rustfs API and console. + pub tls_path: Option, + + /// License key for enterprise features + pub license: Option, + + /// Region for the server, used for signing and other region-specific behavior + pub region: Option, + + /// Enable KMS encryption for server-side encryption + pub kms_enable: bool, + + /// KMS backend type (local or vault) + pub kms_backend: String, + + /// KMS key directory for local backend + pub kms_key_dir: Option, + + /// Vault address for vault backend + pub kms_vault_address: Option, + + /// Vault token for vault backend + pub kms_vault_token: Option, + + /// Default KMS key ID for encryption + pub kms_default_key_id: Option, + + /// Disable adaptive buffer sizing with workload profiles + pub buffer_profile_disable: bool, + + /// Workload profile for adaptive buffer sizing + pub buffer_profile: String, +} + +impl Config { + /// Create Config from Opt + pub(super) fn from_opt(opt: Opt) -> std::io::Result { + let Opt { + volumes, + address, + server_domains, + access_key, + access_key_file, + secret_key, + secret_key_file, + console_enable, + console_address, + obs_endpoint, + tls_path, + license, + region, + kms_enable, + kms_backend, + kms_key_dir, + kms_vault_address, + kms_vault_token, + kms_default_key_id, + buffer_profile_disable, + buffer_profile, + } = opt; + + let access_key = resolve_credential(access_key, access_key_file.as_ref(), ENV_RUSTFS_ROOT_USER, DEFAULT_ACCESS_KEY)?; + let secret_key = resolve_credential(secret_key, secret_key_file.as_ref(), ENV_RUSTFS_ROOT_PASSWORD, DEFAULT_SECRET_KEY)?; + + // Region is optional, but if not set, we should default to "us-east-1" for signing compatibility with AWS S3 clients + let region = region.or_else(|| Some(RUSTFS_REGION.to_string())); + + Ok(Config { + volumes, + address, + server_domains, + access_key, + secret_key, + console_enable, + console_address, + obs_endpoint, + tls_path, + license, + region, + kms_enable, + kms_backend, + kms_key_dir, + kms_vault_address, + kms_vault_token, + kms_default_key_id, + buffer_profile_disable, + buffer_profile, + }) + } + + /// Parse the command line arguments and environment arguments from [`Opt`] and convert them + /// into a ready to use [`Config`]. + /// + /// Supports both `rustfs ` (legacy) and `rustfs server `. + /// + /// This includes some intermediate checks for mutually exclusive options. + #[allow(dead_code)] // used in config_test + pub fn parse() -> std::io::Result { + let _ = apply_external_env_compat(); + let args: Vec = std::env::args().collect(); + let opt = Opt::parse_from(args); + Self::from_opt(opt) + } +} + +impl std::fmt::Debug for Config { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Config") + .field("volumes", &self.volumes) + .field("address", &self.address) + .field("server_domains", &self.server_domains) + .field("access_key", &self.access_key) + .field("secret_key", &Masked(Some(&self.secret_key))) // Hide sensitive values + .field("console_enable", &self.console_enable) + .field("console_address", &self.console_address) + .field("obs_endpoint", &self.obs_endpoint) + .field("tls_path", &self.tls_path) + .field("license", &Masked(self.license.as_deref())) + .field("region", &self.region) + .field("kms_enable", &self.kms_enable) + .field("kms_backend", &self.kms_backend) + .field("kms_key_dir", &self.kms_key_dir) + .field("kms_vault_address", &self.kms_vault_address) + .field("kms_vault_token", &Masked(self.kms_vault_token.as_deref())) + .field("kms_default_key_id", &self.kms_default_key_id) + .field("buffer_profile_disable", &self.buffer_profile_disable) + .field("buffer_profile", &self.buffer_profile) + .finish() + } +} diff --git a/rustfs/src/config/info.rs b/rustfs/src/config/info.rs new file mode 100644 index 000000000..0ce0fe982 --- /dev/null +++ b/rustfs/src/config/info.rs @@ -0,0 +1,656 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! System information display module. +//! +//! This module provides the `--info` command functionality for querying and displaying +//! various system information including system basics, runtime stats, build info, +//! configuration, and dependencies. + +use super::{InfoOpts, InfoType}; +use crate::version::build; +use rustfs_credentials::Masked; +use std::fmt; + +/// CPU information +struct CpuInfo { + /// Number of logical CPU cores + core_count: usize, + /// CPU brand/vendor name + brand: String, + /// CPU frequency in MHz + frequency_mhz: u64, + /// CPU usage percentage (0-100) + usage_percent: f64, +} + +impl CpuInfo { + fn collect(sys: &sysinfo::System) -> Self { + let core_count = sys.cpus().len(); + let brand = sys + .cpus() + .first() + .map(|c| c.brand().to_string()) + .unwrap_or_else(|| "Unknown".to_string()); + let frequency_mhz = sys.cpus().first().map(|c| c.frequency()).unwrap_or(0); + + // Calculate average CPU usage + let usage_percent = if core_count > 0 { + sys.cpus().iter().map(|c| c.cpu_usage() as f64).sum::() / core_count as f64 + } else { + 0.0 + }; + + Self { + core_count, + brand, + frequency_mhz, + usage_percent, + } + } + + fn format(&self) -> String { + format!( + "CPU Cores: {}\n\ + CPU Brand: {}\n\ + CPU Frequency: {} MHz\n\ + CPU Usage: {:.1}%", + self.core_count, self.brand, self.frequency_mhz, self.usage_percent + ) + } +} + +/// Memory information +struct MemoryInfo { + /// Total system memory in bytes + total_bytes: u64, + /// Used memory in bytes + used_bytes: u64, + /// Available/free memory in bytes + available_bytes: u64, + /// Total swap memory in bytes + total_swap_bytes: u64, + /// Used swap memory in bytes + used_swap_bytes: u64, + /// Memory usage percentage + usage_percent: f64, +} + +impl MemoryInfo { + fn collect(sys: &sysinfo::System) -> Self { + let total_bytes = sys.total_memory(); + let used_bytes = sys.used_memory(); + let available_bytes = sys.available_memory(); + let total_swap_bytes = sys.total_swap(); + let used_swap_bytes = sys.used_swap(); + + let usage_percent = if total_bytes > 0 { + (used_bytes as f64 / total_bytes as f64) * 100.0 + } else { + 0.0 + }; + + Self { + total_bytes, + used_bytes, + available_bytes, + total_swap_bytes, + used_swap_bytes, + usage_percent, + } + } + + fn format_bytes(bytes: u64) -> String { + const KB: u64 = 1024; + const MB: u64 = KB * 1024; + const GB: u64 = MB * 1024; + const TB: u64 = GB * 1024; + + if bytes >= TB { + format!("{:.2} TB", bytes as f64 / TB as f64) + } else if bytes >= GB { + format!("{:.2} GB", bytes as f64 / GB as f64) + } else if bytes >= MB { + format!("{:.2} MB", bytes as f64 / MB as f64) + } else if bytes >= KB { + format!("{:.2} KB", bytes as f64 / KB as f64) + } else { + format!("{} B", bytes) + } + } + + fn format(&self) -> String { + format!( + "Total Memory: {}\n\ + Used Memory: {} ({:.1}%)\n\ + Available Memory: {}\n\ + Total Swap: {}\n\ + Used Swap: {}", + Self::format_bytes(self.total_bytes), + Self::format_bytes(self.used_bytes), + self.usage_percent, + Self::format_bytes(self.available_bytes), + Self::format_bytes(self.total_swap_bytes), + Self::format_bytes(self.used_swap_bytes) + ) + } +} + +/// Disk information +struct DiskInfo { + /// Disk mount point + mount_point: String, + /// Disk name/device + name: String, + /// File system type + file_system: String, + /// Total space in bytes + total_bytes: u64, + /// Used space in bytes + used_bytes: u64, + /// Available space in bytes + available_bytes: u64, + /// Usage percentage + usage_percent: f64, + /// Is this a removable disk + is_removable: bool, +} + +impl DiskInfo { + fn collect_all() -> Vec { + let disks = sysinfo::Disks::new_with_refreshed_list(); + disks + .iter() + .map(|disk| { + let total_bytes = disk.total_space(); + let available_bytes = disk.available_space(); + let used_bytes = total_bytes.saturating_sub(available_bytes); + let usage_percent = if total_bytes > 0 { + (used_bytes as f64 / total_bytes as f64) * 100.0 + } else { + 0.0 + }; + + Self { + mount_point: disk.mount_point().to_string_lossy().to_string(), + name: disk.name().to_string_lossy().to_string(), + file_system: format!("{:?}", disk.file_system()), + total_bytes, + used_bytes, + available_bytes, + usage_percent, + is_removable: disk.is_removable(), + } + }) + .collect() + } + + fn format(&self) -> String { + format!( + " [{}] {}\n\ + Mount: {}\n\ + Type: {}\n\ + Total: {}\n\ + Used: {} ({:.1}%)\n\ + Available: {}\n\ + Removable: {}", + if self.is_removable { "R" } else { "F" }, + self.name, + self.mount_point, + self.file_system, + MemoryInfo::format_bytes(self.total_bytes), + MemoryInfo::format_bytes(self.used_bytes), + self.usage_percent, + MemoryInfo::format_bytes(self.available_bytes), + self.is_removable + ) + } +} + +/// Service disk information (disk where the service data is stored) +struct ServiceDiskInfo { + /// Disk mount point + mount_point: String, + /// Disk name/device + name: String, + /// File system type + file_system: String, + /// Total space in bytes + total_bytes: u64, + /// Used space in bytes + used_bytes: u64, + /// Available space in bytes + available_bytes: u64, + /// Usage percentage + usage_percent: f64, + /// Volume paths served by this disk + volume_paths: Vec, +} + +impl ServiceDiskInfo { + fn collect(volumes: &[String]) -> Option { + // Find the disk that contains the first volume path + let first_volume = volumes.first()?; + let volume_path = std::path::Path::new(first_volume); + + // Find the disk that contains this volume + let disks = sysinfo::Disks::new_with_refreshed_list(); + for disk in disks.iter() { + let mount_point = disk.mount_point(); + if volume_path.starts_with(mount_point) { + let total_bytes = disk.total_space(); + let available_bytes = disk.available_space(); + let used_bytes = total_bytes.saturating_sub(available_bytes); + let usage_percent = if total_bytes > 0 { + (used_bytes as f64 / total_bytes as f64) * 100.0 + } else { + 0.0 + }; + + // Find all volumes on this disk + let volume_paths: Vec = volumes + .iter() + .filter(|v| std::path::Path::new(v).starts_with(mount_point)) + .cloned() + .collect(); + + return Some(Self { + mount_point: mount_point.to_string_lossy().to_string(), + name: disk.name().to_string_lossy().to_string(), + file_system: format!("{:?}", disk.file_system()), + total_bytes, + used_bytes, + available_bytes, + usage_percent, + volume_paths, + }); + } + } + + None + } + + fn format(&self) -> String { + let volumes_str = if self.volume_paths.is_empty() { + "(none)".to_string() + } else { + self.volume_paths.join(", ") + }; + + format!( + "=== Service Disk Information ===\n\ + Disk: {}\n\ + Mount Point: {}\n\ + File System: {}\n\ + Total Space: {}\n\ + Used Space: {} ({:.1}%)\n\ + Available Space: {}\n\ + Served Volumes: {}", + self.name, + self.mount_point, + self.file_system, + MemoryInfo::format_bytes(self.total_bytes), + MemoryInfo::format_bytes(self.used_bytes), + self.usage_percent, + MemoryInfo::format_bytes(self.available_bytes), + volumes_str + ) + } +} + +/// System basic information +struct SystemInfo { + os_type: String, + os_version: String, + architecture: String, + hostname: String, + kernel_version: String, + /// CPU information + cpu: CpuInfo, + /// Memory information + memory: MemoryInfo, + /// All disk information + disks: Vec, +} + +impl SystemInfo { + fn collect() -> Self { + // Create system info collector + let mut sys = sysinfo::System::new_all(); + sys.refresh_all(); + sys.refresh_cpu_all(); + + Self { + os_type: sysinfo::System::distribution_id(), + os_version: sysinfo::System::long_os_version().unwrap_or_else(|| "Unknown".to_string()), + architecture: std::env::consts::ARCH.to_string(), + hostname: sysinfo::System::host_name().unwrap_or_else(|| "Unknown".to_string()), + kernel_version: sysinfo::System::kernel_long_version(), + cpu: CpuInfo::collect(&sys), + memory: MemoryInfo::collect(&sys), + disks: DiskInfo::collect_all(), + } + } + + fn format(&self) -> String { + let disks_str = if self.disks.is_empty() { + " (no disks found)".to_string() + } else { + self.disks.iter().map(|d| d.format()).collect::>().join("\n") + }; + + format!( + "=== System Information ===\n\ + OS: {}\n\ + OS Version: {}\n\ + Architecture: {}\n\ + Hostname: {}\n\ + Kernel Version: {}\n\ + \n\ + === CPU Information ===\n\ + {}\n\ + \n\ + === Memory Information ===\n\ + {}\n\ + \n\ + === Disk Information ===\n\ + {}", + self.os_type, + self.os_version, + self.architecture, + self.hostname, + self.kernel_version, + self.cpu.format(), + self.memory.format(), + disks_str + ) + } +} +impl fmt::Display for SystemInfo { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.format()) + } +} + +/// Runtime information +struct RuntimeInfo { + process_id: u32, + memory_usage_mb: f64, + cpu_usage_percent: f64, + thread_count: usize, +} + +impl RuntimeInfo { + fn collect() -> Self { + let (process_id, memory_usage_mb, cpu_usage_percent) = if let Ok(pid) = sysinfo::get_current_pid() { + let mut sys = sysinfo::System::new(); + sys.refresh_processes_specifics( + sysinfo::ProcessesToUpdate::Some(&[pid]), + true, + sysinfo::ProcessRefreshKind::everything(), + ); + + let process = sys.process(pid); + let (memory_usage_mb, cpu_usage_percent) = if let Some(p) = process { + let memory = p.memory() as f64 / 1024.0 / 1024.0; // Convert to MB + let cpu = p.cpu_usage() as f64; + (memory, cpu) + } else { + (0.0, 0.0) + }; + + (pid.as_u32(), memory_usage_mb, cpu_usage_percent) + } else { + // Failed to retrieve current PID; degrade gracefully by + // skipping per-process stats and using sentinel values. + (0, 0.0, 0.0) + }; + + // // Get available CPU parallelism (roughly, logical cores available to the process) + let cpu_parallelism = std::thread::available_parallelism().map(|p| p.get()).unwrap_or(1); + + Self { + process_id, + memory_usage_mb, + cpu_usage_percent, + thread_count: cpu_parallelism, + } + } + + fn format(&self) -> String { + let pid_display = if self.process_id == 0 { + "unknown".to_string() + } else { + self.process_id.to_string() + }; + + format!( + "=== Runtime Information ===\n\ + Process ID: {}\n\ + Memory Usage: {:.2} MB\n\ + CPU Usage: {:.2}%\n\ + CPU Parallelism (logical cores): {}", + pid_display, self.memory_usage_mb, self.cpu_usage_percent, self.thread_count + ) + } +} +impl fmt::Display for RuntimeInfo { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.format()) + } +} + +/// Build information (using shadow-rs generated constants) +struct BuildInfo; + +impl BuildInfo { + fn format() -> String { + format!( + "=== Build Information ===\n\ + Version: {}\n\ + Build Time: {}\n\ + Build Profile: {}\n\ + Build OS: {}\n\ + Rust Version: {}\n\ + Git Branch: {}\n\ + Git Commit: {}\n\ + Git Tag: {}\n\ + Git Status: {}", + build::PKG_VERSION, + build::BUILD_TIME, + build::BUILD_RUST_CHANNEL, + build::BUILD_OS, + build::RUST_VERSION, + build::BRANCH, + build::COMMIT_HASH, + build::TAG, + build::GIT_STATUS_FILE + ) + } +} + +impl fmt::Display for BuildInfo { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", Self::format()) + } +} + +/// Configuration information display +fn format_config_info() -> String { + // Get config snapshot for display (from global if initialized, otherwise from env) + let snapshot = super::get_config_snapshot_for_display(); + + // Get workload profile info + let workload_info = get_workload_profile_info(); + + // Mask the access key for display + let masked_access_key = &Masked(Some(&snapshot.access_key)); + + format!( + "=== Configuration Information ===\n\ + Server Address: {}\n\ + Console Enable: {}\n\ + Console Address: {}\n\ + Region: {}\n\ + Access Key: {}\n\ + Secret Key: ****\n\ + OBS Endpoint: {}\n\ + TLS Path: {}\n\ + KMS Enabled: {}\n\ + KMS Backend: {}\n\ + Buffer Profile: {}\n\ + {}", + snapshot.address, + snapshot.console_enable, + snapshot.console_address, + snapshot.region.as_deref().unwrap_or("(not set)"), + masked_access_key, + if snapshot.obs_endpoint.is_empty() { + "(not set)" + } else { + &snapshot.obs_endpoint + }, + snapshot.tls_path.as_deref().unwrap_or("(not set)"), + snapshot.kms_enable, + snapshot.kms_backend, + snapshot.buffer_profile, + workload_info + ) +} + +/// Get workload profile information from global buffer config +fn get_workload_profile_info() -> String { + use super::workload_profiles::{get_global_buffer_config, is_buffer_profile_enabled}; + + if !is_buffer_profile_enabled() { + return "Workload Profile: (disabled)".to_string(); + } + + let config = get_global_buffer_config(); + let profile = config.workload_profile(); + let name = config.workload_name(); + let buffer_config = profile.config(); + + format!( + "Workload Profile: {}\n\ + Buffer Min Size: {} bytes\n\ + Buffer Max Size: {} bytes\n\ + Default Unknown: {} bytes", + name, buffer_config.min_size, buffer_config.max_size, buffer_config.default_unknown + ) +} + +/// Dependency information +fn format_deps_info() -> String { + let mut output = String::from("=== Build Features ===\n"); + + // Check which features are enabled at compile time + let features = [ + ("metrics", cfg!(feature = "metrics"), "Metrics collection and reporting"), + ("ftps", cfg!(feature = "ftps"), "FTPS protocol support"), + ("swift", cfg!(feature = "swift"), "Swift storage backend"), + ("webdav", cfg!(feature = "webdav"), "WebDAV protocol support"), + ("license", cfg!(feature = "license"), "License validation"), + ("full", cfg!(feature = "full"), "All features enabled"), + ]; + + let enabled_count = features.iter().filter(|(_, enabled, _)| *enabled).count(); + output.push_str(&format!("Enabled Features: {}/{}\n\n", enabled_count, features.len())); + + output.push_str("Feature Status:\n"); + for (name, enabled, description) in features { + let status = if enabled { "[x]" } else { "[ ]" }; + output.push_str(&format!(" {} {} - {}\n", status, name, description)); + } + + // Show default features info + output.push_str("\n--- Default Features ---\n"); + output.push_str(" metrics (enabled by default)\n"); + + // Show feature dependencies + output.push_str("\n--- Feature Dependencies ---\n"); + output.push_str(" full = metrics + ftps + swift + webdav\n"); + output.push_str(" ftps -> rustfs-protocols/ftps\n"); + output.push_str(" swift -> rustfs-protocols/swift\n"); + output.push_str(" webdav -> rustfs-protocols/webdav\n"); + + output +} + +/// Execute the info command +pub fn execute_info(opts: &InfoOpts) { + execute_info_with_volumes(opts, &[]) +} + +/// Execute info command with volume paths for service disk information +pub fn execute_info_with_volumes(opts: &InfoOpts, volumes: &[String]) { + let info_type = if opts.all { + None // None means display all + } else { + opts.info_type + }; + + match info_type { + None => { + // Display all information + println!("{}", SystemInfo::collect()); + println!(); + println!("{}", RuntimeInfo::collect()); + println!(); + println!("{}", BuildInfo); + println!(); + println!("{}", format_config_info()); + println!(); + // Display service disk info if volumes are configured + if let Some(service_disk) = ServiceDiskInfo::collect(volumes) { + println!("{}", service_disk.format()); + println!(); + } + println!("{}", format_deps_info()); + } + Some(InfoType::System) => { + println!("{}", SystemInfo::collect()); + } + Some(InfoType::Runtime) => { + println!("{}", RuntimeInfo::collect()); + } + Some(InfoType::Build) => { + println!("{}", BuildInfo); + } + Some(InfoType::Config) => { + println!("{}", format_config_info()); + } + Some(InfoType::Deps) => { + println!("{}", format_deps_info()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_system_info_collect() { + let info = SystemInfo::collect(); + assert!(!info.os_type.is_empty()); + assert!(!info.architecture.is_empty()); + } + + #[test] + fn test_runtime_info_collect() { + let info = RuntimeInfo::collect(); + assert!(info.process_id > 0); + } +} diff --git a/rustfs/src/config/mod.rs b/rustfs/src/config/mod.rs index ea6b20a64..d51d920fb 100644 --- a/rustfs/src/config/mod.rs +++ b/rustfs/src/config/mod.rs @@ -12,464 +12,50 @@ // See the License for the specific language governing permissions and // limitations under the License. -use clap::builder::NonEmptyStringValueParser; -use clap::{Args, Parser, Subcommand}; -use const_str::concat; -use rustfs_config::RUSTFS_REGION; -use rustfs_utils::{apply_external_env_compat, get_env_opt_str}; -use std::path::PathBuf; -use std::string::ToString; +//! Configuration module for RustFS. +//! +//! This module is organized into the following submodules: +//! +//! - [`cli`]: Command-line interface definitions (Cli, Commands, ServerOpts, InfoOpts) +//! - [`opt`]: Parsed server options (Opt) and parsing methods +//! - [`config_struct`]: Server configuration (Config) +//! - [`snapshot`]: Configuration snapshot for info command +//! - [`info`]: Info command execution +//! +//! # Usage +//! +//! ```ignore +//! use rustfs::config::{Config, Opt, CommandResult}; +//! +//! // Parse command line arguments +//! let result = Opt::parse_command(std::env::args())?; +//! +//! match result { +//! CommandResult::Server(config) => { +//! // Start server with config +//! } +//! CommandResult::Info(opts) => { +//! // Display info +//! } +//! } +//! ``` -shadow_rs::shadow!(build); - -pub mod workload_profiles; +mod cli; +mod config_struct; +mod info; +mod opt; +mod snapshot; #[cfg(test)] mod config_test; -#[allow(clippy::const_is_empty)] -const SHORT_VERSION: &str = { - if !build::TAG.is_empty() { - build::TAG - } else if !build::SHORT_COMMIT.is_empty() { - concat!("@", build::SHORT_COMMIT) - } else { - build::PKG_VERSION - } -}; +// Re-export public types +pub use cli::{CommandResult, InfoOpts, InfoType}; +pub use config_struct::Config; +pub use info::execute_info; +pub use opt::Opt; +pub use snapshot::{get_config_snapshot_for_display, init_config_snapshot}; -const LONG_VERSION: &str = concat!( - concat!(SHORT_VERSION, "\n"), - concat!("build time : ", build::BUILD_TIME, "\n"), - concat!("build profile: ", build::BUILD_RUST_CHANNEL, "\n"), - concat!("build os : ", build::BUILD_OS, "\n"), - concat!("rust version : ", build::RUST_VERSION, "\n"), - concat!("rust channel : ", build::RUST_CHANNEL, "\n"), - concat!("git branch : ", build::BRANCH, "\n"), - concat!("git commit : ", build::COMMIT_HASH, "\n"), - concat!("git tag : ", build::TAG, "\n"), - concat!("git status :\n", build::GIT_STATUS_FILE), -); - -/// Known subcommands. When the first arg matches one of these, it is treated as a subcommand. -const KNOWN_SUBCOMMANDS: &[&str] = &["server"]; - -/// Preprocess argv for legacy compatibility: `rustfs ` and `rustfs --address ...` are -/// treated as `rustfs server ` and `rustfs server --address ...` respectively. -/// Also: `rustfs` with no args becomes `rustfs server` (volumes from env). -fn preprocess_args_for_legacy(args: Vec) -> Vec { - if args.len() < 2 { - // rustfs -> rustfs server (volumes from RUSTFS_VOLUMES env) - return vec![args[0].clone(), "server".to_string()]; - } - let first = &args[1]; - // If first arg looks like a subcommand, do nothing - if KNOWN_SUBCOMMANDS.contains(&first.as_str()) { - return args; - } - // If first arg is a global flag (--help, --version), do nothing - if first == "--help" || first == "-h" || first == "--version" || first == "-V" { - return args; - } - // Legacy: rustfs or rustfs --address ... -> rustfs server - let mut out = vec![args[0].clone(), "server".to_string()]; - out.extend(args[1..].iter().cloned()); - out -} - -#[derive(Parser, Clone)] -#[command(name = "rustfs", version = SHORT_VERSION, long_version = LONG_VERSION)] -struct Cli { - #[command(subcommand)] - command: Option, -} - -#[derive(Subcommand, Clone)] -enum Commands { - /// Start the object storage server (default when no subcommand is given) - Server(ServerOpts), -} - -#[derive(Args, Clone)] -struct ServerOpts { - /// DIR points to a directory on a filesystem. - #[arg( - required = true, - env = "RUSTFS_VOLUMES", - value_delimiter = ' ', - value_parser = NonEmptyStringValueParser::new() - )] - pub volumes: Vec, - - /// bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname - #[arg( - long, - default_value_t = rustfs_config::DEFAULT_ADDRESS.to_string(), - env = "RUSTFS_ADDRESS" - )] - pub address: String, - - /// Domain name used for virtual-hosted-style requests. - #[arg( - long, - env = "RUSTFS_SERVER_DOMAINS", - value_delimiter = ',', - value_parser = NonEmptyStringValueParser::new() - )] - pub server_domains: Vec, - - /// Access key used for authentication. - #[arg(long, env = "RUSTFS_ACCESS_KEY", group = "access-key")] - pub access_key: Option, - - /// Access key stored in a file used for authentication. - #[arg(long, env = "RUSTFS_ACCESS_KEY_FILE", group = "access-key")] - pub access_key_file: Option, - - /// Secret key used for authentication. - #[arg(long, env = "RUSTFS_SECRET_KEY", group = "secret-key")] - pub secret_key: Option, - - /// Secret key stored in a file used for authentication. - #[arg(long, env = "RUSTFS_SECRET_KEY_FILE", group = "secret-key")] - pub secret_key_file: Option, - - /// Enable console server - #[arg( - long, - default_value_t = rustfs_config::DEFAULT_CONSOLE_ENABLE, - env = "RUSTFS_CONSOLE_ENABLE" - )] - pub console_enable: bool, - - /// Console server bind address - #[arg( - long, - default_value_t = rustfs_config::DEFAULT_CONSOLE_ADDRESS.to_string(), - env = "RUSTFS_CONSOLE_ADDRESS" - )] - pub console_address: String, - - /// Observability endpoint for trace, metrics and logs,only support grpc mode. - #[arg( - long, - default_value_t = rustfs_config::DEFAULT_OBS_ENDPOINT.to_string(), - env = "RUSTFS_OBS_ENDPOINT" - )] - pub obs_endpoint: String, - - /// tls path for rustfs API and console. - #[arg(long, env = "RUSTFS_TLS_PATH")] - pub tls_path: Option, - - #[arg(long, env = "RUSTFS_LICENSE")] - pub license: Option, - - #[arg(long, env = "RUSTFS_REGION")] - pub region: Option, - - /// Enable KMS encryption for server-side encryption - #[arg(long, default_value_t = false, env = "RUSTFS_KMS_ENABLE")] - pub kms_enable: bool, - - /// KMS backend type (local or vault) - #[arg(long, default_value_t = String::from("local"), env = "RUSTFS_KMS_BACKEND")] - pub kms_backend: String, - - /// KMS key directory for local backend - #[arg(long, env = "RUSTFS_KMS_KEY_DIR")] - pub kms_key_dir: Option, - - /// Vault address for vault backend - #[arg(long, env = "RUSTFS_KMS_VAULT_ADDRESS")] - pub kms_vault_address: Option, - - /// Vault token for vault backend - #[arg(long, env = "RUSTFS_KMS_VAULT_TOKEN")] - pub kms_vault_token: Option, - - /// Default KMS key ID for encryption - #[arg(long, env = "RUSTFS_KMS_DEFAULT_KEY_ID")] - pub kms_default_key_id: Option, - - /// Disable adaptive buffer sizing with workload profiles - /// Set this flag to use legacy fixed-size buffer behavior from PR #869 - #[arg(long, default_value_t = false, env = "RUSTFS_BUFFER_PROFILE_DISABLE")] - pub buffer_profile_disable: bool, - - /// Workload profile for adaptive buffer sizing - /// Options: GeneralPurpose, AiTraining, DataAnalytics, WebWorkload, IndustrialIoT, SecureStorage - #[arg(long, default_value_t = String::from("GeneralPurpose"), env = "RUSTFS_BUFFER_PROFILE")] - pub buffer_profile: String, -} - -/// Parsed server options. Public for tests and backward compatibility. -/// Use `Opt::parse_from` or `Config::parse()` to obtain. -#[derive(Clone)] -pub struct Opt { - pub volumes: Vec, - pub address: String, - pub server_domains: Vec, - pub access_key: Option, - pub access_key_file: Option, - pub secret_key: Option, - pub secret_key_file: Option, - pub console_enable: bool, - pub console_address: String, - pub obs_endpoint: String, - pub tls_path: Option, - pub license: Option, - pub region: Option, - pub kms_enable: bool, - pub kms_backend: String, - pub kms_key_dir: Option, - pub kms_vault_address: Option, - pub kms_vault_token: Option, - pub kms_default_key_id: Option, - pub buffer_profile_disable: bool, - pub buffer_profile: String, -} - -impl Opt { - fn from_server_opts(o: ServerOpts) -> Self { - Self { - volumes: o.volumes, - address: o.address, - server_domains: o.server_domains, - access_key: o.access_key, - access_key_file: o.access_key_file, - secret_key: o.secret_key, - secret_key_file: o.secret_key_file, - console_enable: o.console_enable, - console_address: o.console_address, - obs_endpoint: o.obs_endpoint, - tls_path: o.tls_path, - license: o.license, - region: o.region, - kms_enable: o.kms_enable, - kms_backend: o.kms_backend, - kms_key_dir: o.kms_key_dir, - kms_vault_address: o.kms_vault_address, - kms_vault_token: o.kms_vault_token, - kms_default_key_id: o.kms_default_key_id, - buffer_profile_disable: o.buffer_profile_disable, - buffer_profile: o.buffer_profile, - } - } - - /// Parse from preprocessed args. Supports both `rustfs ` and `rustfs server `. - pub fn parse_from(args: I) -> Self - where - I: IntoIterator, - T: Into + Clone, - { - let _ = apply_external_env_compat(); - let args: Vec = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect(); - let args = preprocess_args_for_legacy(args); - let cli = Cli::parse_from(args); - let Commands::Server(opts) = cli.command.expect("server is the default subcommand"); - Self::from_server_opts(opts) - } - - /// Try parse from args, returns error on invalid input. - #[allow(dead_code)] // used in config_test - pub fn try_parse_from(args: I) -> Result - where - I: IntoIterator, - T: Into + Clone, - { - let _ = apply_external_env_compat(); - let args: Vec = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect(); - let args = preprocess_args_for_legacy(args); - let cli = Cli::try_parse_from(args)?; - let Commands::Server(opts) = cli.command.expect("server is the default subcommand"); - Ok(Self::from_server_opts(opts)) - } - - /// Parse from env::args(). Used by Config::parse(). - fn parse() -> Self { - let args: Vec = std::env::args().collect(); - Self::parse_from(args) - } -} - -#[derive(Clone)] -pub struct Config { - /// DIR points to a directory on a filesystem. - pub volumes: Vec, - - /// bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname - pub address: String, - - /// Domain name used for virtual-hosted-style requests. - pub server_domains: Vec, - - /// Access key used for authentication. - pub access_key: String, - - /// Secret key used for authentication. - pub secret_key: String, - - /// Enable console server - pub console_enable: bool, - - /// Console server bind address - pub console_address: String, - - /// Observability endpoint for trace, metrics and logs,only support grpc mode. - pub obs_endpoint: String, - - /// tls path for rustfs API and console. - pub tls_path: Option, - - /// License key for enterprise features - pub license: Option, - - /// Region for the server, used for signing and other region-specific behavior - pub region: Option, - - /// Enable KMS encryption for server-side encryption - pub kms_enable: bool, - - /// KMS backend type (local or vault) - pub kms_backend: String, - - /// KMS key directory for local backend - pub kms_key_dir: Option, - - /// Vault address for vault backend - pub kms_vault_address: Option, - - /// Vault token for vault backend - pub kms_vault_token: Option, - - /// Default KMS key ID for encryption - pub kms_default_key_id: Option, - - /// Disable adaptive buffer sizing with workload profiles - /// Set this flag to use legacy fixed-size buffer behavior from PR #869 - pub buffer_profile_disable: bool, - - /// Workload profile for adaptive buffer sizing - /// Options: GeneralPurpose, AiTraining, DataAnalytics, WebWorkload, IndustrialIoT, SecureStorage - pub buffer_profile: String, -} - -impl Config { - fn from_opt(opt: Opt) -> std::io::Result { - let Opt { - volumes, - address, - server_domains, - access_key, - access_key_file, - secret_key, - secret_key_file, - console_enable, - console_address, - obs_endpoint, - tls_path, - license, - region, - kms_enable, - kms_backend, - kms_key_dir, - kms_vault_address, - kms_vault_token, - kms_default_key_id, - buffer_profile_disable, - buffer_profile, - } = opt; - - let access_key = access_key - .map(Ok) - .or_else(|| { - let path = access_key_file.as_ref()?; - Some(std::fs::read_to_string(path)) - }) - .or_else(|| get_env_opt_str("RUSTFS_ROOT_USER").map(Ok)) - .transpose()? - .unwrap_or_else(|| { - // neither argument was specified ... using default - rustfs_credentials::DEFAULT_ACCESS_KEY.to_string() - }) - .trim() - .to_string(); - - let secret_key = secret_key - .map(Ok) - .or_else(|| { - let path = secret_key_file.as_ref()?; - Some(std::fs::read_to_string(path)) - }) - .or_else(|| get_env_opt_str("RUSTFS_ROOT_PASSWORD").map(Ok)) - .transpose()? - .unwrap_or_else(|| { - // neither argument was specified ... using default - rustfs_credentials::DEFAULT_SECRET_KEY.to_string() - }) - .trim() - .to_string(); - - // Region is optional, but if not set, we should default to "us-east-1" for signing compatibility with AWS S3 clients - let region = region.or_else(|| Some(RUSTFS_REGION.to_string())); - - Ok(Config { - volumes, - address, - server_domains, - access_key, - secret_key, - console_enable, - console_address, - obs_endpoint, - tls_path, - license, - region, - kms_enable, - kms_backend, - kms_key_dir, - kms_vault_address, - kms_vault_token, - kms_default_key_id, - buffer_profile_disable, - buffer_profile, - }) - } - - /// Parse the command line arguments and environment arguments from [`Opt`] and convert them - /// into a ready to use [`Config`]. - /// - /// Supports both `rustfs ` (legacy) and `rustfs server `. - /// - /// This includes some intermediate checks for mutually exclusive options. - pub fn parse() -> std::io::Result { - Self::from_opt(Opt::parse()) - } -} - -impl std::fmt::Debug for Config { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Config") - .field("volumes", &self.volumes) - .field("address", &self.address) - .field("server_domains", &self.server_domains) - .field("access_key", &self.access_key) - .field("secret_key", &rustfs_credentials::Masked(Some(&self.secret_key))) // Hide sensitive values - .field("console_enable", &self.console_enable) - .field("console_address", &self.console_address) - .field("obs_endpoint", &self.obs_endpoint) - .field("tls_path", &self.tls_path) - .field("license", &rustfs_credentials::Masked(self.license.as_deref())) - .field("region", &self.region) - .field("kms_enable", &self.kms_enable) - .field("kms_backend", &self.kms_backend) - .field("kms_key_dir", &self.kms_key_dir) - .field("kms_vault_address", &self.kms_vault_address) - .field("kms_vault_token", &rustfs_credentials::Masked(self.kms_vault_token.as_deref())) - .field("kms_default_key_id", &self.kms_default_key_id) - .field("buffer_profile_disable", &self.buffer_profile_disable) - .field("buffer_profile", &self.buffer_profile) - .finish() - } -} +// Re-export workload profiles +mod workload_profiles; +pub use workload_profiles::*; diff --git a/rustfs/src/config/opt.rs b/rustfs/src/config/opt.rs new file mode 100644 index 000000000..5e35d6e3d --- /dev/null +++ b/rustfs/src/config/opt.rs @@ -0,0 +1,161 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Parsed server options. +//! +//! This module contains the `Opt` struct which holds parsed server options +//! and methods for parsing command line arguments. + +use super::Config; +use super::cli::{Cli, CommandResult, Commands, ServerOpts, default_server_opts, preprocess_args_for_legacy}; +use crate::apply_external_env_compat; +use CommandResult::Server; +use clap::Parser; +use std::path::PathBuf; + +/// Parsed server options. Public for tests and backward compatibility. +/// Use `Opt::parse_from` or `Config::parse()` to obtain. +#[derive(Clone)] +pub struct Opt { + pub volumes: Vec, + pub address: String, + pub server_domains: Vec, + pub access_key: Option, + pub access_key_file: Option, + pub secret_key: Option, + pub secret_key_file: Option, + pub console_enable: bool, + pub console_address: String, + pub obs_endpoint: String, + pub tls_path: Option, + pub license: Option, + pub region: Option, + pub kms_enable: bool, + pub kms_backend: String, + pub kms_key_dir: Option, + pub kms_vault_address: Option, + pub kms_vault_token: Option, + pub kms_default_key_id: Option, + pub buffer_profile_disable: bool, + pub buffer_profile: String, +} + +impl Opt { + /// Create Opt from ServerOpts + pub(super) fn from_server_opts(o: ServerOpts) -> Self { + Self { + volumes: o.volumes, + address: o.address, + server_domains: o.server_domains, + access_key: o.access_key, + access_key_file: o.access_key_file, + secret_key: o.secret_key, + secret_key_file: o.secret_key_file, + console_enable: o.console_enable, + console_address: o.console_address, + obs_endpoint: o.obs_endpoint, + tls_path: o.tls_path, + license: o.license, + region: o.region, + kms_enable: o.kms_enable, + kms_backend: o.kms_backend, + kms_key_dir: o.kms_key_dir, + kms_vault_address: o.kms_vault_address, + kms_vault_token: o.kms_vault_token, + kms_default_key_id: o.kms_default_key_id, + buffer_profile_disable: o.buffer_profile_disable, + buffer_profile: o.buffer_profile, + } + } + + /// Parse from preprocessed args. Supports both `rustfs ` and `rustfs server `. + #[allow(dead_code)] // used in config_test + pub fn parse_from(args: I) -> Self + where + I: IntoIterator, + T: Into + Clone, + { + let _ = apply_external_env_compat(); + let args: Vec = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect(); + let args = preprocess_args_for_legacy(args); + let cli = Cli::parse_from(args); + match cli.command { + Some(Commands::Server(opts)) => Self::from_server_opts(*opts), + Some(Commands::Info(_)) => { + // This should not happen in parse_from, as it's handled by parse_command + panic!("Info command should be handled by parse_command"); + } + None => { + // Default to server with empty volumes (will be filled from env) + Self::from_server_opts(default_server_opts()) + } + } + } + + /// Parse from preprocessed args and return the command type. + /// Returns Ok(Info(opts)) if info command, Ok(Server(opts)) if server command. + pub fn parse_command(args: I) -> Result + where + I: IntoIterator, + T: Into + Clone, + { + let _ = apply_external_env_compat(); + let args: Vec = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect(); + let args = preprocess_args_for_legacy(args); + let cli = Cli::try_parse_from(args)?; + match cli.command { + Some(Commands::Info(opts)) => Ok(CommandResult::Info(opts)), + Some(Commands::Server(opts)) => Self::server_command_result(Self::from_server_opts(*opts)), + None => { + // Default to server with empty volumes (will be filled from env) + Self::server_command_result(Self::from_server_opts(default_server_opts())) + } + } + } + + // Helper to convert Opt to CommandResult::Server with error handling + fn server_command_result(opt: Opt) -> Result { + Ok(Server(Box::new(Config::from_opt(opt).map_err(|e| { + clap::Error::raw(clap::error::ErrorKind::ValueValidation, e.to_string()) + })?))) + } + + /// Try parse from args, returns error on invalid input. + #[allow(dead_code)] // used in config_test + pub fn try_parse_from(args: I) -> Result + where + I: IntoIterator, + T: Into + Clone, + { + let _ = apply_external_env_compat(); + let args: Vec = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect(); + let args = preprocess_args_for_legacy(args); + let cli = Cli::try_parse_from(args)?; + match cli.command { + Some(Commands::Server(opts)) => Ok(Self::from_server_opts(*opts)), + Some(Commands::Info(_)) => Err(clap::Error::new(clap::error::ErrorKind::DisplayHelp)), + None => { + // Default to server with empty volumes + Ok(Self::from_server_opts(default_server_opts())) + } + } + } + + /// Parse from env::args(). Used by Config::parse(). + #[allow(dead_code)] // used in config_test + fn parse() -> Self { + let args: Vec = std::env::args().collect(); + Self::parse_from(args) + } +} diff --git a/rustfs/src/config/snapshot.rs b/rustfs/src/config/snapshot.rs new file mode 100644 index 000000000..d62d95be9 --- /dev/null +++ b/rustfs/src/config/snapshot.rs @@ -0,0 +1,164 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Configuration snapshot for info command. +//! +//! This module provides a lightweight snapshot of configuration values +//! that can be accessed globally without needing the full Config struct. + +use super::Config; +use crate::config::config_struct::resolve_credential; +use rustfs_config::{ + DEFAULT_ADDRESS, DEFAULT_BUFFER_PROFILE, DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_KMS_BACKEND, + DEFAULT_KMS_ENABLE, DEFAULT_OBS_ENDPOINT, ENV_RUSTFS_ACCESS_KEY, ENV_RUSTFS_ACCESS_KEY_FILE, ENV_RUSTFS_ADDRESS, + ENV_RUSTFS_BUFFER_PROFILE, ENV_RUSTFS_CONSOLE_ADDRESS, ENV_RUSTFS_CONSOLE_ENABLE, ENV_RUSTFS_KMS_BACKEND, + ENV_RUSTFS_KMS_ENABLE, ENV_RUSTFS_OBS_ENDPOINT, ENV_RUSTFS_REGION, ENV_RUSTFS_ROOT_USER, ENV_RUSTFS_TLS_PATH, RUSTFS_REGION, +}; +use rustfs_credentials::DEFAULT_ACCESS_KEY; +use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_str}; +use std::sync::OnceLock; + +/// Fallback snapshot used only for display when the global snapshot +/// has not yet been initialized (e.g., for the `--info` command). +/// This avoids leaking memory while still providing a `'static` reference. +static DISPLAY_CONFIG_SNAPSHOT: OnceLock = OnceLock::new(); + +/// Configuration snapshot for info command display. +/// This stores key configuration values that can be accessed without +/// needing the full Config struct. +#[derive(Clone, Debug)] +pub struct ConfigSnapshot { + /// Server bind address + pub address: String, + /// Console server enabled + pub console_enable: bool, + /// Console server address + pub console_address: String, + /// Server region + pub region: Option, + /// Access key (for display, should be masked) + pub access_key: String, + /// OBS endpoint + pub obs_endpoint: String, + /// TLS path + pub tls_path: Option, + /// KMS enabled + pub kms_enable: bool, + /// KMS backend type + pub kms_backend: String, + /// Buffer profile + pub buffer_profile: String, +} + +impl ConfigSnapshot { + /// Create a snapshot from Config + pub fn from_config(config: &Config) -> Self { + Self { + address: config.address.clone(), + console_enable: config.console_enable, + console_address: config.console_address.clone(), + region: config.region.clone(), + access_key: config.access_key.clone(), + obs_endpoint: config.obs_endpoint.clone(), + tls_path: config.tls_path.clone(), + kms_enable: config.kms_enable, + kms_backend: config.kms_backend.clone(), + buffer_profile: config.buffer_profile.clone(), + } + } + + /// Create a default snapshot from environment variables and defaults + pub fn from_env() -> Self { + let access_key = resolve_credential( + get_env_opt_str(ENV_RUSTFS_ACCESS_KEY), + get_env_opt_str(ENV_RUSTFS_ACCESS_KEY_FILE), + ENV_RUSTFS_ROOT_USER, + DEFAULT_ACCESS_KEY, + ) + .unwrap_or_else(|_| DEFAULT_ACCESS_KEY.to_string()); + Self { + address: get_env_str(ENV_RUSTFS_ADDRESS, DEFAULT_ADDRESS), + console_enable: get_env_bool(ENV_RUSTFS_CONSOLE_ENABLE, DEFAULT_CONSOLE_ENABLE), + console_address: get_env_str(ENV_RUSTFS_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ADDRESS), + region: Some(get_env_str(ENV_RUSTFS_REGION, RUSTFS_REGION)), + access_key, + obs_endpoint: get_env_str(ENV_RUSTFS_OBS_ENDPOINT, DEFAULT_OBS_ENDPOINT), + tls_path: get_env_opt_str(ENV_RUSTFS_TLS_PATH), + kms_enable: get_env_bool(ENV_RUSTFS_KMS_ENABLE, DEFAULT_KMS_ENABLE), + kms_backend: get_env_str(ENV_RUSTFS_KMS_BACKEND, DEFAULT_KMS_BACKEND), + buffer_profile: get_env_str(ENV_RUSTFS_BUFFER_PROFILE, DEFAULT_BUFFER_PROFILE), + } + } +} + +/// Global configuration snapshot storage +static GLOBAL_CONFIG_SNAPSHOT: OnceLock = OnceLock::new(); + +/// Initialize the global config snapshot from a Config instance. +/// This should be called once during server startup. +/// +/// This is the ONLY function that can set the global snapshot. +/// Once set, it cannot be changed. +pub fn init_config_snapshot(config: &Config) { + let snapshot = ConfigSnapshot::from_config(config); + if GLOBAL_CONFIG_SNAPSHOT.set(snapshot).is_err() { + // Already initialized, log a warning + tracing::warn!("Config snapshot already initialized, ignoring re-initialization"); + } +} + +/// Get the global config snapshot if initialized. +/// Returns None if not initialized (e.g., when running --info before server starts). +#[allow(dead_code)] // used in info command +pub fn get_config_snapshot() -> Option<&'static ConfigSnapshot> { + GLOBAL_CONFIG_SNAPSHOT.get() +} + +/// Check if the global config snapshot has been initialized. +#[allow(dead_code)] // may be used for debugging +pub fn is_config_snapshot_initialized() -> bool { + GLOBAL_CONFIG_SNAPSHOT.get().is_some() +} + +/// Get config snapshot for display purposes (backward-compatible wrapper). +/// +/// - If the global snapshot is initialized (server has started), returns a reference to it. +/// - If not initialized (e.g., --info command before server starts), returns a temporary +/// snapshot created from environment variables WITHOUT updating the global storage. +/// +/// Despite its name, this function no longer initializes the global snapshot; it simply +/// delegates to `get_config_snapshot_for_display` to ensure the global snapshot is ONLY set +/// by `init_config_snapshot` during server startup. +#[allow(dead_code)] // kept for backward compatibility +pub fn get_or_init_config_snapshot() -> &'static ConfigSnapshot { + get_config_snapshot_for_display() +} +/// Get config snapshot for display, without modifying global state. +/// +/// This function is used by the --info command to display configuration: +/// - Returns the global snapshot if initialized +/// - Otherwise creates a temporary snapshot from environment variables (does NOT store it) +/// +/// Note: This returns a static reference for API compatibility. When the global snapshot +/// is not initialized, it creates a leaked Box to provide a static lifetime. +/// This is safe because it's only used for read-only display purposes. +pub fn get_config_snapshot_for_display() -> &'static ConfigSnapshot { + if let Some(snapshot) = GLOBAL_CONFIG_SNAPSHOT.get() { + snapshot + } else { + // Not initialized - create from env without storing in the global snapshot. + // Use a dedicated OnceLock to cache a single display snapshot without leaking. + DISPLAY_CONFIG_SNAPSHOT.get_or_init(ConfigSnapshot::from_env) + } +} diff --git a/rustfs/src/config/workload_profiles.rs b/rustfs/src/config/workload_profiles.rs index a28afd840..e5426ba89 100644 --- a/rustfs/src/config/workload_profiles.rs +++ b/rustfs/src/config/workload_profiles.rs @@ -69,9 +69,10 @@ pub fn get_global_buffer_config() -> &'static RustFSBufferConfig { } /// Workload profile types that define buffer sizing strategies -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Default)] pub enum WorkloadProfile { /// General purpose - default configuration with balanced performance and memory + #[default] GeneralPurpose, /// AI/ML training: optimized for large sequential reads with maximum throughput AiTraining, @@ -84,7 +85,6 @@ pub enum WorkloadProfile { /// Secure storage: security first, memory constrained for compliance SecureStorage, /// Custom configuration for specialized requirements - #[allow(dead_code)] Custom(BufferConfig), } @@ -106,7 +106,6 @@ pub struct BufferConfig { #[derive(Debug, Clone)] pub struct RustFSBufferConfig { /// Selected workload profile - #[allow(dead_code)] pub workload: WorkloadProfile, /// Computed buffer configuration (either from profile or custom) pub base_config: BufferConfig, @@ -144,6 +143,25 @@ impl WorkloadProfile { } } + /// Create a custom workload profile with specified buffer configuration + /// + /// # Arguments + /// * `min_size` - Minimum buffer size in bytes + /// * `max_size` - Maximum buffer size in bytes + /// * `default_unknown` - Default size for unknown file size scenarios + /// * `thresholds` - File size thresholds and corresponding buffer sizes + /// + /// # Returns + /// A WorkloadProfile::Custom with the specified configuration + pub fn custom(min_size: usize, max_size: usize, default_unknown: usize, thresholds: Vec<(i64, usize)>) -> Self { + WorkloadProfile::Custom(BufferConfig { + min_size, + max_size, + default_unknown, + thresholds, + }) + } + /// Get the buffer configuration for this workload profile pub fn config(&self) -> BufferConfig { match self { @@ -309,7 +327,6 @@ impl BufferConfig { } /// Validate the buffer configuration - #[allow(dead_code)] pub fn validate(&self) -> Result<(), String> { if self.min_size == 0 { return Err("min_size must be greater than 0".to_string()); @@ -361,6 +378,29 @@ impl RustFSBufferConfig { pub fn get_buffer_size(&self, file_size: i64) -> usize { self.base_config.calculate_buffer_size(file_size) } + + /// Get the current workload profile + pub fn workload_profile(&self) -> &WorkloadProfile { + &self.workload + } + + /// Get the name of the current workload profile + pub fn workload_name(&self) -> String { + match &self.workload { + WorkloadProfile::GeneralPurpose => "GeneralPurpose".to_string(), + WorkloadProfile::AiTraining => "AiTraining".to_string(), + WorkloadProfile::DataAnalytics => "DataAnalytics".to_string(), + WorkloadProfile::WebWorkload => "WebWorkload".to_string(), + WorkloadProfile::IndustrialIoT => "IndustrialIoT".to_string(), + WorkloadProfile::SecureStorage => "SecureStorage".to_string(), + WorkloadProfile::Custom(_) => "Custom".to_string(), + } + } + + /// Validate the buffer configuration + pub fn validate(&self) -> Result<(), String> { + self.base_config.validate() + } } impl Default for RustFSBufferConfig { @@ -616,6 +656,41 @@ mod tests { assert_eq!(WorkloadProfile::from_name(""), WorkloadProfile::GeneralPurpose); } + #[test] + fn test_custom_workload_profile() { + // Create a custom profile with specific buffer sizes + let custom_profile = WorkloadProfile::custom( + 32 * KI_B, // min_size: 32KB + 2 * MI_B, // max_size: 2MB + 256 * KI_B, // default_unknown: 256KB + vec![ + (MI_B as i64, 64 * KI_B), // < 1MB: 64KB + (10 * MI_B as i64, 128 * KI_B), // 1MB-10MB: 128KB + (i64::MAX, 512 * KI_B), // >= 10MB: 512KB + ], + ); + + // Verify it's a Custom variant + match &custom_profile { + WorkloadProfile::Custom(config) => { + assert_eq!(config.min_size, 32 * KI_B); + assert_eq!(config.max_size, 2 * MI_B); + assert_eq!(config.default_unknown, 256 * KI_B); + assert_eq!(config.thresholds.len(), 3); + } + _ => panic!("Expected Custom variant"), + } + + // Test buffer size calculation with custom profile + let buffer_config = RustFSBufferConfig::new(custom_profile); + assert_eq!(buffer_config.get_buffer_size(500 * KI_B as i64), 64 * KI_B); + assert_eq!(buffer_config.get_buffer_size(5 * MI_B as i64), 128 * KI_B); + assert_eq!(buffer_config.get_buffer_size(100 * MI_B as i64), 512 * KI_B); + + // Test validation + assert!(buffer_config.validate().is_ok()); + } + #[test] fn test_global_buffer_config() { use super::{is_buffer_profile_enabled, set_buffer_profile_enabled}; diff --git a/rustfs/src/init.rs b/rustfs/src/init.rs index 7d203ae80..2032196d4 100644 --- a/rustfs/src/init.rs +++ b/rustfs/src/init.rs @@ -14,10 +14,14 @@ use crate::storage::{process_lambda_configurations, process_queue_configurations, process_topic_configurations}; use crate::{admin, config, version}; -use rustfs_config::{DEFAULT_UPDATE_CHECK, ENV_UPDATE_CHECK, RUSTFS_REGION}; +use rustfs_config::{ + DEFAULT_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MIN_SIZE, DEFAULT_BUFFER_PROFILE, DEFAULT_BUFFER_UNKNOWN_SIZE, DEFAULT_UPDATE_CHECK, + ENV_RUSTFS_BUFFER_DEFAULT_SIZE, ENV_RUSTFS_BUFFER_MAX_SIZE, ENV_RUSTFS_BUFFER_MIN_SIZE, ENV_UPDATE_CHECK, RUSTFS_REGION, +}; use rustfs_ecstore::bucket::metadata_sys; use rustfs_notify::notifier_global; use rustfs_targets::arn::{ARN, TargetIDError}; +use rustfs_utils::get_env_usize; use s3s::s3_error; use std::env; use std::io::Error; @@ -297,10 +301,9 @@ pub(crate) async fn init_kms_system(config: &config::Config) -> std::io::Result< /// # Arguments /// * `config` - The application configuration options pub(crate) fn init_buffer_profile_system(config: &config::Config) { - use crate::config::workload_profiles::{ - RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled, - }; + use crate::config::{RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled}; + // Whether buffer profiling is disabled or not, it is enabled by default, unless the user explicitly sets '--buffer-profile-disable' or 'RUSTFS_BUFFER_PROFILE_DISABLE=true' if config.buffer_profile_disable { // User explicitly disabled buffer profiling - use GeneralPurpose profile in disabled mode info!("Buffer profiling disabled via --buffer-profile-disable, using GeneralPurpose profile"); @@ -310,13 +313,59 @@ pub(crate) fn init_buffer_profile_system(config: &config::Config) { info!("Buffer profiling enabled with profile: {}", config.buffer_profile); // Parse the workload profile from configuration string - let profile = WorkloadProfile::from_name(&config.buffer_profile); + // Support a custom profile when buffer_profile is set to "custom"; + // its sizes are controlled via RUSTFS_BUFFER_MIN_SIZE, RUSTFS_BUFFER_MAX_SIZE, + // and RUSTFS_BUFFER_DEFAULT_SIZE environment variables. + let profile = if config.buffer_profile.eq_ignore_ascii_case("custom") { + // Try to create custom profile from environment variables + let min_size = get_env_usize(ENV_RUSTFS_BUFFER_MIN_SIZE, DEFAULT_BUFFER_MIN_SIZE); + let max_size = get_env_usize(ENV_RUSTFS_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MAX_SIZE); + let default_unknown = get_env_usize(ENV_RUSTFS_BUFFER_DEFAULT_SIZE, DEFAULT_BUFFER_UNKNOWN_SIZE); + + info!( + "Creating custom buffer profile: min={}, max={}, default={}", + min_size, max_size, default_unknown + ); + WorkloadProfile::custom( + min_size, + max_size, + default_unknown, + vec![ + (1024 * 1024, 64 * 1024), // < 1MB: 64KB + (100 * 1024 * 1024, 256 * 1024), // 1MB-100MB: 256KB + (i64::MAX, 1024 * 1024), // >= 100MB: 1MB + ], + ) + } else { + WorkloadProfile::from_name(&config.buffer_profile) + }; // Log the selected profile for operational visibility info!("Active buffer profile: {:?}", profile); + // Create and validate buffer configuration + let mut buffer_config = RustFSBufferConfig::new(profile); + if let Err(e) = buffer_config.validate() { + warn!("Buffer configuration validation failed: {}. Falling back to GeneralPurpose profile.", e); + // Fall back to a known-good profile to avoid installing an invalid configuration + let fallback_profile = WorkloadProfile::from_name(DEFAULT_BUFFER_PROFILE); + info!("Using fallback buffer profile: {:?}", fallback_profile); + let fallback_config = RustFSBufferConfig::new(fallback_profile); + if let Err(e2) = fallback_config.validate() { + error!( + "Fallback buffer configuration validation failed: {}. Aborting buffer profiling initialization.", + e2 + ); + panic!("Failed to initialize a valid RustFS buffer configuration"); + } + buffer_config = fallback_config; + } + + // Log the workload profile name + info!("Workload profile: {}", buffer_config.workload_name()); + // Initialize the global buffer configuration - init_global_buffer_config(RustFSBufferConfig::new(profile)); + init_global_buffer_config(buffer_config); // Enable buffer profiling globally set_buffer_profile_enabled(true); diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index b72265188..8daced71c 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -151,8 +151,30 @@ fn format_external_prefix_mappings(report: &ExternalEnvCompatReport) -> String { } async fn async_main() -> Result<()> { - // Parse the obtained parameters - let config = config::Config::parse()?; + // Parse command line arguments + let args: Vec = std::env::args().collect(); + let command_result = match config::Opt::parse_command(args) { + Ok(result) => result, + Err(e) => { + eprintln!("Command parse failed, error: {}", e); + std::process::exit(1); + } + }; + + // Handle info command + if let config::CommandResult::Info(opts) = command_result { + config::execute_info(&opts); + return Ok(()); + } + + // Get config for server command + let config = match command_result { + config::CommandResult::Server(cfg) => cfg, + config::CommandResult::Info(_) => unreachable!(), + }; + + // Initialize the global config snapshot for info command + config::init_config_snapshot(&config); // Initialize the configuration init_license(config.license.clone()); @@ -211,7 +233,7 @@ async fn async_main() -> Result<()> { } // Run parameters - match run(config).await { + match run(*config).await { Ok(_) => Ok(()), Err(e) => { error!("Server encountered an error and is shutting down: {}", e); diff --git a/rustfs/src/storage/ecfs_extend.rs b/rustfs/src/storage/ecfs_extend.rs index ef3be5b7e..25c400ac4 100644 --- a/rustfs/src/storage/ecfs_extend.rs +++ b/rustfs/src/storage/ecfs_extend.rs @@ -12,9 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::config::workload_profiles::{ - RustFSBufferConfig, WorkloadProfile, get_global_buffer_config, is_buffer_profile_enabled, -}; +use crate::config::{RustFSBufferConfig, WorkloadProfile, get_global_buffer_config, is_buffer_profile_enabled}; use crate::error::ApiError; use crate::server::cors; use crate::storage::ecfs::ListObjectUnorderedQuery; @@ -119,6 +117,22 @@ pub(crate) fn apply_lock_retention(object_lock_config: Option= 10MB: 512KB +/// ], +/// ); +/// let buffer_size = get_adaptive_buffer_size_with_profile( +/// 5 * 1024 * 1024, +/// Some(custom_profile) +/// ); /// ``` /// #[allow(dead_code)] diff --git a/rustfs/src/storage/ecfs_test.rs b/rustfs/src/storage/ecfs_test.rs index 7e2c9bdaa..f6eed5d4f 100644 --- a/rustfs/src/storage/ecfs_test.rs +++ b/rustfs/src/storage/ecfs_test.rs @@ -14,7 +14,7 @@ #[cfg(test)] mod tests { - use crate::config::workload_profiles::WorkloadProfile; + use crate::config::WorkloadProfile; use crate::server::cors; use crate::storage::ecfs::FS; use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner}; @@ -281,9 +281,7 @@ mod tests { #[test] fn test_phase3_default_behavior() { - use crate::config::workload_profiles::{ - RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled, - }; + use crate::config::{RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled}; const KB: i64 = 1024; const MB: i64 = 1024 * 1024; @@ -304,7 +302,7 @@ mod tests { #[test] fn test_buffer_size_opt_in() { - use crate::config::workload_profiles::{is_buffer_profile_enabled, set_buffer_profile_enabled}; + use crate::config::{is_buffer_profile_enabled, set_buffer_profile_enabled}; const KB: i64 = 1024; const MB: i64 = 1024 * 1024; @@ -336,9 +334,7 @@ mod tests { #[test] fn test_phase4_full_integration() { - use crate::config::workload_profiles::{ - RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled, - }; + use crate::config::{RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled}; const KB: i64 = 1024; const MB: i64 = 1024 * 1024;