feat: Add --info command and refactor config module (#2234)

Signed-off-by: houseme <housemecn@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
houseme
2026-03-20 01:16:45 +08:00
committed by GitHub
parent e1a278aaf8
commit 28f86a505e
19 changed files with 1839 additions and 482 deletions
+17
View File
@@ -60,6 +60,23 @@ impl GlobalReadiness {
pub fn is_ready(&self) -> bool {
self.status.load(Ordering::SeqCst) == SystemStage::FullReady as u8
}
/// Get the current system stage
/// # Returns
/// The current SystemStage of the service
pub fn current_stage(&self) -> SystemStage {
match self.status.load(Ordering::SeqCst) {
0 => SystemStage::Booting,
1 => SystemStage::StorageReady,
2 => SystemStage::IamReady,
3 => SystemStage::FullReady,
invalid => {
debug_assert!(false, "GlobalReadiness::current_stage: invalid status value {}", invalid);
// Fallback to the most conservative stage on invalid values
SystemStage::Booting
}
}
}
}
#[cfg(test)]
+52
View File
@@ -131,9 +131,61 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
/// Environment variable for server volumes.
pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
/// Environment variable for server access key.
pub const ENV_RUSTFS_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
/// Environment variable for server access key file.
pub const ENV_RUSTFS_ACCESS_KEY_FILE: &str = "RUSTFS_ACCESS_KEY_FILE";
/// Environment variable for server root user.
pub const ENV_RUSTFS_ROOT_USER: &str = "RUSTFS_ROOT_USER";
/// Environment variable for server secret key.
pub const ENV_RUSTFS_SECRET_KEY: &str = "RUSTFS_SECRET_KEY";
/// Environment variable for server secret key file.
pub const ENV_RUSTFS_SECRET_KEY_FILE: &str = "RUSTFS_SECRET_KEY_FILE";
/// Environment variable for server root password.
pub const ENV_RUSTFS_ROOT_PASSWORD: &str = "RUSTFS_ROOT_PASSWORD";
/// Environment variable for server OBS endpoint.
pub const ENV_RUSTFS_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
/// Environment variable for console server enable.
pub const ENV_RUSTFS_CONSOLE_ENABLE: &str = "RUSTFS_CONSOLE_ENABLE";
/// Environment variable for console server address.
pub const ENV_RUSTFS_CONSOLE_ADDRESS: &str = "RUSTFS_CONSOLE_ADDRESS";
/// Environment variable for server tls path.
pub const ENV_RUSTFS_TLS_PATH: &str = "RUSTFS_TLS_PATH";
/// Environment variable for server KMS enable.
pub const ENV_RUSTFS_KMS_ENABLE: &str = "RUSTFS_KMS_ENABLE";
/// Default KMS enable for server-side encryption
/// This is the default value for enabling KMS encryption for server-side encryption.
/// Default value: false
pub const DEFAULT_KMS_ENABLE: bool = false;
/// Environment variable for server KMS backend.
pub const ENV_RUSTFS_KMS_BACKEND: &str = "RUSTFS_KMS_BACKEND";
/// Default KMS backend for server-side encryption
/// This is the default KMS backend for server-side encryption.
/// Default value: local
pub const DEFAULT_KMS_BACKEND: &str = "local";
/// Environment variable for selecting the buffer profile used for adaptive buffer sizing.
pub const ENV_RUSTFS_BUFFER_PROFILE: &str = "RUSTFS_BUFFER_PROFILE";
/// Default buffer profile for adaptive buffer sizing
/// This is the default buffer profile for adaptive buffer sizing.
/// It is used to identify the workload profile for adaptive buffer sizing.
/// Default value: GeneralPurpose
pub const DEFAULT_BUFFER_PROFILE: &str = "GeneralPurpose";
/// Default value for the server TLS path if `ENV_RUSTFS_TLS_PATH` is not set.
pub const DEFAULT_RUSTFS_TLS_PATH: &str = "";
+1
View File
@@ -28,3 +28,4 @@ pub(crate) mod runtime;
pub(crate) mod scanner;
pub(crate) mod targets;
pub(crate) mod tls;
pub(crate) mod workload;
+70
View File
@@ -0,0 +1,70 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Workload profile buffer configuration constants.
//!
//! This module defines environment variable keys and default values for
//! custom buffer profile configuration.
use crate::{KI_B, MI_B};
/// Environment variable for minimum buffer size
/// Default: 64KB (65536 bytes)
pub const ENV_RUSTFS_BUFFER_MIN_SIZE: &str = "RUSTFS_BUFFER_MIN_SIZE";
/// Environment variable for maximum buffer size
/// Default: 1MB (1048576 bytes)
pub const ENV_RUSTFS_BUFFER_MAX_SIZE: &str = "RUSTFS_BUFFER_MAX_SIZE";
/// Environment variable for default buffer size (used when file size is unknown)
/// Default: 256KB (262144 bytes)
pub const ENV_RUSTFS_BUFFER_DEFAULT_SIZE: &str = "RUSTFS_BUFFER_DEFAULT_SIZE";
/// Default minimum buffer size: 64KB
pub const DEFAULT_BUFFER_MIN_SIZE: usize = 64 * KI_B;
/// Default maximum buffer size: 1MB
pub const DEFAULT_BUFFER_MAX_SIZE: usize = MI_B;
/// Default buffer size for unknown file size: 256KB
pub const DEFAULT_BUFFER_UNKNOWN_SIZE: usize = 256 * KI_B;
#[cfg(test)]
mod tests {
use super::*;
use crate::MI_B;
#[test]
fn test_default_values() {
assert_eq!(DEFAULT_BUFFER_MIN_SIZE, 65536); // 64KB
assert_eq!(DEFAULT_BUFFER_MAX_SIZE, 1048576); // 1MB
assert_eq!(DEFAULT_BUFFER_UNKNOWN_SIZE, 262144); // 256KB
}
#[test]
fn test_constants() {
assert_eq!(KI_B, 1024);
assert_eq!(MI_B, 1024 * 1024);
assert_eq!(64 * KI_B, DEFAULT_BUFFER_MIN_SIZE);
assert_eq!(MI_B, DEFAULT_BUFFER_MAX_SIZE);
assert_eq!(256 * KI_B, DEFAULT_BUFFER_UNKNOWN_SIZE);
}
#[test]
fn test_env_var_names() {
assert_eq!(ENV_RUSTFS_BUFFER_MIN_SIZE, "RUSTFS_BUFFER_MIN_SIZE");
assert_eq!(ENV_RUSTFS_BUFFER_MAX_SIZE, "RUSTFS_BUFFER_MAX_SIZE");
assert_eq!(ENV_RUSTFS_BUFFER_DEFAULT_SIZE, "RUSTFS_BUFFER_DEFAULT_SIZE");
}
}
+2
View File
@@ -45,6 +45,8 @@ pub use constants::targets::*;
#[cfg(feature = "constants")]
pub use constants::tls::*;
#[cfg(feature = "constants")]
pub use constants::workload::*;
#[cfg(feature = "constants")]
pub mod oidc {
pub use super::constants::oidc::*;
}
+1 -1
View File
@@ -13,9 +13,9 @@
// limitations under the License.
use crate::admin::handlers::health::{HealthProbe, build_component_details, collect_dependency_readiness, health_check_state};
use crate::config::build;
use crate::license::get_license;
use crate::server::{CONSOLE_PREFIX, FAVICON_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, RUSTFS_ADMIN_PREFIX};
use crate::version::build;
use axum::{
Router,
body::Body,
+1 -1
View File
@@ -16,7 +16,7 @@
//! This module introduces explicit dependency injection entry points
//! for storage, IAM, and KMS handles.
use crate::config::workload_profiles::{RustFSBufferConfig, get_global_buffer_config};
use crate::config::{RustFSBufferConfig, get_global_buffer_config};
use async_trait::async_trait;
use rustfs_ecstore::bucket::metadata_sys::{BucketMetadataSys, get_global_bucket_metadata_sys};
use rustfs_ecstore::config::{Config, get_global_server_config};
+1 -1
View File
@@ -15,7 +15,7 @@
//! Object application use-case contracts.
use crate::app::context::{AppContext, default_notify_interface, get_global_app_context};
use crate::config::workload_profiles::RustFSBufferConfig;
use crate::config::RustFSBufferConfig;
use crate::error::ApiError;
use crate::storage::access::{authorize_request, has_bypass_governance_header, req_info_mut};
use crate::storage::concurrency::{
+287
View File
@@ -0,0 +1,287 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! CLI definitions and parsing logic.
//!
//! This module contains the command-line interface definitions including:
//! - `Cli`: Main CLI parser
//! - `Commands`: Subcommands (Server, Info)
//! - `ServerOpts`: Server subcommand options
//! - `InfoOpts`: Info subcommand options
//! - `InfoType`: Information type enum
//! - `CommandResult`: Result of parsing command line arguments
use crate::version::build;
use clap::builder::NonEmptyStringValueParser;
use clap::{Args, Parser, Subcommand, ValueEnum};
use const_str::concat;
use rustfs_config::{DEFAULT_ADDRESS, DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_OBS_ENDPOINT, ENV_RUSTFS_VOLUMES};
use std::path::PathBuf;
// build module is re-exported from crate::build
#[allow(clippy::const_is_empty)]
pub(super) const SHORT_VERSION: &str = {
if !build::TAG.is_empty() {
build::TAG
} else if !build::SHORT_COMMIT.is_empty() {
concat!("@", build::SHORT_COMMIT)
} else {
build::PKG_VERSION
}
};
pub(super) const LONG_VERSION: &str = concat!(
concat!(SHORT_VERSION, "\n"),
concat!("build time : ", build::BUILD_TIME, "\n"),
concat!("build profile: ", build::BUILD_RUST_CHANNEL, "\n"),
concat!("build os : ", build::BUILD_OS, "\n"),
concat!("rust version : ", build::RUST_VERSION, "\n"),
concat!("rust channel : ", build::RUST_CHANNEL, "\n"),
concat!("git branch : ", build::BRANCH, "\n"),
concat!("git commit : ", build::COMMIT_HASH, "\n"),
concat!("git tag : ", build::TAG, "\n"),
concat!("git status :\n", build::GIT_STATUS_FILE),
);
/// Known subcommands. When the first arg matches one of these, it is treated as a subcommand.
pub const KNOWN_SUBCOMMANDS: &[&str] = &["server", "info"];
/// Preprocess argv for legacy compatibility: `rustfs <volume>` and `rustfs --address ...` are
/// treated as `rustfs server <volume>` and `rustfs server --address ...` respectively.
/// Also: `rustfs` with no args becomes `rustfs server` (volumes from env), and `rustfs --info`
/// is treated as `rustfs info`.
pub fn preprocess_args_for_legacy(args: Vec<String>) -> Vec<String> {
if args.len() < 2 {
// rustfs -> rustfs server (volumes from RUSTFS_VOLUMES env)
return vec![args[0].clone(), "server".to_string()];
}
let first = &args[1];
// If first arg looks like a subcommand, do nothing
if KNOWN_SUBCOMMANDS.contains(&first.as_str()) {
return args;
}
// If first arg is --info, treat it as info subcommand
if first == "--info" {
let mut out = vec![args[0].clone(), "info".to_string()];
out.extend(args[2..].iter().cloned());
return out;
}
// If first arg is a global flag (--help, --version), do nothing
if first == "--help" || first == "-h" || first == "--version" || first == "-V" {
return args;
}
// Legacy: rustfs <volume> or rustfs --address ... -> rustfs server <volume|--address ...>
let mut out = vec![args[0].clone(), "server".to_string()];
out.extend(args[1..].iter().cloned());
out
}
/// Main CLI parser
#[derive(Parser, Clone)]
#[command(name = "rustfs", version = SHORT_VERSION, long_version = LONG_VERSION)]
pub struct Cli {
#[command(subcommand)]
pub command: Option<Commands>,
}
/// Available subcommands
#[derive(Subcommand, Clone)]
pub enum Commands {
/// Start the object storage server (default when no subcommand is given)
Server(Box<ServerOpts>),
/// Display system information
Info(InfoOpts),
}
/// Information type to display
#[derive(ValueEnum, Clone, Copy, Debug, PartialEq, Eq)]
pub enum InfoType {
/// System basic information (OS, architecture, hostname, etc.)
System,
/// Runtime information (PID, memory, CPU, threads, etc.)
Runtime,
/// Build information (version, build time, git info, etc.)
Build,
/// Current configuration information
Config,
/// Dependency library versions
Deps,
}
/// Info subcommand options
#[derive(Args, Clone)]
pub struct InfoOpts {
/// Display all information types
#[arg(long, conflicts_with = "info_type")]
pub all: bool,
/// Type of information to display
#[arg(value_enum, conflicts_with = "all")]
pub info_type: Option<InfoType>,
}
/// Server subcommand options
#[derive(Args, Clone)]
pub struct ServerOpts {
/// DIR points to a directory on a filesystem.
#[arg(
required = true,
env = "RUSTFS_VOLUMES",
value_delimiter = ' ',
value_parser = NonEmptyStringValueParser::new()
)]
pub volumes: Vec<String>,
/// bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname
#[arg(
long,
default_value_t = rustfs_config::DEFAULT_ADDRESS.to_string(),
env = "RUSTFS_ADDRESS"
)]
pub address: String,
/// Domain name used for virtual-hosted-style requests.
#[arg(
long,
env = "RUSTFS_SERVER_DOMAINS",
value_delimiter = ',',
value_parser = NonEmptyStringValueParser::new()
)]
pub server_domains: Vec<String>,
/// Access key used for authentication.
#[arg(long, env = "RUSTFS_ACCESS_KEY", group = "access-key")]
pub access_key: Option<String>,
/// Access key stored in a file used for authentication.
#[arg(long, env = "RUSTFS_ACCESS_KEY_FILE", group = "access-key")]
pub access_key_file: Option<PathBuf>,
/// Secret key used for authentication.
#[arg(long, env = "RUSTFS_SECRET_KEY", group = "secret-key")]
pub secret_key: Option<String>,
/// Secret key stored in a file used for authentication.
#[arg(long, env = "RUSTFS_SECRET_KEY_FILE", group = "secret-key")]
pub secret_key_file: Option<PathBuf>,
/// Enable console server
#[arg(
long,
default_value_t = rustfs_config::DEFAULT_CONSOLE_ENABLE,
env = "RUSTFS_CONSOLE_ENABLE"
)]
pub console_enable: bool,
/// Console server bind address
#[arg(
long,
default_value_t = rustfs_config::DEFAULT_CONSOLE_ADDRESS.to_string(),
env = "RUSTFS_CONSOLE_ADDRESS"
)]
pub console_address: String,
/// Observability endpoint for trace, metrics and logs,only support grpc mode.
#[arg(
long,
default_value_t = rustfs_config::DEFAULT_OBS_ENDPOINT.to_string(),
env = "RUSTFS_OBS_ENDPOINT"
)]
pub obs_endpoint: String,
/// tls path for rustfs API and console.
#[arg(long, env = "RUSTFS_TLS_PATH")]
pub tls_path: Option<String>,
#[arg(long, env = "RUSTFS_LICENSE")]
pub license: Option<String>,
#[arg(long, env = "RUSTFS_REGION")]
pub region: Option<String>,
/// Enable KMS encryption for server-side encryption
#[arg(long, default_value_t = false, env = "RUSTFS_KMS_ENABLE")]
pub kms_enable: bool,
/// KMS backend type (local or vault)
#[arg(long, default_value_t = rustfs_config::DEFAULT_KMS_BACKEND.to_string(), env = "RUSTFS_KMS_BACKEND")]
pub kms_backend: String,
/// KMS key directory for local backend
#[arg(long, env = "RUSTFS_KMS_KEY_DIR")]
pub kms_key_dir: Option<String>,
/// Vault address for vault backend
#[arg(long, env = "RUSTFS_KMS_VAULT_ADDRESS")]
pub kms_vault_address: Option<String>,
/// Vault token for vault backend
#[arg(long, env = "RUSTFS_KMS_VAULT_TOKEN")]
pub kms_vault_token: Option<String>,
/// Default KMS key ID for encryption
#[arg(long, env = "RUSTFS_KMS_DEFAULT_KEY_ID")]
pub kms_default_key_id: Option<String>,
/// Disable adaptive buffer sizing with workload profiles
/// Set this flag to use legacy fixed-size buffer behavior from PR #869
#[arg(long, default_value_t = false, env = "RUSTFS_BUFFER_PROFILE_DISABLE")]
pub buffer_profile_disable: bool,
/// Workload profile for adaptive buffer sizing
/// Options: GeneralPurpose, AiTraining, DataAnalytics, WebWorkload, IndustrialIoT, SecureStorage
#[arg(long, default_value_t = rustfs_config::DEFAULT_BUFFER_PROFILE.to_string(), env = "RUSTFS_BUFFER_PROFILE")]
pub buffer_profile: String,
}
/// Result of parsing command line arguments
#[derive(Clone)]
pub enum CommandResult {
/// Server command with configuration
Server(Box<super::Config>),
/// Info command with options
Info(InfoOpts),
}
/// Create default ServerOpts from environment variables
pub fn default_server_opts() -> ServerOpts {
ServerOpts {
volumes: std::env::var(ENV_RUSTFS_VOLUMES)
.unwrap_or_default()
.split(' ')
.filter(|s| !s.is_empty())
.map(|s| s.to_string())
.collect(),
address: DEFAULT_ADDRESS.to_string(),
server_domains: vec![],
access_key: None,
access_key_file: None,
secret_key: None,
secret_key_file: None,
console_enable: DEFAULT_CONSOLE_ENABLE,
console_address: DEFAULT_CONSOLE_ADDRESS.to_string(),
obs_endpoint: DEFAULT_OBS_ENDPOINT.to_string(),
tls_path: None,
license: None,
region: None,
kms_enable: false,
kms_backend: "local".to_string(),
kms_key_dir: None,
kms_vault_address: None,
kms_vault_token: None,
kms_default_key_id: None,
buffer_profile_disable: false,
buffer_profile: "GeneralPurpose".to_string(),
}
}
+205
View File
@@ -0,0 +1,205 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Server configuration.
//!
//! This module contains the `Config` struct which holds the final server configuration
//! after processing command line arguments, environment variables, and files.
use super::Opt;
use crate::apply_external_env_compat;
use rustfs_config::{ENV_RUSTFS_ROOT_PASSWORD, ENV_RUSTFS_ROOT_USER, RUSTFS_REGION};
use rustfs_credentials::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, Masked};
/// Helper function to resolve credentials from multiple sources with precedence:
/// 1. Inline value (if provided)
/// 2. File value (if provided, read the content of the file)
/// 3. Environment variable (if set)
/// 4. Default value (if none of the above are provided)
pub(crate) fn resolve_credential<T: AsRef<std::path::Path>>(
inline_value: Option<String>,
file_value: Option<T>,
env_key: &str,
default_value: &str,
) -> std::io::Result<String> {
let value = inline_value
.map(Ok)
.or_else(|| file_value.map(std::fs::read_to_string))
.or_else(|| rustfs_utils::get_env_opt_str(env_key).map(Ok))
.transpose()?
.unwrap_or_else(|| default_value.to_string());
Ok(value.trim().to_string())
}
/// Server configuration.
///
/// This struct holds all configuration values needed to run the server.
/// It is created from `Opt` which is parsed from command line arguments.
#[derive(Clone)]
pub struct Config {
/// DIR points to a directory on a filesystem.
pub volumes: Vec<String>,
/// bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname
pub address: String,
/// Domain name used for virtual-hosted-style requests.
pub server_domains: Vec<String>,
/// Access key used for authentication.
pub access_key: String,
/// Secret key used for authentication.
pub secret_key: String,
/// Enable console server
pub console_enable: bool,
/// Console server bind address
pub console_address: String,
/// Observability endpoint for trace, metrics and logs,only support grpc mode.
pub obs_endpoint: String,
/// tls path for rustfs API and console.
pub tls_path: Option<String>,
/// License key for enterprise features
pub license: Option<String>,
/// Region for the server, used for signing and other region-specific behavior
pub region: Option<String>,
/// Enable KMS encryption for server-side encryption
pub kms_enable: bool,
/// KMS backend type (local or vault)
pub kms_backend: String,
/// KMS key directory for local backend
pub kms_key_dir: Option<String>,
/// Vault address for vault backend
pub kms_vault_address: Option<String>,
/// Vault token for vault backend
pub kms_vault_token: Option<String>,
/// Default KMS key ID for encryption
pub kms_default_key_id: Option<String>,
/// Disable adaptive buffer sizing with workload profiles
pub buffer_profile_disable: bool,
/// Workload profile for adaptive buffer sizing
pub buffer_profile: String,
}
impl Config {
/// Create Config from Opt
pub(super) fn from_opt(opt: Opt) -> std::io::Result<Self> {
let Opt {
volumes,
address,
server_domains,
access_key,
access_key_file,
secret_key,
secret_key_file,
console_enable,
console_address,
obs_endpoint,
tls_path,
license,
region,
kms_enable,
kms_backend,
kms_key_dir,
kms_vault_address,
kms_vault_token,
kms_default_key_id,
buffer_profile_disable,
buffer_profile,
} = opt;
let access_key = resolve_credential(access_key, access_key_file.as_ref(), ENV_RUSTFS_ROOT_USER, DEFAULT_ACCESS_KEY)?;
let secret_key = resolve_credential(secret_key, secret_key_file.as_ref(), ENV_RUSTFS_ROOT_PASSWORD, DEFAULT_SECRET_KEY)?;
// Region is optional, but if not set, we should default to "us-east-1" for signing compatibility with AWS S3 clients
let region = region.or_else(|| Some(RUSTFS_REGION.to_string()));
Ok(Config {
volumes,
address,
server_domains,
access_key,
secret_key,
console_enable,
console_address,
obs_endpoint,
tls_path,
license,
region,
kms_enable,
kms_backend,
kms_key_dir,
kms_vault_address,
kms_vault_token,
kms_default_key_id,
buffer_profile_disable,
buffer_profile,
})
}
/// Parse the command line arguments and environment arguments from [`Opt`] and convert them
/// into a ready to use [`Config`].
///
/// Supports both `rustfs <volume>` (legacy) and `rustfs server <volume>`.
///
/// This includes some intermediate checks for mutually exclusive options.
#[allow(dead_code)] // used in config_test
pub fn parse() -> std::io::Result<Self> {
let _ = apply_external_env_compat();
let args: Vec<String> = std::env::args().collect();
let opt = Opt::parse_from(args);
Self::from_opt(opt)
}
}
impl std::fmt::Debug for Config {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Config")
.field("volumes", &self.volumes)
.field("address", &self.address)
.field("server_domains", &self.server_domains)
.field("access_key", &self.access_key)
.field("secret_key", &Masked(Some(&self.secret_key))) // Hide sensitive values
.field("console_enable", &self.console_enable)
.field("console_address", &self.console_address)
.field("obs_endpoint", &self.obs_endpoint)
.field("tls_path", &self.tls_path)
.field("license", &Masked(self.license.as_deref()))
.field("region", &self.region)
.field("kms_enable", &self.kms_enable)
.field("kms_backend", &self.kms_backend)
.field("kms_key_dir", &self.kms_key_dir)
.field("kms_vault_address", &self.kms_vault_address)
.field("kms_vault_token", &Masked(self.kms_vault_token.as_deref()))
.field("kms_default_key_id", &self.kms_default_key_id)
.field("buffer_profile_disable", &self.buffer_profile_disable)
.field("buffer_profile", &self.buffer_profile)
.finish()
}
}
+656
View File
@@ -0,0 +1,656 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! System information display module.
//!
//! This module provides the `--info` command functionality for querying and displaying
//! various system information including system basics, runtime stats, build info,
//! configuration, and dependencies.
use super::{InfoOpts, InfoType};
use crate::version::build;
use rustfs_credentials::Masked;
use std::fmt;
/// CPU information
struct CpuInfo {
/// Number of logical CPU cores
core_count: usize,
/// CPU brand/vendor name
brand: String,
/// CPU frequency in MHz
frequency_mhz: u64,
/// CPU usage percentage (0-100)
usage_percent: f64,
}
impl CpuInfo {
fn collect(sys: &sysinfo::System) -> Self {
let core_count = sys.cpus().len();
let brand = sys
.cpus()
.first()
.map(|c| c.brand().to_string())
.unwrap_or_else(|| "Unknown".to_string());
let frequency_mhz = sys.cpus().first().map(|c| c.frequency()).unwrap_or(0);
// Calculate average CPU usage
let usage_percent = if core_count > 0 {
sys.cpus().iter().map(|c| c.cpu_usage() as f64).sum::<f64>() / core_count as f64
} else {
0.0
};
Self {
core_count,
brand,
frequency_mhz,
usage_percent,
}
}
fn format(&self) -> String {
format!(
"CPU Cores: {}\n\
CPU Brand: {}\n\
CPU Frequency: {} MHz\n\
CPU Usage: {:.1}%",
self.core_count, self.brand, self.frequency_mhz, self.usage_percent
)
}
}
/// Memory information
struct MemoryInfo {
/// Total system memory in bytes
total_bytes: u64,
/// Used memory in bytes
used_bytes: u64,
/// Available/free memory in bytes
available_bytes: u64,
/// Total swap memory in bytes
total_swap_bytes: u64,
/// Used swap memory in bytes
used_swap_bytes: u64,
/// Memory usage percentage
usage_percent: f64,
}
impl MemoryInfo {
fn collect(sys: &sysinfo::System) -> Self {
let total_bytes = sys.total_memory();
let used_bytes = sys.used_memory();
let available_bytes = sys.available_memory();
let total_swap_bytes = sys.total_swap();
let used_swap_bytes = sys.used_swap();
let usage_percent = if total_bytes > 0 {
(used_bytes as f64 / total_bytes as f64) * 100.0
} else {
0.0
};
Self {
total_bytes,
used_bytes,
available_bytes,
total_swap_bytes,
used_swap_bytes,
usage_percent,
}
}
fn format_bytes(bytes: u64) -> String {
const KB: u64 = 1024;
const MB: u64 = KB * 1024;
const GB: u64 = MB * 1024;
const TB: u64 = GB * 1024;
if bytes >= TB {
format!("{:.2} TB", bytes as f64 / TB as f64)
} else if bytes >= GB {
format!("{:.2} GB", bytes as f64 / GB as f64)
} else if bytes >= MB {
format!("{:.2} MB", bytes as f64 / MB as f64)
} else if bytes >= KB {
format!("{:.2} KB", bytes as f64 / KB as f64)
} else {
format!("{} B", bytes)
}
}
fn format(&self) -> String {
format!(
"Total Memory: {}\n\
Used Memory: {} ({:.1}%)\n\
Available Memory: {}\n\
Total Swap: {}\n\
Used Swap: {}",
Self::format_bytes(self.total_bytes),
Self::format_bytes(self.used_bytes),
self.usage_percent,
Self::format_bytes(self.available_bytes),
Self::format_bytes(self.total_swap_bytes),
Self::format_bytes(self.used_swap_bytes)
)
}
}
/// Disk information
struct DiskInfo {
/// Disk mount point
mount_point: String,
/// Disk name/device
name: String,
/// File system type
file_system: String,
/// Total space in bytes
total_bytes: u64,
/// Used space in bytes
used_bytes: u64,
/// Available space in bytes
available_bytes: u64,
/// Usage percentage
usage_percent: f64,
/// Is this a removable disk
is_removable: bool,
}
impl DiskInfo {
fn collect_all() -> Vec<Self> {
let disks = sysinfo::Disks::new_with_refreshed_list();
disks
.iter()
.map(|disk| {
let total_bytes = disk.total_space();
let available_bytes = disk.available_space();
let used_bytes = total_bytes.saturating_sub(available_bytes);
let usage_percent = if total_bytes > 0 {
(used_bytes as f64 / total_bytes as f64) * 100.0
} else {
0.0
};
Self {
mount_point: disk.mount_point().to_string_lossy().to_string(),
name: disk.name().to_string_lossy().to_string(),
file_system: format!("{:?}", disk.file_system()),
total_bytes,
used_bytes,
available_bytes,
usage_percent,
is_removable: disk.is_removable(),
}
})
.collect()
}
fn format(&self) -> String {
format!(
" [{}] {}\n\
Mount: {}\n\
Type: {}\n\
Total: {}\n\
Used: {} ({:.1}%)\n\
Available: {}\n\
Removable: {}",
if self.is_removable { "R" } else { "F" },
self.name,
self.mount_point,
self.file_system,
MemoryInfo::format_bytes(self.total_bytes),
MemoryInfo::format_bytes(self.used_bytes),
self.usage_percent,
MemoryInfo::format_bytes(self.available_bytes),
self.is_removable
)
}
}
/// Service disk information (disk where the service data is stored)
struct ServiceDiskInfo {
/// Disk mount point
mount_point: String,
/// Disk name/device
name: String,
/// File system type
file_system: String,
/// Total space in bytes
total_bytes: u64,
/// Used space in bytes
used_bytes: u64,
/// Available space in bytes
available_bytes: u64,
/// Usage percentage
usage_percent: f64,
/// Volume paths served by this disk
volume_paths: Vec<String>,
}
impl ServiceDiskInfo {
fn collect(volumes: &[String]) -> Option<Self> {
// Find the disk that contains the first volume path
let first_volume = volumes.first()?;
let volume_path = std::path::Path::new(first_volume);
// Find the disk that contains this volume
let disks = sysinfo::Disks::new_with_refreshed_list();
for disk in disks.iter() {
let mount_point = disk.mount_point();
if volume_path.starts_with(mount_point) {
let total_bytes = disk.total_space();
let available_bytes = disk.available_space();
let used_bytes = total_bytes.saturating_sub(available_bytes);
let usage_percent = if total_bytes > 0 {
(used_bytes as f64 / total_bytes as f64) * 100.0
} else {
0.0
};
// Find all volumes on this disk
let volume_paths: Vec<String> = volumes
.iter()
.filter(|v| std::path::Path::new(v).starts_with(mount_point))
.cloned()
.collect();
return Some(Self {
mount_point: mount_point.to_string_lossy().to_string(),
name: disk.name().to_string_lossy().to_string(),
file_system: format!("{:?}", disk.file_system()),
total_bytes,
used_bytes,
available_bytes,
usage_percent,
volume_paths,
});
}
}
None
}
fn format(&self) -> String {
let volumes_str = if self.volume_paths.is_empty() {
"(none)".to_string()
} else {
self.volume_paths.join(", ")
};
format!(
"=== Service Disk Information ===\n\
Disk: {}\n\
Mount Point: {}\n\
File System: {}\n\
Total Space: {}\n\
Used Space: {} ({:.1}%)\n\
Available Space: {}\n\
Served Volumes: {}",
self.name,
self.mount_point,
self.file_system,
MemoryInfo::format_bytes(self.total_bytes),
MemoryInfo::format_bytes(self.used_bytes),
self.usage_percent,
MemoryInfo::format_bytes(self.available_bytes),
volumes_str
)
}
}
/// System basic information
struct SystemInfo {
os_type: String,
os_version: String,
architecture: String,
hostname: String,
kernel_version: String,
/// CPU information
cpu: CpuInfo,
/// Memory information
memory: MemoryInfo,
/// All disk information
disks: Vec<DiskInfo>,
}
impl SystemInfo {
fn collect() -> Self {
// Create system info collector
let mut sys = sysinfo::System::new_all();
sys.refresh_all();
sys.refresh_cpu_all();
Self {
os_type: sysinfo::System::distribution_id(),
os_version: sysinfo::System::long_os_version().unwrap_or_else(|| "Unknown".to_string()),
architecture: std::env::consts::ARCH.to_string(),
hostname: sysinfo::System::host_name().unwrap_or_else(|| "Unknown".to_string()),
kernel_version: sysinfo::System::kernel_long_version(),
cpu: CpuInfo::collect(&sys),
memory: MemoryInfo::collect(&sys),
disks: DiskInfo::collect_all(),
}
}
fn format(&self) -> String {
let disks_str = if self.disks.is_empty() {
" (no disks found)".to_string()
} else {
self.disks.iter().map(|d| d.format()).collect::<Vec<_>>().join("\n")
};
format!(
"=== System Information ===\n\
OS: {}\n\
OS Version: {}\n\
Architecture: {}\n\
Hostname: {}\n\
Kernel Version: {}\n\
\n\
=== CPU Information ===\n\
{}\n\
\n\
=== Memory Information ===\n\
{}\n\
\n\
=== Disk Information ===\n\
{}",
self.os_type,
self.os_version,
self.architecture,
self.hostname,
self.kernel_version,
self.cpu.format(),
self.memory.format(),
disks_str
)
}
}
impl fmt::Display for SystemInfo {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.format())
}
}
/// Runtime information
struct RuntimeInfo {
process_id: u32,
memory_usage_mb: f64,
cpu_usage_percent: f64,
thread_count: usize,
}
impl RuntimeInfo {
fn collect() -> Self {
let (process_id, memory_usage_mb, cpu_usage_percent) = if let Ok(pid) = sysinfo::get_current_pid() {
let mut sys = sysinfo::System::new();
sys.refresh_processes_specifics(
sysinfo::ProcessesToUpdate::Some(&[pid]),
true,
sysinfo::ProcessRefreshKind::everything(),
);
let process = sys.process(pid);
let (memory_usage_mb, cpu_usage_percent) = if let Some(p) = process {
let memory = p.memory() as f64 / 1024.0 / 1024.0; // Convert to MB
let cpu = p.cpu_usage() as f64;
(memory, cpu)
} else {
(0.0, 0.0)
};
(pid.as_u32(), memory_usage_mb, cpu_usage_percent)
} else {
// Failed to retrieve current PID; degrade gracefully by
// skipping per-process stats and using sentinel values.
(0, 0.0, 0.0)
};
// // Get available CPU parallelism (roughly, logical cores available to the process)
let cpu_parallelism = std::thread::available_parallelism().map(|p| p.get()).unwrap_or(1);
Self {
process_id,
memory_usage_mb,
cpu_usage_percent,
thread_count: cpu_parallelism,
}
}
fn format(&self) -> String {
let pid_display = if self.process_id == 0 {
"unknown".to_string()
} else {
self.process_id.to_string()
};
format!(
"=== Runtime Information ===\n\
Process ID: {}\n\
Memory Usage: {:.2} MB\n\
CPU Usage: {:.2}%\n\
CPU Parallelism (logical cores): {}",
pid_display, self.memory_usage_mb, self.cpu_usage_percent, self.thread_count
)
}
}
impl fmt::Display for RuntimeInfo {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.format())
}
}
/// Build information (using shadow-rs generated constants)
struct BuildInfo;
impl BuildInfo {
fn format() -> String {
format!(
"=== Build Information ===\n\
Version: {}\n\
Build Time: {}\n\
Build Profile: {}\n\
Build OS: {}\n\
Rust Version: {}\n\
Git Branch: {}\n\
Git Commit: {}\n\
Git Tag: {}\n\
Git Status: {}",
build::PKG_VERSION,
build::BUILD_TIME,
build::BUILD_RUST_CHANNEL,
build::BUILD_OS,
build::RUST_VERSION,
build::BRANCH,
build::COMMIT_HASH,
build::TAG,
build::GIT_STATUS_FILE
)
}
}
impl fmt::Display for BuildInfo {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", Self::format())
}
}
/// Configuration information display
fn format_config_info() -> String {
// Get config snapshot for display (from global if initialized, otherwise from env)
let snapshot = super::get_config_snapshot_for_display();
// Get workload profile info
let workload_info = get_workload_profile_info();
// Mask the access key for display
let masked_access_key = &Masked(Some(&snapshot.access_key));
format!(
"=== Configuration Information ===\n\
Server Address: {}\n\
Console Enable: {}\n\
Console Address: {}\n\
Region: {}\n\
Access Key: {}\n\
Secret Key: ****\n\
OBS Endpoint: {}\n\
TLS Path: {}\n\
KMS Enabled: {}\n\
KMS Backend: {}\n\
Buffer Profile: {}\n\
{}",
snapshot.address,
snapshot.console_enable,
snapshot.console_address,
snapshot.region.as_deref().unwrap_or("(not set)"),
masked_access_key,
if snapshot.obs_endpoint.is_empty() {
"(not set)"
} else {
&snapshot.obs_endpoint
},
snapshot.tls_path.as_deref().unwrap_or("(not set)"),
snapshot.kms_enable,
snapshot.kms_backend,
snapshot.buffer_profile,
workload_info
)
}
/// Get workload profile information from global buffer config
fn get_workload_profile_info() -> String {
use super::workload_profiles::{get_global_buffer_config, is_buffer_profile_enabled};
if !is_buffer_profile_enabled() {
return "Workload Profile: (disabled)".to_string();
}
let config = get_global_buffer_config();
let profile = config.workload_profile();
let name = config.workload_name();
let buffer_config = profile.config();
format!(
"Workload Profile: {}\n\
Buffer Min Size: {} bytes\n\
Buffer Max Size: {} bytes\n\
Default Unknown: {} bytes",
name, buffer_config.min_size, buffer_config.max_size, buffer_config.default_unknown
)
}
/// Dependency information
fn format_deps_info() -> String {
let mut output = String::from("=== Build Features ===\n");
// Check which features are enabled at compile time
let features = [
("metrics", cfg!(feature = "metrics"), "Metrics collection and reporting"),
("ftps", cfg!(feature = "ftps"), "FTPS protocol support"),
("swift", cfg!(feature = "swift"), "Swift storage backend"),
("webdav", cfg!(feature = "webdav"), "WebDAV protocol support"),
("license", cfg!(feature = "license"), "License validation"),
("full", cfg!(feature = "full"), "All features enabled"),
];
let enabled_count = features.iter().filter(|(_, enabled, _)| *enabled).count();
output.push_str(&format!("Enabled Features: {}/{}\n\n", enabled_count, features.len()));
output.push_str("Feature Status:\n");
for (name, enabled, description) in features {
let status = if enabled { "[x]" } else { "[ ]" };
output.push_str(&format!(" {} {} - {}\n", status, name, description));
}
// Show default features info
output.push_str("\n--- Default Features ---\n");
output.push_str(" metrics (enabled by default)\n");
// Show feature dependencies
output.push_str("\n--- Feature Dependencies ---\n");
output.push_str(" full = metrics + ftps + swift + webdav\n");
output.push_str(" ftps -> rustfs-protocols/ftps\n");
output.push_str(" swift -> rustfs-protocols/swift\n");
output.push_str(" webdav -> rustfs-protocols/webdav\n");
output
}
/// Execute the info command
pub fn execute_info(opts: &InfoOpts) {
execute_info_with_volumes(opts, &[])
}
/// Execute info command with volume paths for service disk information
pub fn execute_info_with_volumes(opts: &InfoOpts, volumes: &[String]) {
let info_type = if opts.all {
None // None means display all
} else {
opts.info_type
};
match info_type {
None => {
// Display all information
println!("{}", SystemInfo::collect());
println!();
println!("{}", RuntimeInfo::collect());
println!();
println!("{}", BuildInfo);
println!();
println!("{}", format_config_info());
println!();
// Display service disk info if volumes are configured
if let Some(service_disk) = ServiceDiskInfo::collect(volumes) {
println!("{}", service_disk.format());
println!();
}
println!("{}", format_deps_info());
}
Some(InfoType::System) => {
println!("{}", SystemInfo::collect());
}
Some(InfoType::Runtime) => {
println!("{}", RuntimeInfo::collect());
}
Some(InfoType::Build) => {
println!("{}", BuildInfo);
}
Some(InfoType::Config) => {
println!("{}", format_config_info());
}
Some(InfoType::Deps) => {
println!("{}", format_deps_info());
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_system_info_collect() {
let info = SystemInfo::collect();
assert!(!info.os_type.is_empty());
assert!(!info.architecture.is_empty());
}
#[test]
fn test_runtime_info_collect() {
let info = RuntimeInfo::collect();
assert!(info.process_id > 0);
}
}
+41 -455
View File
@@ -12,464 +12,50 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use clap::builder::NonEmptyStringValueParser;
use clap::{Args, Parser, Subcommand};
use const_str::concat;
use rustfs_config::RUSTFS_REGION;
use rustfs_utils::{apply_external_env_compat, get_env_opt_str};
use std::path::PathBuf;
use std::string::ToString;
//! Configuration module for RustFS.
//!
//! This module is organized into the following submodules:
//!
//! - [`cli`]: Command-line interface definitions (Cli, Commands, ServerOpts, InfoOpts)
//! - [`opt`]: Parsed server options (Opt) and parsing methods
//! - [`config_struct`]: Server configuration (Config)
//! - [`snapshot`]: Configuration snapshot for info command
//! - [`info`]: Info command execution
//!
//! # Usage
//!
//! ```ignore
//! use rustfs::config::{Config, Opt, CommandResult};
//!
//! // Parse command line arguments
//! let result = Opt::parse_command(std::env::args())?;
//!
//! match result {
//! CommandResult::Server(config) => {
//! // Start server with config
//! }
//! CommandResult::Info(opts) => {
//! // Display info
//! }
//! }
//! ```
shadow_rs::shadow!(build);
pub mod workload_profiles;
mod cli;
mod config_struct;
mod info;
mod opt;
mod snapshot;
#[cfg(test)]
mod config_test;
#[allow(clippy::const_is_empty)]
const SHORT_VERSION: &str = {
if !build::TAG.is_empty() {
build::TAG
} else if !build::SHORT_COMMIT.is_empty() {
concat!("@", build::SHORT_COMMIT)
} else {
build::PKG_VERSION
}
};
// Re-export public types
pub use cli::{CommandResult, InfoOpts, InfoType};
pub use config_struct::Config;
pub use info::execute_info;
pub use opt::Opt;
pub use snapshot::{get_config_snapshot_for_display, init_config_snapshot};
const LONG_VERSION: &str = concat!(
concat!(SHORT_VERSION, "\n"),
concat!("build time : ", build::BUILD_TIME, "\n"),
concat!("build profile: ", build::BUILD_RUST_CHANNEL, "\n"),
concat!("build os : ", build::BUILD_OS, "\n"),
concat!("rust version : ", build::RUST_VERSION, "\n"),
concat!("rust channel : ", build::RUST_CHANNEL, "\n"),
concat!("git branch : ", build::BRANCH, "\n"),
concat!("git commit : ", build::COMMIT_HASH, "\n"),
concat!("git tag : ", build::TAG, "\n"),
concat!("git status :\n", build::GIT_STATUS_FILE),
);
/// Known subcommands. When the first arg matches one of these, it is treated as a subcommand.
const KNOWN_SUBCOMMANDS: &[&str] = &["server"];
/// Preprocess argv for legacy compatibility: `rustfs <volume>` and `rustfs --address ...` are
/// treated as `rustfs server <volume>` and `rustfs server --address ...` respectively.
/// Also: `rustfs` with no args becomes `rustfs server` (volumes from env).
fn preprocess_args_for_legacy(args: Vec<String>) -> Vec<String> {
if args.len() < 2 {
// rustfs -> rustfs server (volumes from RUSTFS_VOLUMES env)
return vec![args[0].clone(), "server".to_string()];
}
let first = &args[1];
// If first arg looks like a subcommand, do nothing
if KNOWN_SUBCOMMANDS.contains(&first.as_str()) {
return args;
}
// If first arg is a global flag (--help, --version), do nothing
if first == "--help" || first == "-h" || first == "--version" || first == "-V" {
return args;
}
// Legacy: rustfs <volume> or rustfs --address ... -> rustfs server <volume|--address ...>
let mut out = vec![args[0].clone(), "server".to_string()];
out.extend(args[1..].iter().cloned());
out
}
#[derive(Parser, Clone)]
#[command(name = "rustfs", version = SHORT_VERSION, long_version = LONG_VERSION)]
struct Cli {
#[command(subcommand)]
command: Option<Commands>,
}
#[derive(Subcommand, Clone)]
enum Commands {
/// Start the object storage server (default when no subcommand is given)
Server(ServerOpts),
}
#[derive(Args, Clone)]
struct ServerOpts {
/// DIR points to a directory on a filesystem.
#[arg(
required = true,
env = "RUSTFS_VOLUMES",
value_delimiter = ' ',
value_parser = NonEmptyStringValueParser::new()
)]
pub volumes: Vec<String>,
/// bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname
#[arg(
long,
default_value_t = rustfs_config::DEFAULT_ADDRESS.to_string(),
env = "RUSTFS_ADDRESS"
)]
pub address: String,
/// Domain name used for virtual-hosted-style requests.
#[arg(
long,
env = "RUSTFS_SERVER_DOMAINS",
value_delimiter = ',',
value_parser = NonEmptyStringValueParser::new()
)]
pub server_domains: Vec<String>,
/// Access key used for authentication.
#[arg(long, env = "RUSTFS_ACCESS_KEY", group = "access-key")]
pub access_key: Option<String>,
/// Access key stored in a file used for authentication.
#[arg(long, env = "RUSTFS_ACCESS_KEY_FILE", group = "access-key")]
pub access_key_file: Option<PathBuf>,
/// Secret key used for authentication.
#[arg(long, env = "RUSTFS_SECRET_KEY", group = "secret-key")]
pub secret_key: Option<String>,
/// Secret key stored in a file used for authentication.
#[arg(long, env = "RUSTFS_SECRET_KEY_FILE", group = "secret-key")]
pub secret_key_file: Option<PathBuf>,
/// Enable console server
#[arg(
long,
default_value_t = rustfs_config::DEFAULT_CONSOLE_ENABLE,
env = "RUSTFS_CONSOLE_ENABLE"
)]
pub console_enable: bool,
/// Console server bind address
#[arg(
long,
default_value_t = rustfs_config::DEFAULT_CONSOLE_ADDRESS.to_string(),
env = "RUSTFS_CONSOLE_ADDRESS"
)]
pub console_address: String,
/// Observability endpoint for trace, metrics and logs,only support grpc mode.
#[arg(
long,
default_value_t = rustfs_config::DEFAULT_OBS_ENDPOINT.to_string(),
env = "RUSTFS_OBS_ENDPOINT"
)]
pub obs_endpoint: String,
/// tls path for rustfs API and console.
#[arg(long, env = "RUSTFS_TLS_PATH")]
pub tls_path: Option<String>,
#[arg(long, env = "RUSTFS_LICENSE")]
pub license: Option<String>,
#[arg(long, env = "RUSTFS_REGION")]
pub region: Option<String>,
/// Enable KMS encryption for server-side encryption
#[arg(long, default_value_t = false, env = "RUSTFS_KMS_ENABLE")]
pub kms_enable: bool,
/// KMS backend type (local or vault)
#[arg(long, default_value_t = String::from("local"), env = "RUSTFS_KMS_BACKEND")]
pub kms_backend: String,
/// KMS key directory for local backend
#[arg(long, env = "RUSTFS_KMS_KEY_DIR")]
pub kms_key_dir: Option<String>,
/// Vault address for vault backend
#[arg(long, env = "RUSTFS_KMS_VAULT_ADDRESS")]
pub kms_vault_address: Option<String>,
/// Vault token for vault backend
#[arg(long, env = "RUSTFS_KMS_VAULT_TOKEN")]
pub kms_vault_token: Option<String>,
/// Default KMS key ID for encryption
#[arg(long, env = "RUSTFS_KMS_DEFAULT_KEY_ID")]
pub kms_default_key_id: Option<String>,
/// Disable adaptive buffer sizing with workload profiles
/// Set this flag to use legacy fixed-size buffer behavior from PR #869
#[arg(long, default_value_t = false, env = "RUSTFS_BUFFER_PROFILE_DISABLE")]
pub buffer_profile_disable: bool,
/// Workload profile for adaptive buffer sizing
/// Options: GeneralPurpose, AiTraining, DataAnalytics, WebWorkload, IndustrialIoT, SecureStorage
#[arg(long, default_value_t = String::from("GeneralPurpose"), env = "RUSTFS_BUFFER_PROFILE")]
pub buffer_profile: String,
}
/// Parsed server options. Public for tests and backward compatibility.
/// Use `Opt::parse_from` or `Config::parse()` to obtain.
#[derive(Clone)]
pub struct Opt {
pub volumes: Vec<String>,
pub address: String,
pub server_domains: Vec<String>,
pub access_key: Option<String>,
pub access_key_file: Option<PathBuf>,
pub secret_key: Option<String>,
pub secret_key_file: Option<PathBuf>,
pub console_enable: bool,
pub console_address: String,
pub obs_endpoint: String,
pub tls_path: Option<String>,
pub license: Option<String>,
pub region: Option<String>,
pub kms_enable: bool,
pub kms_backend: String,
pub kms_key_dir: Option<String>,
pub kms_vault_address: Option<String>,
pub kms_vault_token: Option<String>,
pub kms_default_key_id: Option<String>,
pub buffer_profile_disable: bool,
pub buffer_profile: String,
}
impl Opt {
fn from_server_opts(o: ServerOpts) -> Self {
Self {
volumes: o.volumes,
address: o.address,
server_domains: o.server_domains,
access_key: o.access_key,
access_key_file: o.access_key_file,
secret_key: o.secret_key,
secret_key_file: o.secret_key_file,
console_enable: o.console_enable,
console_address: o.console_address,
obs_endpoint: o.obs_endpoint,
tls_path: o.tls_path,
license: o.license,
region: o.region,
kms_enable: o.kms_enable,
kms_backend: o.kms_backend,
kms_key_dir: o.kms_key_dir,
kms_vault_address: o.kms_vault_address,
kms_vault_token: o.kms_vault_token,
kms_default_key_id: o.kms_default_key_id,
buffer_profile_disable: o.buffer_profile_disable,
buffer_profile: o.buffer_profile,
}
}
/// Parse from preprocessed args. Supports both `rustfs <volume>` and `rustfs server <volume>`.
pub fn parse_from<I, T>(args: I) -> Self
where
I: IntoIterator<Item = T>,
T: Into<std::ffi::OsString> + Clone,
{
let _ = apply_external_env_compat();
let args: Vec<String> = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect();
let args = preprocess_args_for_legacy(args);
let cli = Cli::parse_from(args);
let Commands::Server(opts) = cli.command.expect("server is the default subcommand");
Self::from_server_opts(opts)
}
/// Try parse from args, returns error on invalid input.
#[allow(dead_code)] // used in config_test
pub fn try_parse_from<I, T>(args: I) -> Result<Self, clap::Error>
where
I: IntoIterator<Item = T>,
T: Into<std::ffi::OsString> + Clone,
{
let _ = apply_external_env_compat();
let args: Vec<String> = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect();
let args = preprocess_args_for_legacy(args);
let cli = Cli::try_parse_from(args)?;
let Commands::Server(opts) = cli.command.expect("server is the default subcommand");
Ok(Self::from_server_opts(opts))
}
/// Parse from env::args(). Used by Config::parse().
fn parse() -> Self {
let args: Vec<String> = std::env::args().collect();
Self::parse_from(args)
}
}
#[derive(Clone)]
pub struct Config {
/// DIR points to a directory on a filesystem.
pub volumes: Vec<String>,
/// bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname
pub address: String,
/// Domain name used for virtual-hosted-style requests.
pub server_domains: Vec<String>,
/// Access key used for authentication.
pub access_key: String,
/// Secret key used for authentication.
pub secret_key: String,
/// Enable console server
pub console_enable: bool,
/// Console server bind address
pub console_address: String,
/// Observability endpoint for trace, metrics and logs,only support grpc mode.
pub obs_endpoint: String,
/// tls path for rustfs API and console.
pub tls_path: Option<String>,
/// License key for enterprise features
pub license: Option<String>,
/// Region for the server, used for signing and other region-specific behavior
pub region: Option<String>,
/// Enable KMS encryption for server-side encryption
pub kms_enable: bool,
/// KMS backend type (local or vault)
pub kms_backend: String,
/// KMS key directory for local backend
pub kms_key_dir: Option<String>,
/// Vault address for vault backend
pub kms_vault_address: Option<String>,
/// Vault token for vault backend
pub kms_vault_token: Option<String>,
/// Default KMS key ID for encryption
pub kms_default_key_id: Option<String>,
/// Disable adaptive buffer sizing with workload profiles
/// Set this flag to use legacy fixed-size buffer behavior from PR #869
pub buffer_profile_disable: bool,
/// Workload profile for adaptive buffer sizing
/// Options: GeneralPurpose, AiTraining, DataAnalytics, WebWorkload, IndustrialIoT, SecureStorage
pub buffer_profile: String,
}
impl Config {
fn from_opt(opt: Opt) -> std::io::Result<Self> {
let Opt {
volumes,
address,
server_domains,
access_key,
access_key_file,
secret_key,
secret_key_file,
console_enable,
console_address,
obs_endpoint,
tls_path,
license,
region,
kms_enable,
kms_backend,
kms_key_dir,
kms_vault_address,
kms_vault_token,
kms_default_key_id,
buffer_profile_disable,
buffer_profile,
} = opt;
let access_key = access_key
.map(Ok)
.or_else(|| {
let path = access_key_file.as_ref()?;
Some(std::fs::read_to_string(path))
})
.or_else(|| get_env_opt_str("RUSTFS_ROOT_USER").map(Ok))
.transpose()?
.unwrap_or_else(|| {
// neither argument was specified ... using default
rustfs_credentials::DEFAULT_ACCESS_KEY.to_string()
})
.trim()
.to_string();
let secret_key = secret_key
.map(Ok)
.or_else(|| {
let path = secret_key_file.as_ref()?;
Some(std::fs::read_to_string(path))
})
.or_else(|| get_env_opt_str("RUSTFS_ROOT_PASSWORD").map(Ok))
.transpose()?
.unwrap_or_else(|| {
// neither argument was specified ... using default
rustfs_credentials::DEFAULT_SECRET_KEY.to_string()
})
.trim()
.to_string();
// Region is optional, but if not set, we should default to "us-east-1" for signing compatibility with AWS S3 clients
let region = region.or_else(|| Some(RUSTFS_REGION.to_string()));
Ok(Config {
volumes,
address,
server_domains,
access_key,
secret_key,
console_enable,
console_address,
obs_endpoint,
tls_path,
license,
region,
kms_enable,
kms_backend,
kms_key_dir,
kms_vault_address,
kms_vault_token,
kms_default_key_id,
buffer_profile_disable,
buffer_profile,
})
}
/// Parse the command line arguments and environment arguments from [`Opt`] and convert them
/// into a ready to use [`Config`].
///
/// Supports both `rustfs <volume>` (legacy) and `rustfs server <volume>`.
///
/// This includes some intermediate checks for mutually exclusive options.
pub fn parse() -> std::io::Result<Self> {
Self::from_opt(Opt::parse())
}
}
impl std::fmt::Debug for Config {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Config")
.field("volumes", &self.volumes)
.field("address", &self.address)
.field("server_domains", &self.server_domains)
.field("access_key", &self.access_key)
.field("secret_key", &rustfs_credentials::Masked(Some(&self.secret_key))) // Hide sensitive values
.field("console_enable", &self.console_enable)
.field("console_address", &self.console_address)
.field("obs_endpoint", &self.obs_endpoint)
.field("tls_path", &self.tls_path)
.field("license", &rustfs_credentials::Masked(self.license.as_deref()))
.field("region", &self.region)
.field("kms_enable", &self.kms_enable)
.field("kms_backend", &self.kms_backend)
.field("kms_key_dir", &self.kms_key_dir)
.field("kms_vault_address", &self.kms_vault_address)
.field("kms_vault_token", &rustfs_credentials::Masked(self.kms_vault_token.as_deref()))
.field("kms_default_key_id", &self.kms_default_key_id)
.field("buffer_profile_disable", &self.buffer_profile_disable)
.field("buffer_profile", &self.buffer_profile)
.finish()
}
}
// Re-export workload profiles
mod workload_profiles;
pub use workload_profiles::*;
+161
View File
@@ -0,0 +1,161 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Parsed server options.
//!
//! This module contains the `Opt` struct which holds parsed server options
//! and methods for parsing command line arguments.
use super::Config;
use super::cli::{Cli, CommandResult, Commands, ServerOpts, default_server_opts, preprocess_args_for_legacy};
use crate::apply_external_env_compat;
use CommandResult::Server;
use clap::Parser;
use std::path::PathBuf;
/// Parsed server options. Public for tests and backward compatibility.
/// Use `Opt::parse_from` or `Config::parse()` to obtain.
#[derive(Clone)]
pub struct Opt {
pub volumes: Vec<String>,
pub address: String,
pub server_domains: Vec<String>,
pub access_key: Option<String>,
pub access_key_file: Option<PathBuf>,
pub secret_key: Option<String>,
pub secret_key_file: Option<PathBuf>,
pub console_enable: bool,
pub console_address: String,
pub obs_endpoint: String,
pub tls_path: Option<String>,
pub license: Option<String>,
pub region: Option<String>,
pub kms_enable: bool,
pub kms_backend: String,
pub kms_key_dir: Option<String>,
pub kms_vault_address: Option<String>,
pub kms_vault_token: Option<String>,
pub kms_default_key_id: Option<String>,
pub buffer_profile_disable: bool,
pub buffer_profile: String,
}
impl Opt {
/// Create Opt from ServerOpts
pub(super) fn from_server_opts(o: ServerOpts) -> Self {
Self {
volumes: o.volumes,
address: o.address,
server_domains: o.server_domains,
access_key: o.access_key,
access_key_file: o.access_key_file,
secret_key: o.secret_key,
secret_key_file: o.secret_key_file,
console_enable: o.console_enable,
console_address: o.console_address,
obs_endpoint: o.obs_endpoint,
tls_path: o.tls_path,
license: o.license,
region: o.region,
kms_enable: o.kms_enable,
kms_backend: o.kms_backend,
kms_key_dir: o.kms_key_dir,
kms_vault_address: o.kms_vault_address,
kms_vault_token: o.kms_vault_token,
kms_default_key_id: o.kms_default_key_id,
buffer_profile_disable: o.buffer_profile_disable,
buffer_profile: o.buffer_profile,
}
}
/// Parse from preprocessed args. Supports both `rustfs <volume>` and `rustfs server <volume>`.
#[allow(dead_code)] // used in config_test
pub fn parse_from<I, T>(args: I) -> Self
where
I: IntoIterator<Item = T>,
T: Into<std::ffi::OsString> + Clone,
{
let _ = apply_external_env_compat();
let args: Vec<String> = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect();
let args = preprocess_args_for_legacy(args);
let cli = Cli::parse_from(args);
match cli.command {
Some(Commands::Server(opts)) => Self::from_server_opts(*opts),
Some(Commands::Info(_)) => {
// This should not happen in parse_from, as it's handled by parse_command
panic!("Info command should be handled by parse_command");
}
None => {
// Default to server with empty volumes (will be filled from env)
Self::from_server_opts(default_server_opts())
}
}
}
/// Parse from preprocessed args and return the command type.
/// Returns Ok(Info(opts)) if info command, Ok(Server(opts)) if server command.
pub fn parse_command<I, T>(args: I) -> Result<CommandResult, clap::Error>
where
I: IntoIterator<Item = T>,
T: Into<std::ffi::OsString> + Clone,
{
let _ = apply_external_env_compat();
let args: Vec<String> = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect();
let args = preprocess_args_for_legacy(args);
let cli = Cli::try_parse_from(args)?;
match cli.command {
Some(Commands::Info(opts)) => Ok(CommandResult::Info(opts)),
Some(Commands::Server(opts)) => Self::server_command_result(Self::from_server_opts(*opts)),
None => {
// Default to server with empty volumes (will be filled from env)
Self::server_command_result(Self::from_server_opts(default_server_opts()))
}
}
}
// Helper to convert Opt to CommandResult::Server with error handling
fn server_command_result(opt: Opt) -> Result<CommandResult, clap::Error> {
Ok(Server(Box::new(Config::from_opt(opt).map_err(|e| {
clap::Error::raw(clap::error::ErrorKind::ValueValidation, e.to_string())
})?)))
}
/// Try parse from args, returns error on invalid input.
#[allow(dead_code)] // used in config_test
pub fn try_parse_from<I, T>(args: I) -> Result<Self, clap::Error>
where
I: IntoIterator<Item = T>,
T: Into<std::ffi::OsString> + Clone,
{
let _ = apply_external_env_compat();
let args: Vec<String> = args.into_iter().map(|a| a.into().to_string_lossy().into_owned()).collect();
let args = preprocess_args_for_legacy(args);
let cli = Cli::try_parse_from(args)?;
match cli.command {
Some(Commands::Server(opts)) => Ok(Self::from_server_opts(*opts)),
Some(Commands::Info(_)) => Err(clap::Error::new(clap::error::ErrorKind::DisplayHelp)),
None => {
// Default to server with empty volumes
Ok(Self::from_server_opts(default_server_opts()))
}
}
}
/// Parse from env::args(). Used by Config::parse().
#[allow(dead_code)] // used in config_test
fn parse() -> Self {
let args: Vec<String> = std::env::args().collect();
Self::parse_from(args)
}
}
+164
View File
@@ -0,0 +1,164 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Configuration snapshot for info command.
//!
//! This module provides a lightweight snapshot of configuration values
//! that can be accessed globally without needing the full Config struct.
use super::Config;
use crate::config::config_struct::resolve_credential;
use rustfs_config::{
DEFAULT_ADDRESS, DEFAULT_BUFFER_PROFILE, DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_KMS_BACKEND,
DEFAULT_KMS_ENABLE, DEFAULT_OBS_ENDPOINT, ENV_RUSTFS_ACCESS_KEY, ENV_RUSTFS_ACCESS_KEY_FILE, ENV_RUSTFS_ADDRESS,
ENV_RUSTFS_BUFFER_PROFILE, ENV_RUSTFS_CONSOLE_ADDRESS, ENV_RUSTFS_CONSOLE_ENABLE, ENV_RUSTFS_KMS_BACKEND,
ENV_RUSTFS_KMS_ENABLE, ENV_RUSTFS_OBS_ENDPOINT, ENV_RUSTFS_REGION, ENV_RUSTFS_ROOT_USER, ENV_RUSTFS_TLS_PATH, RUSTFS_REGION,
};
use rustfs_credentials::DEFAULT_ACCESS_KEY;
use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_str};
use std::sync::OnceLock;
/// Fallback snapshot used only for display when the global snapshot
/// has not yet been initialized (e.g., for the `--info` command).
/// This avoids leaking memory while still providing a `'static` reference.
static DISPLAY_CONFIG_SNAPSHOT: OnceLock<ConfigSnapshot> = OnceLock::new();
/// Configuration snapshot for info command display.
/// This stores key configuration values that can be accessed without
/// needing the full Config struct.
#[derive(Clone, Debug)]
pub struct ConfigSnapshot {
/// Server bind address
pub address: String,
/// Console server enabled
pub console_enable: bool,
/// Console server address
pub console_address: String,
/// Server region
pub region: Option<String>,
/// Access key (for display, should be masked)
pub access_key: String,
/// OBS endpoint
pub obs_endpoint: String,
/// TLS path
pub tls_path: Option<String>,
/// KMS enabled
pub kms_enable: bool,
/// KMS backend type
pub kms_backend: String,
/// Buffer profile
pub buffer_profile: String,
}
impl ConfigSnapshot {
/// Create a snapshot from Config
pub fn from_config(config: &Config) -> Self {
Self {
address: config.address.clone(),
console_enable: config.console_enable,
console_address: config.console_address.clone(),
region: config.region.clone(),
access_key: config.access_key.clone(),
obs_endpoint: config.obs_endpoint.clone(),
tls_path: config.tls_path.clone(),
kms_enable: config.kms_enable,
kms_backend: config.kms_backend.clone(),
buffer_profile: config.buffer_profile.clone(),
}
}
/// Create a default snapshot from environment variables and defaults
pub fn from_env() -> Self {
let access_key = resolve_credential(
get_env_opt_str(ENV_RUSTFS_ACCESS_KEY),
get_env_opt_str(ENV_RUSTFS_ACCESS_KEY_FILE),
ENV_RUSTFS_ROOT_USER,
DEFAULT_ACCESS_KEY,
)
.unwrap_or_else(|_| DEFAULT_ACCESS_KEY.to_string());
Self {
address: get_env_str(ENV_RUSTFS_ADDRESS, DEFAULT_ADDRESS),
console_enable: get_env_bool(ENV_RUSTFS_CONSOLE_ENABLE, DEFAULT_CONSOLE_ENABLE),
console_address: get_env_str(ENV_RUSTFS_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ADDRESS),
region: Some(get_env_str(ENV_RUSTFS_REGION, RUSTFS_REGION)),
access_key,
obs_endpoint: get_env_str(ENV_RUSTFS_OBS_ENDPOINT, DEFAULT_OBS_ENDPOINT),
tls_path: get_env_opt_str(ENV_RUSTFS_TLS_PATH),
kms_enable: get_env_bool(ENV_RUSTFS_KMS_ENABLE, DEFAULT_KMS_ENABLE),
kms_backend: get_env_str(ENV_RUSTFS_KMS_BACKEND, DEFAULT_KMS_BACKEND),
buffer_profile: get_env_str(ENV_RUSTFS_BUFFER_PROFILE, DEFAULT_BUFFER_PROFILE),
}
}
}
/// Global configuration snapshot storage
static GLOBAL_CONFIG_SNAPSHOT: OnceLock<ConfigSnapshot> = OnceLock::new();
/// Initialize the global config snapshot from a Config instance.
/// This should be called once during server startup.
///
/// This is the ONLY function that can set the global snapshot.
/// Once set, it cannot be changed.
pub fn init_config_snapshot(config: &Config) {
let snapshot = ConfigSnapshot::from_config(config);
if GLOBAL_CONFIG_SNAPSHOT.set(snapshot).is_err() {
// Already initialized, log a warning
tracing::warn!("Config snapshot already initialized, ignoring re-initialization");
}
}
/// Get the global config snapshot if initialized.
/// Returns None if not initialized (e.g., when running --info before server starts).
#[allow(dead_code)] // used in info command
pub fn get_config_snapshot() -> Option<&'static ConfigSnapshot> {
GLOBAL_CONFIG_SNAPSHOT.get()
}
/// Check if the global config snapshot has been initialized.
#[allow(dead_code)] // may be used for debugging
pub fn is_config_snapshot_initialized() -> bool {
GLOBAL_CONFIG_SNAPSHOT.get().is_some()
}
/// Get config snapshot for display purposes (backward-compatible wrapper).
///
/// - If the global snapshot is initialized (server has started), returns a reference to it.
/// - If not initialized (e.g., --info command before server starts), returns a temporary
/// snapshot created from environment variables WITHOUT updating the global storage.
///
/// Despite its name, this function no longer initializes the global snapshot; it simply
/// delegates to `get_config_snapshot_for_display` to ensure the global snapshot is ONLY set
/// by `init_config_snapshot` during server startup.
#[allow(dead_code)] // kept for backward compatibility
pub fn get_or_init_config_snapshot() -> &'static ConfigSnapshot {
get_config_snapshot_for_display()
}
/// Get config snapshot for display, without modifying global state.
///
/// This function is used by the --info command to display configuration:
/// - Returns the global snapshot if initialized
/// - Otherwise creates a temporary snapshot from environment variables (does NOT store it)
///
/// Note: This returns a static reference for API compatibility. When the global snapshot
/// is not initialized, it creates a leaked Box to provide a static lifetime.
/// This is safe because it's only used for read-only display purposes.
pub fn get_config_snapshot_for_display() -> &'static ConfigSnapshot {
if let Some(snapshot) = GLOBAL_CONFIG_SNAPSHOT.get() {
snapshot
} else {
// Not initialized - create from env without storing in the global snapshot.
// Use a dedicated OnceLock to cache a single display snapshot without leaking.
DISPLAY_CONFIG_SNAPSHOT.get_or_init(ConfigSnapshot::from_env)
}
}
+79 -4
View File
@@ -69,9 +69,10 @@ pub fn get_global_buffer_config() -> &'static RustFSBufferConfig {
}
/// Workload profile types that define buffer sizing strategies
#[derive(Debug, Clone, PartialEq)]
#[derive(Debug, Clone, PartialEq, Default)]
pub enum WorkloadProfile {
/// General purpose - default configuration with balanced performance and memory
#[default]
GeneralPurpose,
/// AI/ML training: optimized for large sequential reads with maximum throughput
AiTraining,
@@ -84,7 +85,6 @@ pub enum WorkloadProfile {
/// Secure storage: security first, memory constrained for compliance
SecureStorage,
/// Custom configuration for specialized requirements
#[allow(dead_code)]
Custom(BufferConfig),
}
@@ -106,7 +106,6 @@ pub struct BufferConfig {
#[derive(Debug, Clone)]
pub struct RustFSBufferConfig {
/// Selected workload profile
#[allow(dead_code)]
pub workload: WorkloadProfile,
/// Computed buffer configuration (either from profile or custom)
pub base_config: BufferConfig,
@@ -144,6 +143,25 @@ impl WorkloadProfile {
}
}
/// Create a custom workload profile with specified buffer configuration
///
/// # Arguments
/// * `min_size` - Minimum buffer size in bytes
/// * `max_size` - Maximum buffer size in bytes
/// * `default_unknown` - Default size for unknown file size scenarios
/// * `thresholds` - File size thresholds and corresponding buffer sizes
///
/// # Returns
/// A WorkloadProfile::Custom with the specified configuration
pub fn custom(min_size: usize, max_size: usize, default_unknown: usize, thresholds: Vec<(i64, usize)>) -> Self {
WorkloadProfile::Custom(BufferConfig {
min_size,
max_size,
default_unknown,
thresholds,
})
}
/// Get the buffer configuration for this workload profile
pub fn config(&self) -> BufferConfig {
match self {
@@ -309,7 +327,6 @@ impl BufferConfig {
}
/// Validate the buffer configuration
#[allow(dead_code)]
pub fn validate(&self) -> Result<(), String> {
if self.min_size == 0 {
return Err("min_size must be greater than 0".to_string());
@@ -361,6 +378,29 @@ impl RustFSBufferConfig {
pub fn get_buffer_size(&self, file_size: i64) -> usize {
self.base_config.calculate_buffer_size(file_size)
}
/// Get the current workload profile
pub fn workload_profile(&self) -> &WorkloadProfile {
&self.workload
}
/// Get the name of the current workload profile
pub fn workload_name(&self) -> String {
match &self.workload {
WorkloadProfile::GeneralPurpose => "GeneralPurpose".to_string(),
WorkloadProfile::AiTraining => "AiTraining".to_string(),
WorkloadProfile::DataAnalytics => "DataAnalytics".to_string(),
WorkloadProfile::WebWorkload => "WebWorkload".to_string(),
WorkloadProfile::IndustrialIoT => "IndustrialIoT".to_string(),
WorkloadProfile::SecureStorage => "SecureStorage".to_string(),
WorkloadProfile::Custom(_) => "Custom".to_string(),
}
}
/// Validate the buffer configuration
pub fn validate(&self) -> Result<(), String> {
self.base_config.validate()
}
}
impl Default for RustFSBufferConfig {
@@ -616,6 +656,41 @@ mod tests {
assert_eq!(WorkloadProfile::from_name(""), WorkloadProfile::GeneralPurpose);
}
#[test]
fn test_custom_workload_profile() {
// Create a custom profile with specific buffer sizes
let custom_profile = WorkloadProfile::custom(
32 * KI_B, // min_size: 32KB
2 * MI_B, // max_size: 2MB
256 * KI_B, // default_unknown: 256KB
vec![
(MI_B as i64, 64 * KI_B), // < 1MB: 64KB
(10 * MI_B as i64, 128 * KI_B), // 1MB-10MB: 128KB
(i64::MAX, 512 * KI_B), // >= 10MB: 512KB
],
);
// Verify it's a Custom variant
match &custom_profile {
WorkloadProfile::Custom(config) => {
assert_eq!(config.min_size, 32 * KI_B);
assert_eq!(config.max_size, 2 * MI_B);
assert_eq!(config.default_unknown, 256 * KI_B);
assert_eq!(config.thresholds.len(), 3);
}
_ => panic!("Expected Custom variant"),
}
// Test buffer size calculation with custom profile
let buffer_config = RustFSBufferConfig::new(custom_profile);
assert_eq!(buffer_config.get_buffer_size(500 * KI_B as i64), 64 * KI_B);
assert_eq!(buffer_config.get_buffer_size(5 * MI_B as i64), 128 * KI_B);
assert_eq!(buffer_config.get_buffer_size(100 * MI_B as i64), 512 * KI_B);
// Test validation
assert!(buffer_config.validate().is_ok());
}
#[test]
fn test_global_buffer_config() {
use super::{is_buffer_profile_enabled, set_buffer_profile_enabled};
+55 -6
View File
@@ -14,10 +14,14 @@
use crate::storage::{process_lambda_configurations, process_queue_configurations, process_topic_configurations};
use crate::{admin, config, version};
use rustfs_config::{DEFAULT_UPDATE_CHECK, ENV_UPDATE_CHECK, RUSTFS_REGION};
use rustfs_config::{
DEFAULT_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MIN_SIZE, DEFAULT_BUFFER_PROFILE, DEFAULT_BUFFER_UNKNOWN_SIZE, DEFAULT_UPDATE_CHECK,
ENV_RUSTFS_BUFFER_DEFAULT_SIZE, ENV_RUSTFS_BUFFER_MAX_SIZE, ENV_RUSTFS_BUFFER_MIN_SIZE, ENV_UPDATE_CHECK, RUSTFS_REGION,
};
use rustfs_ecstore::bucket::metadata_sys;
use rustfs_notify::notifier_global;
use rustfs_targets::arn::{ARN, TargetIDError};
use rustfs_utils::get_env_usize;
use s3s::s3_error;
use std::env;
use std::io::Error;
@@ -297,10 +301,9 @@ pub(crate) async fn init_kms_system(config: &config::Config) -> std::io::Result<
/// # Arguments
/// * `config` - The application configuration options
pub(crate) fn init_buffer_profile_system(config: &config::Config) {
use crate::config::workload_profiles::{
RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled,
};
use crate::config::{RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled};
// Whether buffer profiling is disabled or not, it is enabled by default, unless the user explicitly sets '--buffer-profile-disable' or 'RUSTFS_BUFFER_PROFILE_DISABLE=true'
if config.buffer_profile_disable {
// User explicitly disabled buffer profiling - use GeneralPurpose profile in disabled mode
info!("Buffer profiling disabled via --buffer-profile-disable, using GeneralPurpose profile");
@@ -310,13 +313,59 @@ pub(crate) fn init_buffer_profile_system(config: &config::Config) {
info!("Buffer profiling enabled with profile: {}", config.buffer_profile);
// Parse the workload profile from configuration string
let profile = WorkloadProfile::from_name(&config.buffer_profile);
// Support a custom profile when buffer_profile is set to "custom";
// its sizes are controlled via RUSTFS_BUFFER_MIN_SIZE, RUSTFS_BUFFER_MAX_SIZE,
// and RUSTFS_BUFFER_DEFAULT_SIZE environment variables.
let profile = if config.buffer_profile.eq_ignore_ascii_case("custom") {
// Try to create custom profile from environment variables
let min_size = get_env_usize(ENV_RUSTFS_BUFFER_MIN_SIZE, DEFAULT_BUFFER_MIN_SIZE);
let max_size = get_env_usize(ENV_RUSTFS_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MAX_SIZE);
let default_unknown = get_env_usize(ENV_RUSTFS_BUFFER_DEFAULT_SIZE, DEFAULT_BUFFER_UNKNOWN_SIZE);
info!(
"Creating custom buffer profile: min={}, max={}, default={}",
min_size, max_size, default_unknown
);
WorkloadProfile::custom(
min_size,
max_size,
default_unknown,
vec![
(1024 * 1024, 64 * 1024), // < 1MB: 64KB
(100 * 1024 * 1024, 256 * 1024), // 1MB-100MB: 256KB
(i64::MAX, 1024 * 1024), // >= 100MB: 1MB
],
)
} else {
WorkloadProfile::from_name(&config.buffer_profile)
};
// Log the selected profile for operational visibility
info!("Active buffer profile: {:?}", profile);
// Create and validate buffer configuration
let mut buffer_config = RustFSBufferConfig::new(profile);
if let Err(e) = buffer_config.validate() {
warn!("Buffer configuration validation failed: {}. Falling back to GeneralPurpose profile.", e);
// Fall back to a known-good profile to avoid installing an invalid configuration
let fallback_profile = WorkloadProfile::from_name(DEFAULT_BUFFER_PROFILE);
info!("Using fallback buffer profile: {:?}", fallback_profile);
let fallback_config = RustFSBufferConfig::new(fallback_profile);
if let Err(e2) = fallback_config.validate() {
error!(
"Fallback buffer configuration validation failed: {}. Aborting buffer profiling initialization.",
e2
);
panic!("Failed to initialize a valid RustFS buffer configuration");
}
buffer_config = fallback_config;
}
// Log the workload profile name
info!("Workload profile: {}", buffer_config.workload_name());
// Initialize the global buffer configuration
init_global_buffer_config(RustFSBufferConfig::new(profile));
init_global_buffer_config(buffer_config);
// Enable buffer profiling globally
set_buffer_profile_enabled(true);
+25 -3
View File
@@ -151,8 +151,30 @@ fn format_external_prefix_mappings(report: &ExternalEnvCompatReport) -> String {
}
async fn async_main() -> Result<()> {
// Parse the obtained parameters
let config = config::Config::parse()?;
// Parse command line arguments
let args: Vec<String> = std::env::args().collect();
let command_result = match config::Opt::parse_command(args) {
Ok(result) => result,
Err(e) => {
eprintln!("Command parse failed, error: {}", e);
std::process::exit(1);
}
};
// Handle info command
if let config::CommandResult::Info(opts) = command_result {
config::execute_info(&opts);
return Ok(());
}
// Get config for server command
let config = match command_result {
config::CommandResult::Server(cfg) => cfg,
config::CommandResult::Info(_) => unreachable!(),
};
// Initialize the global config snapshot for info command
config::init_config_snapshot(&config);
// Initialize the configuration
init_license(config.license.clone());
@@ -211,7 +233,7 @@ async fn async_main() -> Result<()> {
}
// Run parameters
match run(config).await {
match run(*config).await {
Ok(_) => Ok(()),
Err(e) => {
error!("Server encountered an error and is shutting down: {}", e);
+17 -3
View File
@@ -12,9 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::config::workload_profiles::{
RustFSBufferConfig, WorkloadProfile, get_global_buffer_config, is_buffer_profile_enabled,
};
use crate::config::{RustFSBufferConfig, WorkloadProfile, get_global_buffer_config, is_buffer_profile_enabled};
use crate::error::ApiError;
use crate::server::cors;
use crate::storage::ecfs::ListObjectUnorderedQuery;
@@ -119,6 +117,22 @@ pub(crate) fn apply_lock_retention(object_lock_config: Option<ObjectLockConfigur
/// 10 * 1024 * 1024,
/// Some(WorkloadProfile::SecureStorage)
/// );
///
/// // Use custom profile for specialized requirements
/// let custom_profile = WorkloadProfile::custom(
/// 32 * 1024, // min_size: 32KB
/// 2 * 1024 * 1024, // max_size: 2MB
/// 256 * 1024, // default_unknown: 256KB
/// vec![
/// (1024 * 1024, 64 * 1024), // < 1MB: 64KB
/// (10 * 1024 * 1024, 128 * 1024), // 1MB-10MB: 128KB
/// (i64::MAX, 512 * 1024), // >= 10MB: 512KB
/// ],
/// );
/// let buffer_size = get_adaptive_buffer_size_with_profile(
/// 5 * 1024 * 1024,
/// Some(custom_profile)
/// );
/// ```
///
#[allow(dead_code)]
+4 -8
View File
@@ -14,7 +14,7 @@
#[cfg(test)]
mod tests {
use crate::config::workload_profiles::WorkloadProfile;
use crate::config::WorkloadProfile;
use crate::server::cors;
use crate::storage::ecfs::FS;
use crate::storage::s3_api::common::{rustfs_initiator, rustfs_owner};
@@ -281,9 +281,7 @@ mod tests {
#[test]
fn test_phase3_default_behavior() {
use crate::config::workload_profiles::{
RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled,
};
use crate::config::{RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled};
const KB: i64 = 1024;
const MB: i64 = 1024 * 1024;
@@ -304,7 +302,7 @@ mod tests {
#[test]
fn test_buffer_size_opt_in() {
use crate::config::workload_profiles::{is_buffer_profile_enabled, set_buffer_profile_enabled};
use crate::config::{is_buffer_profile_enabled, set_buffer_profile_enabled};
const KB: i64 = 1024;
const MB: i64 = 1024 * 1024;
@@ -336,9 +334,7 @@ mod tests {
#[test]
fn test_phase4_full_integration() {
use crate::config::workload_profiles::{
RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled,
};
use crate::config::{RustFSBufferConfig, WorkloadProfile, init_global_buffer_config, set_buffer_profile_enabled};
const KB: i64 = 1024;
const MB: i64 = 1024 * 1024;