mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
refactor: unify credential env constants and deploy env usage (#2821)
Co-authored-by: Henry Guo <marshawcoco@gmail.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: loverustfs <hello@rustfs.com> Co-authored-by: 安正超 <anzhengchao@gmail.com>
This commit is contained in:
@@ -0,0 +1,52 @@
|
||||
services:
|
||||
rustfs:
|
||||
image: rustfs/rustfs:1.0.0-alpha.99-glibc
|
||||
container_name: rustfs-issue-2715-test
|
||||
security_opt:
|
||||
- "no-new-privileges:true"
|
||||
ports:
|
||||
- "19000:9000"
|
||||
- "19001:9001"
|
||||
environment:
|
||||
- RUSTFS_VOLUMES=/data/rustfs{0...8}
|
||||
- RUSTFS_ADDRESS=0.0.0.0:9000
|
||||
- RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001
|
||||
- RUSTFS_CONSOLE_ENABLE=true
|
||||
- RUSTFS_CORS_ALLOWED_ORIGINS=*
|
||||
- RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS=*
|
||||
- RUSTFS_ACCESS_KEY=admin
|
||||
- RUSTFS_SECRET_KEY=admin
|
||||
- RUSTFS_OBS_LOGGER_LEVEL=info
|
||||
- RUSTFS_OBS_ENDPOINT=http://otel-collector:4318
|
||||
- RUSTFS_OBS_PROFILING_ENDPOINT=http://pyroscope:4040
|
||||
- RUSTFS_STORAGE_CLASS_STANDARD=EC:2
|
||||
- RUSTFS_STORAGE_CLASS_RRS=EC:1
|
||||
- RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true
|
||||
- RUSTFS_OBS_LOG_DIRECTORY=/opt/rustfs/logs
|
||||
extra_hosts:
|
||||
- "otel-collector:host-gateway"
|
||||
- "pyroscope:host-gateway"
|
||||
volumes:
|
||||
- ./deploy/data/issue-2715/rustfs0:/data/rustfs0
|
||||
- ./deploy/data/issue-2715/rustfs1:/data/rustfs1
|
||||
- ./deploy/data/issue-2715/rustfs2:/data/rustfs2
|
||||
- ./deploy/data/issue-2715/rustfs3:/data/rustfs3
|
||||
- ./deploy/data/issue-2715/rustfs4:/data/rustfs4
|
||||
- ./deploy/data/issue-2715/rustfs5:/data/rustfs5
|
||||
- ./deploy/data/issue-2715/rustfs6:/data/rustfs6
|
||||
- ./deploy/data/issue-2715/rustfs7:/data/rustfs7
|
||||
- ./deploy/data/issue-2715/rustfs8:/data/rustfs8
|
||||
- ./deploy/logs/issue-2715:/opt/rustfs/logs
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD",
|
||||
"sh",
|
||||
"-c",
|
||||
"curl -f http://127.0.0.1:9000/health && curl -f http://127.0.0.1:9001/rustfs/console/health"
|
||||
]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
@@ -149,18 +149,12 @@ pub const ENV_RUSTFS_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
|
||||
/// Environment variable for server access key file.
|
||||
pub const ENV_RUSTFS_ACCESS_KEY_FILE: &str = "RUSTFS_ACCESS_KEY_FILE";
|
||||
|
||||
/// Environment variable for server root user.
|
||||
pub const ENV_RUSTFS_ROOT_USER: &str = "RUSTFS_ROOT_USER";
|
||||
|
||||
/// Environment variable for server secret key.
|
||||
pub const ENV_RUSTFS_SECRET_KEY: &str = "RUSTFS_SECRET_KEY";
|
||||
|
||||
/// Environment variable for server secret key file.
|
||||
pub const ENV_RUSTFS_SECRET_KEY_FILE: &str = "RUSTFS_SECRET_KEY_FILE";
|
||||
|
||||
/// Environment variable for server root password.
|
||||
pub const ENV_RUSTFS_ROOT_PASSWORD: &str = "RUSTFS_ROOT_PASSWORD";
|
||||
|
||||
/// Environment variable for server OBS endpoint.
|
||||
pub const ENV_RUSTFS_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
|
||||
|
||||
|
||||
@@ -183,11 +183,8 @@ pub async fn get_local_server_property() -> ServerProperties {
|
||||
};
|
||||
|
||||
// let mut sensitive = HashSet::new();
|
||||
// sensitive.insert(ENV_ACCESS_KEY.to_string());
|
||||
// sensitive.insert(ENV_SECRET_KEY.to_string());
|
||||
// sensitive.insert(ENV_ROOT_USER.to_string());
|
||||
// sensitive.insert(ENV_ROOT_PASSWORD.to_string());
|
||||
|
||||
// sensitive.insert(rustfs_config::ENV_RUSTFS_ACCESS_KEY.to_string());
|
||||
// sensitive.insert(rustfs_config::ENV_RUSTFS_SECRET_KEY.to_string());
|
||||
if let Some(store) = new_object_layer_fn() {
|
||||
let storage_info = store.local_storage_info().await;
|
||||
props.state = ITEM_ONLINE.to_string();
|
||||
|
||||
@@ -42,10 +42,6 @@ pub static DEFAULT_KVS: LazyLock<OnceLock<HashMap<String, KVS>>> = LazyLock::new
|
||||
pub static GLOBAL_SERVER_CONFIG: LazyLock<OnceLock<Config>> = LazyLock::new(OnceLock::new);
|
||||
pub static GLOBAL_CONFIG_SYS: LazyLock<ConfigSys> = LazyLock::new(ConfigSys::new);
|
||||
|
||||
pub const ENV_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
|
||||
pub const ENV_SECRET_KEY: &str = "RUSTFS_SECRET_KEY";
|
||||
pub const ENV_ROOT_USER: &str = "RUSTFS_ROOT_USER";
|
||||
pub const ENV_ROOT_PASSWORD: &str = "RUSTFS_ROOT_PASSWORD";
|
||||
pub static RUSTFS_CONFIG_PREFIX: &str = "config";
|
||||
|
||||
pub struct ConfigSys {}
|
||||
|
||||
@@ -885,8 +885,8 @@ mod tests {
|
||||
network.insert("ip".to_string(), "192.168.1.100".to_string());
|
||||
|
||||
let mut env_vars = HashMap::new();
|
||||
env_vars.insert("RUSTFS_ROOT_USER".to_string(), "admin".to_string());
|
||||
env_vars.insert("RUSTFS_ROOT_PASSWORD".to_string(), "password".to_string());
|
||||
env_vars.insert("RUSTFS_ACCESS_KEY".to_string(), "admin".to_string());
|
||||
env_vars.insert("RUSTFS_SECRET_KEY".to_string(), "password".to_string());
|
||||
|
||||
let server_props = ServerProperties {
|
||||
state: "online".to_string(),
|
||||
|
||||
@@ -16,18 +16,13 @@ Group=rustfs
|
||||
# working directory
|
||||
WorkingDirectory=/opt/rustfs
|
||||
|
||||
# environment variable configuration and main program (Option 1: Directly specify arguments)
|
||||
Environment=RUSTFS_ACCESS_KEY=rustfsadmin
|
||||
Environment=RUSTFS_SECRET_KEY=rustfsadmin
|
||||
ExecStart=/usr/local/bin/rustfs \
|
||||
--address 0.0.0.0:9000 \
|
||||
--volumes /data/rustfs/vol1,/data/rustfs/vol2 \
|
||||
--console-enable
|
||||
# environment variable configuration and main program (single active ExecStart).
|
||||
# Credentials are loaded from /etc/default/rustfs below. Replace the sample values before deployment.
|
||||
|
||||
# environment variable configuration (Option 2: Use environment variables)
|
||||
# rustfs example file see: `../config/rustfs.env`
|
||||
EnvironmentFile=-/etc/default/rustfs
|
||||
ExecStart=/usr/local/bin/rustfs $RUSTFS_VOLUMES $RUSTFS_OPTS
|
||||
EnvironmentFile=/etc/default/rustfs
|
||||
ExecStart=/usr/local/bin/rustfs $RUSTFS_VOLUMES
|
||||
|
||||
# service log configuration
|
||||
LogsDirectory=rustfs
|
||||
|
||||
+14
-14
@@ -1,23 +1,23 @@
|
||||
# RustFS administrator username
|
||||
RUSTFS_ROOT_USER=rustfsadmin
|
||||
# RustFS administrator password
|
||||
RUSTFS_ROOT_PASSWORD=rustfsadmin
|
||||
# RustFS administrator access key. Replace before deployment; do not use public defaults.
|
||||
RUSTFS_ACCESS_KEY=REPLACE_WITH_UNIQUE_ACCESS_KEY
|
||||
# RustFS administrator secret key. Replace before deployment; do not use public defaults.
|
||||
RUSTFS_SECRET_KEY=REPLACE_WITH_UNIQUE_SECRET_KEY
|
||||
|
||||
# RustFS data volume storage paths.
|
||||
# Data volume configuration example path: deploy/data/rustfs.env
|
||||
# RustFS data volume storage paths, supports multiple volumes from vol1 to vol4
|
||||
RUSTFS_VOLUMES="./deploy/deploy/vol{1...4}"
|
||||
# RustFS service startup parameters, specifying listen address and port
|
||||
RUSTFS_OPTS="--address :9000"
|
||||
RUSTFS_VOLUMES="./deploy/data/vol{1...4}"
|
||||
# RustFS service listen address and port
|
||||
RUSTFS_ADDRESS=":9000"
|
||||
RUSTFS_ADDRESS=0.0.0.0:9000
|
||||
# Enable RustFS console functionality
|
||||
RUSTFS_CONSOLE_ENABLE=true
|
||||
# RustFS service domain configuration
|
||||
RUSTFS_SERVER_DOMAINS=127.0.0.1:9000
|
||||
# RustFS license content
|
||||
RUSTFS_LICENSE="license content"
|
||||
# RustFS console listen address and port
|
||||
RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001
|
||||
# Optional service domain configuration for virtual-hosted-style requests (comma-separated).
|
||||
# RUSTFS_SERVER_DOMAINS=s3.example.com
|
||||
# Optional RustFS license content
|
||||
# RUSTFS_LICENSE=REPLACE_WITH_LICENSE_CONTENT
|
||||
# Observability configuration endpoint: RUSTFS_OBS_ENDPOINT
|
||||
RUSTFS_OBS_ENDPOINT=http://localhost:4318
|
||||
# TLS certificates directory path: deploy/certs
|
||||
# Optional TLS certificates directory path: deploy/certs
|
||||
RUSTFS_TLS_PATH=/etc/default/tls
|
||||
|
||||
|
||||
@@ -20,27 +20,59 @@
|
||||
use super::Opt;
|
||||
use crate::apply_external_env_compat;
|
||||
use rustfs_config::{
|
||||
DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, ENV_RUSTFS_ROOT_PASSWORD, ENV_RUSTFS_ROOT_USER, RUSTFS_REGION,
|
||||
DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, ENV_RUSTFS_ACCESS_KEY, ENV_RUSTFS_SECRET_KEY, RUSTFS_REGION,
|
||||
};
|
||||
use rustfs_credentials::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, Masked};
|
||||
use std::collections::HashSet;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
|
||||
pub(crate) const LEGACY_ENV_RUSTFS_ROOT_USER: &str = "RUSTFS_ROOT_USER";
|
||||
pub(crate) const LEGACY_ENV_RUSTFS_ROOT_PASSWORD: &str = "RUSTFS_ROOT_PASSWORD";
|
||||
static LEGACY_CREDENTIAL_WARNED_KEYS: OnceLock<Mutex<HashSet<String>>> = OnceLock::new();
|
||||
|
||||
fn warn_legacy_credential_env_once(legacy_key: &str, canonical_key: &str) {
|
||||
let warned = LEGACY_CREDENTIAL_WARNED_KEYS.get_or_init(|| Mutex::new(HashSet::new()));
|
||||
let mut warned = match warned.lock() {
|
||||
Ok(guard) => guard,
|
||||
Err(poisoned) => poisoned.into_inner(),
|
||||
};
|
||||
if warned.insert(legacy_key.to_string()) {
|
||||
tracing::warn!(
|
||||
"Environment variable {} is deprecated and will be removed at GA; use {} instead",
|
||||
legacy_key,
|
||||
canonical_key
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Helper function to resolve credentials from multiple sources with precedence:
|
||||
/// 1. Inline value (if provided)
|
||||
/// 2. File value (if provided, read the content of the file)
|
||||
/// 3. Environment variable (if set)
|
||||
/// 4. Default value (if none of the above are provided)
|
||||
/// 3. Canonical environment variable (if set)
|
||||
/// 4. Legacy environment aliases (if set)
|
||||
/// 5. Default value (if none of the above are provided)
|
||||
pub(crate) fn resolve_credential<T: AsRef<std::path::Path>>(
|
||||
inline_value: Option<String>,
|
||||
file_value: Option<T>,
|
||||
env_key: &str,
|
||||
legacy_env_keys: &[&str],
|
||||
default_value: &str,
|
||||
) -> std::io::Result<String> {
|
||||
let value = inline_value
|
||||
.map(Ok)
|
||||
.or_else(|| file_value.map(std::fs::read_to_string))
|
||||
.or_else(|| rustfs_utils::get_env_opt_str(env_key).map(Ok))
|
||||
.transpose()?
|
||||
.unwrap_or_else(|| default_value.to_string());
|
||||
let value = if let Some(value) = inline_value {
|
||||
value
|
||||
} else if let Some(path) = file_value {
|
||||
std::fs::read_to_string(path)?
|
||||
} else if let Some(value) = rustfs_utils::get_env_opt_str(env_key) {
|
||||
value
|
||||
} else if let Some((legacy_key, value)) = legacy_env_keys
|
||||
.iter()
|
||||
.find_map(|legacy_key| rustfs_utils::get_env_opt_str(legacy_key).map(|value| (*legacy_key, value)))
|
||||
{
|
||||
warn_legacy_credential_env_once(legacy_key, env_key);
|
||||
value
|
||||
} else {
|
||||
default_value.to_string()
|
||||
};
|
||||
|
||||
Ok(value.trim().to_string())
|
||||
}
|
||||
@@ -169,8 +201,20 @@ impl Config {
|
||||
buffer_profile,
|
||||
} = opt;
|
||||
|
||||
let access_key = resolve_credential(access_key, access_key_file.as_ref(), ENV_RUSTFS_ROOT_USER, DEFAULT_ACCESS_KEY)?;
|
||||
let secret_key = resolve_credential(secret_key, secret_key_file.as_ref(), ENV_RUSTFS_ROOT_PASSWORD, DEFAULT_SECRET_KEY)?;
|
||||
let access_key = resolve_credential(
|
||||
access_key,
|
||||
access_key_file.as_ref(),
|
||||
ENV_RUSTFS_ACCESS_KEY,
|
||||
&[LEGACY_ENV_RUSTFS_ROOT_USER],
|
||||
DEFAULT_ACCESS_KEY,
|
||||
)?;
|
||||
let secret_key = resolve_credential(
|
||||
secret_key,
|
||||
secret_key_file.as_ref(),
|
||||
ENV_RUSTFS_SECRET_KEY,
|
||||
&[LEGACY_ENV_RUSTFS_ROOT_PASSWORD],
|
||||
DEFAULT_SECRET_KEY,
|
||||
)?;
|
||||
|
||||
// Region is optional, but if not set, we should default to "us-east-1" for signing compatibility with AWS S3 clients
|
||||
let region = region.or_else(|| Some(RUSTFS_REGION.to_string()));
|
||||
|
||||
@@ -174,7 +174,24 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_root_envs_are_used_for_bootstrap_credentials() {
|
||||
fn test_access_key_envs_are_used_for_bootstrap_credentials() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
("RUSTFS_VOLUMES", Some("/compat/vol1")),
|
||||
("RUSTFS_ACCESS_KEY", Some("canonical-access")),
|
||||
("RUSTFS_SECRET_KEY", Some("canonical-secret")),
|
||||
],
|
||||
|| {
|
||||
let config = Config::from_opt(Opt::parse_from(["rustfs"])).expect("config should parse");
|
||||
assert_eq!(config.access_key, "canonical-access");
|
||||
assert_eq!(config.secret_key, "canonical-secret");
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn test_root_envs_fallback_for_bootstrap_credentials() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
("RUSTFS_VOLUMES", Some("/compat/vol1")),
|
||||
|
||||
@@ -18,12 +18,12 @@
|
||||
//! that can be accessed globally without needing the full Config struct.
|
||||
|
||||
use super::Config;
|
||||
use crate::config::config_struct::resolve_credential;
|
||||
use crate::config::config_struct::{LEGACY_ENV_RUSTFS_ROOT_USER, resolve_credential};
|
||||
use rustfs_config::{
|
||||
DEFAULT_ADDRESS, DEFAULT_BUFFER_PROFILE, DEFAULT_CONSOLE_ADDRESS, DEFAULT_CONSOLE_ENABLE, DEFAULT_KMS_BACKEND,
|
||||
DEFAULT_KMS_ENABLE, DEFAULT_OBS_ENDPOINT, ENV_RUSTFS_ACCESS_KEY, ENV_RUSTFS_ACCESS_KEY_FILE, ENV_RUSTFS_ADDRESS,
|
||||
ENV_RUSTFS_BUFFER_PROFILE, ENV_RUSTFS_CONSOLE_ADDRESS, ENV_RUSTFS_CONSOLE_ENABLE, ENV_RUSTFS_KMS_BACKEND,
|
||||
ENV_RUSTFS_KMS_ENABLE, ENV_RUSTFS_OBS_ENDPOINT, ENV_RUSTFS_REGION, ENV_RUSTFS_ROOT_USER, ENV_RUSTFS_TLS_PATH, RUSTFS_REGION,
|
||||
ENV_RUSTFS_KMS_ENABLE, ENV_RUSTFS_OBS_ENDPOINT, ENV_RUSTFS_REGION, ENV_RUSTFS_TLS_PATH, RUSTFS_REGION,
|
||||
};
|
||||
use rustfs_credentials::DEFAULT_ACCESS_KEY;
|
||||
use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_str};
|
||||
@@ -83,7 +83,8 @@ impl ConfigSnapshot {
|
||||
let access_key = resolve_credential(
|
||||
get_env_opt_str(ENV_RUSTFS_ACCESS_KEY),
|
||||
get_env_opt_str(ENV_RUSTFS_ACCESS_KEY_FILE),
|
||||
ENV_RUSTFS_ROOT_USER,
|
||||
ENV_RUSTFS_ACCESS_KEY,
|
||||
&[LEGACY_ENV_RUSTFS_ROOT_USER],
|
||||
DEFAULT_ACCESS_KEY,
|
||||
)
|
||||
.unwrap_or_else(|_| DEFAULT_ACCESS_KEY.to_string());
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
RUSTFS_ROOT_USER=rustfsadmin
|
||||
RUSTFS_ROOT_PASSWORD=rustfsadmin
|
||||
RUSTFS_ACCESS_KEY=rustfsadmin
|
||||
RUSTFS_SECRET_KEY=rustfsadmin
|
||||
|
||||
RUSTFS_VOLUMES="http://node{1...4}:7000/data/rustfs{0...3} http://node{5...8}:7000/data/rustfs{0...3}"
|
||||
RUSTFS_ADDRESS=":7000"
|
||||
|
||||
Reference in New Issue
Block a user