gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-13 08:36:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: improve legacy metadata and admin compatibility (#2202)

Browse Source
This commit is contained in:
weisd
2026-03-18 21:05:09 +08:00
committed by GitHub
parent 84077adf17
commit b9b7d86ae4
133 changed files with 11707 additions and 1945 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/utils/src/envs.rs
+293 -27
View File
@@ -13,6 +13,7 @@
// limitations under the License.
use std::{
collections::BTreeSet,
collections::HashSet,
env,
sync::{Mutex, OnceLock},
@@ -30,7 +31,7 @@ use tracing::warn;
/// - `i8`: The parsed value as i8 if successful, otherwise the default value.
///
pub fn get_env_i8(key: &str, default: i8) -> i8 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
@@ -43,7 +44,7 @@ pub fn get_env_i8(key: &str, default: i8) -> i8 {
/// - `Option<i8>`: The parsed value as i8 if successful, otherwise None
///
pub fn get_env_opt_i8(key: &str) -> Option<i8> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
@@ -57,7 +58,7 @@ pub fn get_env_opt_i8(key: &str) -> Option<i8> {
/// - `u8`: The parsed value as u8 if successful, otherwise the default value.
///
pub fn get_env_u8(key: &str, default: u8) -> u8 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
@@ -70,7 +71,7 @@ pub fn get_env_u8(key: &str, default: u8) -> u8 {
/// - `Option<u8>`: The parsed value as u8 if successful, otherwise None
///
pub fn get_env_opt_u8(key: &str) -> Option<u8> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
static WARNED_ENV_MESSAGES: OnceLock<Mutex<HashSet<String>>> = OnceLock::new();
@@ -86,19 +87,182 @@ fn log_once(key: &str, message: impl FnOnce() -> String) {
}
}
fn external_alias_for_key(key: &str) -> Option<String> {
let suffix = key.strip_prefix("RUSTFS_")?;
if is_external_compatible_suffix(suffix) {
Some(format!("{}{}", external_env_prefix(), suffix))
} else {
None
}
}
fn resolve_env_with_aliases(key: &str, deprecated: &[&str]) -> Option<(String, String)> {
if let Ok(value) = env::var(key) {
return Some((key.to_string(), value));
}
let (alias, value) = deprecated
if let Some((alias, value)) = deprecated
.iter()
.find_map(|alias| env::var(alias).ok().map(|value| (*alias, value)))?;
.find_map(|alias| env::var(alias).ok().map(|value| (*alias, value)))
{
let deprecated_key = format!("env_alias:{alias}->{key}");
log_once(&deprecated_key, || {
format!("Environment variable {alias} is deprecated, use {key} instead")
});
return Some((alias.to_string(), value));
}
let alias = external_alias_for_key(key)?;
let value = env::var(&alias).ok()?;
let deprecated_key = format!("env_alias:{alias}->{key}");
log_once(&deprecated_key, || {
format!("Environment variable {alias} is deprecated, use {key} instead")
});
Some((alias.to_string(), value))
Some((alias, value))
}
const EXTERNAL_ENV_PREFIX_BYTES: [u8; 6] = [77, 73, 78, 73, 79, 95];
const EXTERNAL_COMPATIBLE_SUFFIXES: &[&str] = &[
"ACCESS_KEY",
"ACCESS_KEY_FILE",
"ADDRESS",
"API_XFF_HEADER",
"AUDIT_WEBHOOK_AUTH_TOKEN",
"AUDIT_WEBHOOK_CLIENT_CERT",
"AUDIT_WEBHOOK_CLIENT_KEY",
"AUDIT_WEBHOOK_ENABLE",
"AUDIT_WEBHOOK_ENDPOINT",
"AUDIT_WEBHOOK_QUEUE_DIR",
"COMPRESS_ENABLE",
"COMPRESS_EXTENSIONS",
"COMPRESS_MIME_TYPES",
"CONSOLE_ADDRESS",
"DRIVE_ACTIVE_MONITORING",
"ERASURE_SET_DRIVE_COUNT",
"IDENTITY_OPENID_CLAIM_NAME",
"IDENTITY_OPENID_CLAIM_PREFIX",
"IDENTITY_OPENID_CLIENT_ID",
"IDENTITY_OPENID_CLIENT_SECRET",
"IDENTITY_OPENID_CONFIG_URL",
"IDENTITY_OPENID_DISPLAY_NAME",
"IDENTITY_OPENID_REDIRECT_URI",
"IDENTITY_OPENID_SCOPES",
"ILM_EXPIRATION_WORKERS",
"LICENSE",
"NOTIFY_MQTT_BROKER",
"NOTIFY_MQTT_ENABLE",
"NOTIFY_MQTT_KEEP_ALIVE_INTERVAL",
"NOTIFY_MQTT_PASSWORD",
"NOTIFY_MQTT_QOS",
"NOTIFY_MQTT_QUEUE_DIR",
"NOTIFY_MQTT_QUEUE_LIMIT",
"NOTIFY_MQTT_RECONNECT_INTERVAL",
"NOTIFY_MQTT_TOPIC",
"NOTIFY_MQTT_USERNAME",
"NOTIFY_WEBHOOK_AUTH_TOKEN",
"NOTIFY_WEBHOOK_CLIENT_CERT",
"NOTIFY_WEBHOOK_CLIENT_KEY",
"NOTIFY_WEBHOOK_ENABLE",
"NOTIFY_WEBHOOK_ENDPOINT",
"NOTIFY_WEBHOOK_QUEUE_DIR",
"NOTIFY_WEBHOOK_QUEUE_LIMIT",
"POLICY_PLUGIN_AUTH_TOKEN",
"POLICY_PLUGIN_URL",
"PORT",
"REGION",
"ROOT_PASSWORD",
"ROOT_USER",
"SECRET_KEY",
"SECRET_KEY_FILE",
"STORAGE_CLASS_INLINE_BLOCK",
"STORAGE_CLASS_OPTIMIZE",
"STORAGE_CLASS_RRS",
"STORAGE_CLASS_STANDARD",
"VERSION",
"VOLUMES",
];
const EXTERNAL_DYNAMIC_COMPATIBLE_PREFIXES: &[&str] = &["AUDIT_MQTT_", "AUDIT_WEBHOOK_", "NOTIFY_MQTT_", "NOTIFY_WEBHOOK_"];
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct ExternalEnvCompatReport {
pub mapped_pairs: Vec<(String, String)>,
pub conflict_keys: Vec<String>,
}
impl ExternalEnvCompatReport {
pub fn mapped_count(&self) -> usize {
self.mapped_pairs.len()
}
pub fn conflict_count(&self) -> usize {
self.conflict_keys.len()
}
}
fn external_env_prefix() -> &'static str {
static PREFIX: OnceLock<String> = OnceLock::new();
PREFIX
.get_or_init(|| EXTERNAL_ENV_PREFIX_BYTES.iter().map(|&byte| char::from(byte)).collect())
.as_str()
}
fn is_external_compatible_suffix(suffix: &str) -> bool {
EXTERNAL_COMPATIBLE_SUFFIXES.contains(&suffix)
|| EXTERNAL_DYNAMIC_COMPATIBLE_PREFIXES
.iter()
.any(|prefix| suffix.starts_with(prefix))
}
fn build_external_env_compat_report_from_entries<I>(entries: I) -> ExternalEnvCompatReport
where
I: IntoIterator<Item = (String, String)>,
{
let env_map: std::collections::BTreeMap<String, String> = entries.into_iter().collect();
let mut mapped_pairs = BTreeSet::new();
let mut conflict_keys = BTreeSet::new();
let source_prefix = external_env_prefix();
for (source_key, source_value) in env_map.iter() {
let Some(suffix) = source_key.strip_prefix(source_prefix) else {
continue;
};
if !is_external_compatible_suffix(suffix) {
continue;
}
let rustfs_key = format!("RUSTFS_{suffix}");
match env_map.get(&rustfs_key) {
None => {
mapped_pairs.insert((source_key.clone(), rustfs_key));
}
Some(rustfs_value) if rustfs_value != source_value => {
conflict_keys.insert(rustfs_key);
}
Some(_) => {}
}
}
ExternalEnvCompatReport {
mapped_pairs: mapped_pairs.into_iter().collect(),
conflict_keys: conflict_keys.into_iter().collect(),
}
}
/// Build compatibility plan between source-prefixed variables and `RUSTFS_*`.
///
/// Precedence rule:
/// - If both `RUSTFS_*` and source-prefixed variables exist, keep `RUSTFS_*` and record a conflict.
/// - If only source-prefixed variables exist, mark them as mappable to `RUSTFS_*`.
pub fn build_external_env_compat_report() -> ExternalEnvCompatReport {
build_external_env_compat_report_from_entries(env::vars())
}
fn parse_env_value<T>(key: &str) -> Option<T>
where
T: std::str::FromStr,
{
resolve_env_with_aliases(key, &[]).and_then(|(_, value)| value.parse().ok())
}
pub fn get_env_str_with_aliases(key: &str, deprecated: &[&str], default: &str) -> String {
@@ -134,7 +298,7 @@ pub fn get_env_bool_with_aliases(key: &str, deprecated: &[&str], default: bool)
/// - `i16`: The parsed value as i16 if successful, otherwise the default value.
///
pub fn get_env_i16(key: &str, default: i16) -> i16 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
/// 16-bit type: signed i16
@@ -146,7 +310,7 @@ pub fn get_env_i16(key: &str, default: i16) -> i16 {
/// - `Option<i16>`: The parsed value as i16 if successful, otherwise None
///
pub fn get_env_opt_i16(key: &str) -> Option<i16> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
@@ -160,7 +324,7 @@ pub fn get_env_opt_i16(key: &str) -> Option<i16> {
/// - `u16`: The parsed value as u16 if successful, otherwise the default value.
///
pub fn get_env_u16(key: &str, default: u16) -> u16 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
/// 16-bit type: unsigned u16
@@ -172,7 +336,7 @@ pub fn get_env_u16(key: &str, default: u16) -> u16 {
/// - `Option<u16>`: The parsed value as u16 if successful, otherwise None
///
pub fn get_env_u16_opt(key: &str) -> Option<u16> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
/// 16-bit type: unsigned u16
@@ -197,7 +361,7 @@ pub fn get_env_opt_u16(key: &str) -> Option<u16> {
/// - `i32`: The parsed value as i32 if successful, otherwise the default value.
///
pub fn get_env_i32(key: &str, default: i32) -> i32 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
/// 32-bit type: signed i32
@@ -209,7 +373,7 @@ pub fn get_env_i32(key: &str, default: i32) -> i32 {
/// - `Option<i32>`: The parsed value as i32 if successful, otherwise None
///
pub fn get_env_opt_i32(key: &str) -> Option<i32> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
@@ -223,7 +387,7 @@ pub fn get_env_opt_i32(key: &str) -> Option<i32> {
/// - `u32`: The parsed value as u32 if successful, otherwise the default value.
///
pub fn get_env_u32(key: &str, default: u32) -> u32 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
/// 32-bit type: unsigned u32
@@ -235,7 +399,7 @@ pub fn get_env_u32(key: &str, default: u32) -> u32 {
/// - `Option<u32>`: The parsed value as u32 if successful, otherwise None
///
pub fn get_env_opt_u32(key: &str) -> Option<u32> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
///
@@ -247,7 +411,7 @@ pub fn get_env_opt_u32(key: &str) -> Option<u32> {
/// - `f32`: The parsed value as f32 if successful, otherwise the default value
///
pub fn get_env_f32(key: &str, default: f32) -> f32 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
///
@@ -258,7 +422,7 @@ pub fn get_env_f32(key: &str, default: f32) -> f32 {
/// - `Option<f32>`: The parsed value as f32 if successful, otherwise None
///
pub fn get_env_opt_f32(key: &str) -> Option<f32> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
@@ -271,7 +435,7 @@ pub fn get_env_opt_f32(key: &str) -> Option<f32> {
/// - `i64`: The parsed value as i64 if successful, otherwise the default value
///
pub fn get_env_i64(key: &str, default: i64) -> i64 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
///
@@ -282,7 +446,7 @@ pub fn get_env_i64(key: &str, default: i64) -> i64 {
/// - `Option<i64>`: The parsed value as i64 if successful, otherwise None
///
pub fn get_env_opt_i64(key: &str) -> Option<i64> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, returning Option<Option<i64>> if not set or parsing fails.
@@ -294,7 +458,7 @@ pub fn get_env_opt_i64(key: &str) -> Option<i64> {
/// - `Option<Option<i64>>`: The parsed value as i64 if successful, otherwise None
///
pub fn get_env_opt_opt_i64(key: &str) -> Option<Option<i64>> {
env::var(key).ok().map(|v| v.parse().ok())
resolve_env_with_aliases(key, &[]).map(|(_, value)| value.parse().ok())
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
@@ -307,7 +471,7 @@ pub fn get_env_opt_opt_i64(key: &str) -> Option<Option<i64>> {
/// - `u64`: The parsed value as u64 if successful, otherwise the default value.
///
pub fn get_env_u64(key: &str, default: u64) -> u64 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as an unsigned 64-bit integer, returning `None` if not set or parsing fails.
@@ -341,7 +505,7 @@ pub fn get_env_opt_u64_with_aliases(key: &str, deprecated: &[&str]) -> Option<u6
/// - `Option<u64>`: The parsed value as u64 if successful, otherwise None
///
pub fn get_env_opt_u64(key: &str) -> Option<u64> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
@@ -354,7 +518,7 @@ pub fn get_env_opt_u64(key: &str) -> Option<u64> {
/// - `f64`: The parsed value as f64 if successful, otherwise the default value.
///
pub fn get_env_f64(key: &str, default: f64) -> f64 {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
@@ -366,7 +530,7 @@ pub fn get_env_f64(key: &str, default: f64) -> f64 {
/// - `Option<f64>`: The parsed value as f64 if successful, otherwise None
///
pub fn get_env_opt_f64(key: &str) -> Option<f64> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails.
@@ -379,7 +543,7 @@ pub fn get_env_opt_f64(key: &str) -> Option<f64> {
/// - `usize`: The parsed value as usize if successful, otherwise the default value.
///
pub fn get_env_usize(key: &str, default: usize) -> usize {
env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
parse_env_value(key).unwrap_or(default)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
///
@@ -390,7 +554,7 @@ pub fn get_env_usize(key: &str, default: usize) -> usize {
/// - `Option<usize>`: The parsed value as usize if successful, otherwise None
///
pub fn get_env_usize_opt(key: &str) -> Option<usize> {
env::var(key).ok().and_then(|v| v.parse().ok())
parse_env_value(key)
}
/// Retrieve an environment variable as a specific type, returning None if not set or parsing fails.
@@ -427,7 +591,7 @@ pub fn get_env_str(key: &str, default: &str) -> String {
/// - `Option<String>`: The environment variable value if set, otherwise None.
///
pub fn get_env_opt_str(key: &str) -> Option<String> {
env::var(key).ok()
resolve_env_with_aliases(key, &[]).map(|(_, value)| value)
}
/// Retrieve an environment variable as a boolean, with a default value if not set or parsing fails.
@@ -476,3 +640,105 @@ pub fn get_env_opt_bool(key: &str) -> Option<bool> {
None
})
}
/// Copy supported external-prefix variables such as `MINIO_*` into their
/// canonical `RUSTFS_*` names in the current process when the canonical key is
/// missing.
#[allow(unsafe_code)]
pub fn apply_external_env_compat() -> ExternalEnvCompatReport {
let report = build_external_env_compat_report();
for (source_key, rustfs_key) in &report.mapped_pairs {
if let Ok(value) = env::var(source_key) {
// Safety: this helper is intended for early startup bootstrap
// before any background threads are created.
unsafe {
env::set_var(rustfs_key, value);
}
}
}
report
}
#[cfg(test)]
mod tests {
use super::{apply_external_env_compat, build_external_env_compat_report_from_entries, get_env_str};
fn source_key(suffix: &str) -> String {
let mut key = super::external_env_prefix().to_string();
key.push_str(suffix);
key
}
#[test]
fn source_value_is_mapped_when_rustfs_missing() {
let report =
build_external_env_compat_report_from_entries(vec![(source_key("STORAGE_CLASS_STANDARD"), "EC:2".to_string())]);
assert_eq!(report.mapped_count(), 1);
assert!(
report
.mapped_pairs
.iter()
.any(|(input_key, rustfs_key)| input_key == &source_key("STORAGE_CLASS_STANDARD")
&& rustfs_key == "RUSTFS_STORAGE_CLASS_STANDARD")
);
assert_eq!(report.conflict_count(), 0);
}
#[test]
fn rustfs_value_takes_precedence_on_conflict() {
let report = build_external_env_compat_report_from_entries(vec![
("RUSTFS_ERASURE_SET_DRIVE_COUNT".to_string(), "8".to_string()),
(source_key("ERASURE_SET_DRIVE_COUNT"), "16".to_string()),
]);
assert_eq!(report.mapped_count(), 0);
assert_eq!(report.conflict_count(), 1);
assert!(report.conflict_keys.iter().any(|key| key == "RUSTFS_ERASURE_SET_DRIVE_COUNT"));
}
#[test]
fn dynamic_notify_suffix_is_mapped() {
let report =
build_external_env_compat_report_from_entries(vec![(source_key("NOTIFY_WEBHOOK_ENABLE_PRIMARY"), "on".to_string())]);
assert_eq!(report.mapped_count(), 1);
assert!(
report
.mapped_pairs
.iter()
.any(|(input_key, rustfs_key)| input_key == &source_key("NOTIFY_WEBHOOK_ENABLE_PRIMARY")
&& rustfs_key == "RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY")
);
assert_eq!(report.conflict_count(), 0);
}
#[test]
fn unrelated_source_key_is_ignored() {
let report = build_external_env_compat_report_from_entries(vec![(source_key("UNKNOWN_COMPAT_TEST"), "1".to_string())]);
assert_eq!(report.mapped_count(), 0);
assert_eq!(report.conflict_count(), 0);
}
#[test]
fn minio_alias_is_used_for_rustfs_reads() {
temp_env::with_var("MINIO_ROOT_USER", Some("compat-admin"), || {
temp_env::with_var_unset("RUSTFS_ROOT_USER", || {
assert_eq!(get_env_str("RUSTFS_ROOT_USER", "default-user"), "compat-admin");
});
});
}
#[test]
fn apply_external_env_compat_copies_missing_rustfs_keys() {
temp_env::with_var("MINIO_ROOT_USER", Some("compat-admin"), || {
temp_env::with_var_unset("RUSTFS_ROOT_USER", || {
let report = apply_external_env_compat();
assert!(
report
.mapped_pairs
.iter()
.any(|(source_key, rustfs_key)| source_key == "MINIO_ROOT_USER" && rustfs_key == "RUSTFS_ROOT_USER")
);
assert_eq!(std::env::var("RUSTFS_ROOT_USER").as_deref(), Ok("compat-admin"));
});
});
}
}