gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-28 16:07:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(logging): normalize admin telemetry and error messages (#3430)

Browse Source
This commit is contained in:
houseme
2026-06-14 13:27:10 +08:00
committed by GitHub
parent dc82efbab4
commit e8012bd1ba
70 changed files with 4807 additions and 1445 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/kms/src/backends/local.rs
+12 -12
View File
@@ -34,7 +34,7 @@ use std::path::PathBuf;
use std::time::Duration;
use tokio::fs;
use tokio::sync::RwLock;
use tracing::{debug, info, warn};
use tracing::{debug, warn};
/// Local KMS client that stores keys in local files
pub struct LocalKmsClient {
@@ -74,7 +74,7 @@ impl LocalKmsClient {
// Create key directory if it doesn't exist
if !config.key_dir.exists() {
fs::create_dir_all(&config.key_dir).await?;
info!("Created KMS key directory: {:?}", config.key_dir);
debug!(path = ?config.key_dir, "KMS key directory created");
}
// Initialize master cipher if master key is provided
@@ -219,7 +219,7 @@ impl LocalKmsClient {
fs::rename(&temp_path, &key_path).await?;
info!("Saved master key {} to {:?}", master_key.key_id, key_path);
debug!(key_id = %master_key.key_id, path = ?key_path, "Local KMS master key saved");
Ok(())
}
@@ -278,7 +278,7 @@ impl KmsClient for LocalKmsClient {
let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone());
info!("Generated data key for master key: {}", request.master_key_id);
debug!(key_id = %request.master_key_id, "Local KMS data key generated");
Ok(data_key)
}
@@ -334,7 +334,7 @@ impl KmsClient for LocalKmsClient {
.decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce)
.await?;
info!("Successfully decrypted data");
debug!("Local KMS data decrypted");
Ok(plaintext)
}
@@ -367,7 +367,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key.clone());
info!("Created master key: {}", key_id);
debug!(key_id, "Local KMS master key created");
Ok(master_key)
}
@@ -453,7 +453,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
info!("Enabled key: {}", key_id);
debug!(key_id, "Local KMS key enabled");
Ok(())
}
@@ -470,7 +470,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
info!("Disabled key: {}", key_id);
debug!(key_id, "Local KMS key disabled");
Ok(())
}
@@ -492,7 +492,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
warn!("Scheduled key deletion: {}", key_id);
debug!(key_id, "Local KMS key deletion scheduled");
Ok(())
}
@@ -509,7 +509,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key);
info!("Canceled deletion for key: {}", key_id);
debug!(key_id, "Local KMS key deletion canceled");
Ok(())
}
@@ -528,7 +528,7 @@ impl KmsClient for LocalKmsClient {
let mut cache = self.key_cache.write().await;
cache.insert(key_id.to_string(), master_key.clone());
info!("Rotated key: {}", key_id);
debug!(key_id, "Local KMS key rotated");
Ok(master_key)
}
@@ -721,7 +721,7 @@ impl KmsBackend for LocalKmsBackend {
let mut cache = self.client.key_cache.write().await;
cache.remove(key_id);
info!("Immediately deleted key: {}", key_id);
debug!(key_id, "Local KMS key deleted immediately");
// Return success response for immediate deletion
let key_metadata = KeyMetadata {
crates/kms/src/backends/vault.rs
+13 -13
View File
@@ -98,7 +98,7 @@ impl VaultKmsClient {
let client =
VaultClient::new(settings).map_err(|e| KmsError::backend_error(format!("Failed to create Vault client: {e}")))?;
info!("Successfully connected to Vault at {}", config.address);
info!(address = %config.address, "Vault KMS backend connected");
Ok(Self {
client,
@@ -136,7 +136,7 @@ impl VaultKmsClient {
// If encrypted_key_material is empty, generate and store it (fix for old keys)
if key_data.encrypted_key_material.is_empty() {
warn!("Key {} has empty encrypted_key_material, generating and storing new key material", key_id);
warn!(key_id, "Vault KMS key material missing; regenerating");
let key_material = generate_key_material(&key_data.algorithm)?;
key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?;
// Store the updated key data back to Vault
@@ -147,7 +147,7 @@ impl VaultKmsClient {
let key_material = match self.decrypt_key_material(&key_data.encrypted_key_material).await {
Ok(km) => km,
Err(e) => {
warn!("Failed to decrypt key material for key {}: {}, generating new key material", key_id, e);
warn!(key_id, error = %e, "Vault KMS key material decrypt failed; regenerating");
let new_key_material = generate_key_material(&key_data.algorithm)?;
key_data.encrypted_key_material = self.encrypt_key_material(&new_key_material).await?;
// Store the updated key data back to Vault
@@ -210,7 +210,7 @@ impl VaultKmsClient {
// If encrypted_key_material is empty, generate it (this handles the case where
// an old key was created without proper key material)
if existing_key_data.encrypted_key_material.is_empty() {
warn!("Key {} has empty encrypted_key_material, generating new key material", key_id);
warn!(key_id, "Vault KMS key metadata missing encrypted key material");
let key_material = generate_key_material(&existing_key_data.algorithm)?;
existing_key_data.encrypted_key_material = self.encrypt_key_material(&key_material).await?;
}
@@ -316,7 +316,7 @@ impl KmsClient for VaultKmsClient {
let data_key = DataKeyInfo::new(envelope.key_id, 1, Some(plaintext_key), ciphertext, request.key_spec.clone());
info!("Generated data key for master key: {}", request.master_key_id);
debug!(key_id = %request.master_key_id, "Vault KMS data key generated");
Ok(data_key)
}
@@ -373,7 +373,7 @@ impl KmsClient for VaultKmsClient {
.decrypt_with_master_key(&envelope.master_key_id, &envelope.encrypted_key, &envelope.nonce)
.await?;
info!("Successfully decrypted data");
debug!("Vault KMS data decrypted");
Ok(plaintext)
}
@@ -418,7 +418,7 @@ impl KmsClient for VaultKmsClient {
created_by: None,
};
info!("Successfully created master key: {}", key_id);
debug!(key_id, "Vault KMS master key created");
Ok(master_key)
}
@@ -486,7 +486,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::Active;
self.store_key_data(key_id, &key_data).await?;
info!("Enabled key: {}", key_id);
debug!(key_id, "Vault KMS key enabled");
Ok(())
}
@@ -497,7 +497,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::Disabled;
self.store_key_data(key_id, &key_data).await?;
info!("Disabled key: {}", key_id);
debug!(key_id, "Vault KMS key disabled");
Ok(())
}
@@ -513,7 +513,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::PendingDeletion;
self.store_key_data(key_id, &key_data).await?;
info!("Scheduled key deletion: {}", key_id);
debug!(key_id, "Vault KMS key deletion scheduled");
Ok(())
}
@@ -524,7 +524,7 @@ impl KmsClient for VaultKmsClient {
key_data.status = KeyStatus::Active;
self.store_key_data(key_id, &key_data).await?;
info!("Canceled key deletion: {}", key_id);
debug!(key_id, "Vault KMS key deletion canceled");
Ok(())
}
@@ -553,7 +553,7 @@ impl KmsClient for VaultKmsClient {
created_by: None,
};
info!("Successfully rotated key: {}", key_id);
debug!(key_id, "Vault KMS key rotated");
Ok(master_key)
}
@@ -573,7 +573,7 @@ impl KmsClient for VaultKmsClient {
debug!("Vault health check passed - 404 error is expected when no keys exist yet");
Ok(())
} else {
warn!("Vault health check failed: {}", e);
warn!(error = %e, "Vault KMS health check failed");
Err(e)
}
}
crates/kms/src/service.rs
+19 -7
View File
@@ -24,7 +24,7 @@ use rand::random;
use std::collections::HashMap;
use std::io::Cursor;
use tokio::io::{AsyncRead, AsyncReadExt};
use tracing::{debug, info};
use tracing::debug;
use zeroize::Zeroize;
/// Data key for object encryption
@@ -209,7 +209,7 @@ impl ObjectEncryptionService {
// Generate a unique random nonce for this data key
// This ensures each object/part gets a unique base nonce for streaming encryption
let nonce: [u8; 12] = random();
tracing::info!("Generated random nonce for data key: {:02x?}", nonce);
tracing::debug!("Generated random nonce for data key");
let data_key = DataKey {
plaintext_key: data_key_response
@@ -302,7 +302,7 @@ impl ObjectEncryptionService {
key_id: actual_key_id.to_string(),
};
if let Err(KmsError::KeyNotFound { .. }) = self.kms_manager.describe_key(describe_req).await {
info!("Auto-creating SSE-S3 key: {}", actual_key_id);
debug!(key_id = %actual_key_id, "Auto-creating SSE-S3 key");
let create_req = CreateKeyRequest {
key_name: Some(actual_key_id.to_string()),
key_usage: KeyUsage::EncryptDecrypt,
@@ -364,7 +364,13 @@ impl ObjectEncryptionService {
encrypted_data_key: data_key.ciphertext_blob,
};
info!("Successfully encrypted object {}/{} ({} bytes)", bucket, object_key, original_size);
debug!(
bucket,
object = object_key,
original_size,
algorithm = %algorithm.as_str(),
"Object encrypted"
);
Ok(EncryptionResult { ciphertext, metadata })
}
@@ -429,7 +435,13 @@ impl ObjectEncryptionService {
// Decrypt the data
let plaintext = cipher.decrypt(&ciphertext, &metadata.iv, tag, &aad)?;
info!("Successfully decrypted object {}/{} ({} bytes)", bucket, object_key, plaintext.len());
debug!(
bucket,
object = object_key,
plaintext_len = plaintext.len(),
algorithm = %metadata.algorithm,
"Object decrypted"
);
Ok(Box::new(Cursor::new(plaintext)))
}
@@ -507,7 +519,7 @@ impl ObjectEncryptionService {
encrypted_data_key: Vec::new(), // Empty for SSE-C
};
info!(
debug!(
"Successfully encrypted object {}/{} with SSE-C ({} bytes)",
bucket, object_key, original_size
);
@@ -568,7 +580,7 @@ impl ObjectEncryptionService {
// Decrypt the data
let plaintext = cipher.decrypt(&ciphertext, &metadata.iv, tag, &aad)?;
info!(
debug!(
"Successfully decrypted SSE-C object {}/{} ({} bytes)",
bucket,
object_key,
crates/kms/src/service_manager.rs
+61 -11
View File
@@ -25,7 +25,11 @@ use std::sync::{
atomic::{AtomicU64, Ordering},
};
use tokio::sync::{Mutex, RwLock};
use tracing::{error, info, warn};
use tracing::{debug, error, info, warn};
const LOG_COMPONENT_KMS: &str = "kms";
const LOG_SUBSYSTEM_SERVICE: &str = "service";
const EVENT_KMS_SERVICE_STATE: &str = "kms_service_state";
/// KMS service status
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
@@ -106,7 +110,13 @@ impl KmsServiceManager {
*status = KmsServiceStatus::Configured;
}
info!("KMS configuration updated successfully");
debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "configured",
"KMS service configured"
);
Ok(())
}
@@ -132,7 +142,14 @@ impl KmsServiceManager {
}
};
info!("Starting KMS service with backend: {:?}", config.backend);
info!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
backend = ?config.backend,
state = "starting",
"KMS service starting"
);
match self.create_service_version(&config).await {
Ok(service_version) => {
@@ -146,7 +163,13 @@ impl KmsServiceManager {
*status = KmsServiceStatus::Running;
}
info!("KMS service started successfully");
debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "running",
"KMS service running"
);
Ok(())
}
Err(e) => {
@@ -170,7 +193,13 @@ impl KmsServiceManager {
/// Internal stop implementation (called within lifecycle mutex)
async fn stop_internal(&self) -> Result<()> {
info!("Stopping KMS service");
debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "stopping",
"KMS service stopping"
);
// Atomically clear current service version (lock-free, instant)
// Note: Existing Arc references will keep the service alive until operations complete
@@ -184,7 +213,13 @@ impl KmsServiceManager {
}
}
info!("KMS service stopped successfully (existing operations may continue)");
debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "configured",
"KMS service stopped"
);
Ok(())
}
@@ -201,7 +236,13 @@ impl KmsServiceManager {
pub async fn reconfigure(&self, new_config: KmsConfig) -> Result<()> {
let _guard = self.lifecycle_mutex.lock().await;
info!("Reconfiguring KMS service (zero-downtime)");
debug!(
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
state = "reconfiguring",
"KMS service reconfiguring"
);
new_config.validate()?;
// Configure with new config
@@ -230,13 +271,22 @@ impl KmsServiceManager {
if let Some(old_ver) = old_version {
info!(
"KMS service reconfigured successfully: version {} -> {} (old service will be cleaned up when operations complete)",
old_ver, new_service_version.version
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
old_version = old_ver,
new_version = new_service_version.version,
state = "running",
"KMS service reconfigured"
);
} else {
info!(
"KMS service reconfigured successfully: version {} (service started)",
new_service_version.version
event = EVENT_KMS_SERVICE_STATE,
component = LOG_COMPONENT_KMS,
subsystem = LOG_SUBSYSTEM_SERVICE,
new_version = new_service_version.version,
state = "running",
"KMS service started from reconfigure"
);
}
Ok(())