chore: upgrade dependencies and migrate to aws-lc-rs (#1333)

This commit is contained in:
houseme
2026-01-02 00:02:34 +08:00
committed by GitHub
parent 61b3100260
commit 8d7cd4cb1b
96 changed files with 2555 additions and 2775 deletions
Generated
+129 -245
View File
File diff suppressed because it is too large Load Diff
+7 -7
View File
@@ -50,7 +50,7 @@ resolver = "2"
edition = "2024" edition = "2024"
license = "Apache-2.0" license = "Apache-2.0"
repository = "https://github.com/rustfs/rustfs" repository = "https://github.com/rustfs/rustfs"
rust-version = "1.85" rust-version = "1.88"
version = "0.0.5" version = "0.0.5"
homepage = "https://rustfs.com" homepage = "https://rustfs.com"
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. " description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
@@ -100,21 +100,21 @@ async-compression = { version = "0.4.19" }
async-recursion = "1.1.1" async-recursion = "1.1.1"
async-trait = "0.1.89" async-trait = "0.1.89"
axum = "0.8.8" axum = "0.8.8"
axum-server = { version = "0.8.0", features = ["tls-rustls-no-provider"], default-features = false } axum-server = { version = "0.8.0", features = ["tls-rustls"], default-features = false }
futures = "0.3.31" futures = "0.3.31"
futures-core = "0.3.31" futures-core = "0.3.31"
futures-util = "0.3.31" futures-util = "0.3.31"
pollster = "0.4.0" pollster = "0.4.0"
hyper = { version = "1.8.1", features = ["http2", "http1", "server"] } hyper = { version = "1.8.1", features = ["http2", "http1", "server"] }
hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "ring", "webpki-roots"] } hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
hyper-util = { version = "0.1.19", features = ["tokio", "server-auto", "server-graceful"] } hyper-util = { version = "0.1.19", features = ["tokio", "server-auto", "server-graceful"] }
http = "1.4.0" http = "1.4.0"
http-body = "1.0.1" http-body = "1.0.1"
http-body-util = "0.1.3" http-body-util = "0.1.3"
reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2", "system-proxy", "stream", "json", "blocking"] } reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls-no-provider", "charset", "http2", "system-proxy", "stream", "json", "blocking"] }
socket2 = "0.6.1" socket2 = "0.6.1"
tokio = { version = "1.48.0", features = ["fs", "rt-multi-thread"] } tokio = { version = "1.48.0", features = ["fs", "rt-multi-thread"] }
tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "ring"] } tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
tokio-stream = { version = "0.1.17" } tokio-stream = { version = "0.1.17" }
tokio-test = "0.4.4" tokio-test = "0.4.4"
tokio-util = { version = "0.7.17", features = ["io", "compat"] } tokio-util = { version = "0.7.17", features = ["io", "compat"] }
@@ -150,7 +150,7 @@ hmac = { version = "0.13.0-rc.3" }
jsonwebtoken = { version = "10.2.0", features = ["rust_crypto"] } jsonwebtoken = { version = "10.2.0", features = ["rust_crypto"] }
pbkdf2 = "0.13.0-rc.5" pbkdf2 = "0.13.0-rc.5"
rsa = { version = "0.10.0-rc.10" } rsa = { version = "0.10.0-rc.10" }
rustls = { version = "0.23.35", features = ["ring", "logging", "std", "tls12"], default-features = false } rustls = { version = "0.23.35" }
rustls-pemfile = "2.2.0" rustls-pemfile = "2.2.0"
rustls-pki-types = "1.13.2" rustls-pki-types = "1.13.2"
sha1 = "0.11.0-rc.3" sha1 = "0.11.0-rc.3"
@@ -171,7 +171,7 @@ atoi = "2.0.0"
atomic_enum = "0.3.0" atomic_enum = "0.3.0"
aws-config = { version = "1.8.12" } aws-config = { version = "1.8.12" }
aws-credential-types = { version = "1.2.11" } aws-credential-types = { version = "1.2.11" }
aws-sdk-s3 = { version = "1.119.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] } aws-sdk-s3 = { version = "1.119.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
aws-smithy-types = { version = "1.3.5" } aws-smithy-types = { version = "1.3.5" }
base64 = "0.22.1" base64 = "0.22.1"
base64-simd = "0.8.0" base64-simd = "0.8.0"
+1 -1
View File
@@ -348,7 +348,7 @@ impl ErasureSetHealer {
} }
// save checkpoint periodically // save checkpoint periodically
if global_obj_idx % 100 == 0 { if global_obj_idx.is_multiple_of(100) {
checkpoint_manager checkpoint_manager
.update_position(bucket_index, *current_object_index) .update_position(bucket_index, *current_object_index)
.await?; .await?;
+2 -3
View File
@@ -492,14 +492,13 @@ impl HealManager {
for (_, disk_opt) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() { for (_, disk_opt) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() {
if let Some(disk) = disk_opt { if let Some(disk) = disk_opt {
// detect unformatted disk via get_disk_id() // detect unformatted disk via get_disk_id()
if let Err(err) = disk.get_disk_id().await { if let Err(err) = disk.get_disk_id().await
if err == DiskError::UnformattedDisk { && err == DiskError::UnformattedDisk {
endpoints.push(disk.endpoint()); endpoints.push(disk.endpoint());
continue; continue;
} }
} }
} }
}
if endpoints.is_empty() { if endpoints.is_empty() {
info!("start_auto_disk_scanner: No endpoints need healing"); info!("start_auto_disk_scanner: No endpoints need healing");
+3 -3
View File
@@ -541,13 +541,13 @@ impl ResumeUtils {
for entry in entries { for entry in entries {
if entry.ends_with(&format!("_{RESUME_STATE_FILE}")) { if entry.ends_with(&format!("_{RESUME_STATE_FILE}")) {
// Extract task ID from filename: {task_id}_ahm_resume_state.json // Extract task ID from filename: {task_id}_ahm_resume_state.json
if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}")) { if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}"))
if !task_id.is_empty() { && !task_id.is_empty()
{
task_ids.push(task_id.to_string()); task_ids.push(task_id.to_string());
} }
} }
} }
}
debug!("Found {} resumable tasks: {:?}", task_ids.len(), task_ids); debug!("Found {} resumable tasks: {:?}", task_ids.len(), task_ids);
Ok(task_ids) Ok(task_ids)
+3 -3
View File
@@ -83,11 +83,11 @@ pub struct CheckpointManager {
impl CheckpointManager { impl CheckpointManager {
pub fn new(node_id: &str, data_dir: &Path) -> Self { pub fn new(node_id: &str, data_dir: &Path) -> Self {
if !data_dir.exists() { if !data_dir.exists()
if let Err(e) = std::fs::create_dir_all(data_dir) { && let Err(e) = std::fs::create_dir_all(data_dir)
{
error!("create data dir failed {:?}: {}", data_dir, e); error!("create data dir failed {:?}: {}", data_dir, e);
} }
}
let checkpoint_file = data_dir.join(format!("scanner_checkpoint_{node_id}.json")); let checkpoint_file = data_dir.join(format!("scanner_checkpoint_{node_id}.json"));
let backup_file = data_dir.join(format!("scanner_checkpoint_{node_id}.backup")); let backup_file = data_dir.join(format!("scanner_checkpoint_{node_id}.backup"));
+53 -86
View File
@@ -401,11 +401,11 @@ impl Scanner {
let mut latest_update: Option<SystemTime> = None; let mut latest_update: Option<SystemTime> = None;
for snapshot in &outcome.snapshots { for snapshot in &outcome.snapshots {
if let Some(update) = snapshot.last_update { if let Some(update) = snapshot.last_update
if latest_update.is_none_or(|current| update > current) { && latest_update.is_none_or(|current| update > current)
{
latest_update = Some(update); latest_update = Some(update);
} }
}
aggregated.objects_total_count = aggregated.objects_total_count.saturating_add(snapshot.objects_total_count); aggregated.objects_total_count = aggregated.objects_total_count.saturating_add(snapshot.objects_total_count);
aggregated.versions_total_count = aggregated.versions_total_count.saturating_add(snapshot.versions_total_count); aggregated.versions_total_count = aggregated.versions_total_count.saturating_add(snapshot.versions_total_count);
@@ -527,28 +527,20 @@ impl Scanner {
let (disks, _) = set_disks.get_online_disks_with_healing(false).await; let (disks, _) = set_disks.get_online_disks_with_healing(false).await;
if let Some(disk) = disks.first() { if let Some(disk) = disks.first() {
let bucket_path = disk.path().join(bucket_name); let bucket_path = disk.path().join(bucket_name);
if bucket_path.exists() { if bucket_path.exists()
if let Ok(entries) = std::fs::read_dir(&bucket_path) { && let Ok(entries) = std::fs::read_dir(&bucket_path)
{
for entry in entries.flatten() { for entry in entries.flatten() {
if let Ok(file_type) = entry.file_type() { if let Ok(file_type) = entry.file_type()
if file_type.is_dir() { && file_type.is_dir()
if let Some(object_name) = entry.file_name().to_str() { && let Some(object_name) = entry.file_name().to_str()
if !object_name.starts_with('.') { && !object_name.starts_with('.')
{
debug!("Deep scanning object: {}/{}", bucket_name, object_name); debug!("Deep scanning object: {}/{}", bucket_name, object_name);
if let Err(e) = self.verify_object_integrity(bucket_name, object_name).await { if let Err(e) = self.verify_object_integrity(bucket_name, object_name).await {
warn!( warn!("Object integrity verification failed for {}/{}: {}", bucket_name, object_name, e);
"Object integrity verification failed for {}/{}: {}",
bucket_name, object_name, e
);
} else { } else {
debug!( debug!("Object integrity verification passed for {}/{}", bucket_name, object_name);
"Object integrity verification passed for {}/{}",
bucket_name, object_name
);
}
}
}
}
} }
} }
} }
@@ -859,11 +851,11 @@ impl Scanner {
// Phase 2: Minimal EC verification for critical objects only // Phase 2: Minimal EC verification for critical objects only
// Note: The main scanning is now handled by NodeScanner in the background // Note: The main scanning is now handled by NodeScanner in the background
if let Some(ecstore) = rustfs_ecstore::new_object_layer_fn() { if let Some(ecstore) = rustfs_ecstore::new_object_layer_fn()
if let Err(e) = self.minimal_ec_verification(&ecstore).await { && let Err(e) = self.minimal_ec_verification(&ecstore).await
{
error!("Minimal EC verification failed: {}", e); error!("Minimal EC verification failed: {}", e);
} }
}
// Update scan duration // Update scan duration
let scan_duration = SystemTime::now().duration_since(start_time).unwrap_or(Duration::ZERO); let scan_duration = SystemTime::now().duration_since(start_time).unwrap_or(Duration::ZERO);
@@ -950,14 +942,13 @@ impl Scanner {
} }
// If there is still no data, try backend before persisting zeros // If there is still no data, try backend before persisting zeros
if data_usage.buckets_usage.is_empty() { if data_usage.buckets_usage.is_empty()
if let Ok(existing) = rustfs_ecstore::data_usage::load_data_usage_from_backend(ecstore.clone()).await { && let Ok(existing) = rustfs_ecstore::data_usage::load_data_usage_from_backend(ecstore.clone()).await
if !existing.buckets_usage.is_empty() { && !existing.buckets_usage.is_empty()
{
info!("Using existing backend data usage during fallback backoff"); info!("Using existing backend data usage during fallback backoff");
data_usage = existing; data_usage = existing;
} }
}
}
// Avoid overwriting valid backend stats with zeros when fallback is throttled // Avoid overwriting valid backend stats with zeros when fallback is throttled
if data_usage.buckets_usage.is_empty() { if data_usage.buckets_usage.is_empty() {
@@ -1721,8 +1712,7 @@ impl Scanner {
// check disk status, if offline, submit erasure set heal task // check disk status, if offline, submit erasure set heal task
if !metrics.is_online { if !metrics.is_online {
let enable_healing = self.config.read().await.enable_healing; let enable_healing = self.config.read().await.enable_healing;
if enable_healing { if enable_healing && let Some(heal_manager) = &self.heal_manager {
if let Some(heal_manager) = &self.heal_manager {
// Get bucket list for erasure set healing // Get bucket list for erasure set healing
let buckets = match rustfs_ecstore::new_object_layer_fn() { let buckets = match rustfs_ecstore::new_object_layer_fn() {
Some(ecstore) => match ecstore.list_bucket(&ecstore::store_api::BucketOptions::default()).await { Some(ecstore) => match ecstore.list_bucket(&ecstore::store_api::BucketOptions::default()).await {
@@ -1754,7 +1744,6 @@ impl Scanner {
} }
} }
} }
}
// Additional disk info for debugging // Additional disk info for debugging
debug!( debug!(
@@ -1778,8 +1767,7 @@ impl Scanner {
// disk access failed, submit erasure set heal task // disk access failed, submit erasure set heal task
let enable_healing = self.config.read().await.enable_healing; let enable_healing = self.config.read().await.enable_healing;
if enable_healing { if enable_healing && let Some(heal_manager) = &self.heal_manager {
if let Some(heal_manager) = &self.heal_manager {
// Get bucket list for erasure set healing // Get bucket list for erasure set healing
let buckets = match rustfs_ecstore::new_object_layer_fn() { let buckets = match rustfs_ecstore::new_object_layer_fn() {
Some(ecstore) => match ecstore.list_bucket(&ecstore::store_api::BucketOptions::default()).await { Some(ecstore) => match ecstore.list_bucket(&ecstore::store_api::BucketOptions::default()).await {
@@ -1810,7 +1798,6 @@ impl Scanner {
} }
} }
} }
}
return Err(Error::Storage(e.into())); return Err(Error::Storage(e.into()));
} }
@@ -1820,12 +1807,12 @@ impl Scanner {
let mut disk_objects = HashMap::new(); let mut disk_objects = HashMap::new();
for volume in volumes { for volume in volumes {
// check cancel token // check cancel token
if let Some(cancel_token) = get_ahm_services_cancel_token() { if let Some(cancel_token) = get_ahm_services_cancel_token()
if cancel_token.is_cancelled() { && cancel_token.is_cancelled()
{
info!("Cancellation requested, stopping disk scan"); info!("Cancellation requested, stopping disk scan");
break; break;
} }
}
match self.scan_volume(disk, &volume.name).await { match self.scan_volume(disk, &volume.name).await {
Ok(object_metadata) => { Ok(object_metadata) => {
@@ -1955,29 +1942,22 @@ impl Scanner {
// object metadata damaged, submit metadata heal task // object metadata damaged, submit metadata heal task
let enable_healing = self.config.read().await.enable_healing; let enable_healing = self.config.read().await.enable_healing;
if enable_healing { if enable_healing && let Some(heal_manager) = &self.heal_manager {
if let Some(heal_manager) = &self.heal_manager {
let req = HealRequest::metadata(bucket.to_string(), entry.name.clone()); let req = HealRequest::metadata(bucket.to_string(), entry.name.clone());
match heal_manager.submit_heal_request(req).await { match heal_manager.submit_heal_request(req).await {
Ok(task_id) => { Ok(task_id) => {
warn!( warn!("object metadata damaged, submit heal task: {} {} / {}", task_id, bucket, entry.name);
"object metadata damaged, submit heal task: {} {} / {}",
task_id, bucket, entry.name
);
} }
Err(e) => { Err(e) => {
error!( error!("object metadata damaged, submit heal task failed: {} / {} {}", bucket, entry.name, e);
"object metadata damaged, submit heal task failed: {} / {} {}",
bucket, entry.name, e
);
}
} }
} }
} }
} else { } else {
// Apply lifecycle actions // Apply lifecycle actions
if let Some(lifecycle_config) = &lifecycle_config { if let Some(lifecycle_config) = &lifecycle_config
if disk.is_local() { && disk.is_local()
{
let vcfg = BucketVersioningSys::get(bucket).await.ok(); let vcfg = BucketVersioningSys::get(bucket).await.ok();
let mut scanner_item = ScannerItem { let mut scanner_item = ScannerItem {
@@ -2053,7 +2033,6 @@ impl Scanner {
return Err(Error::other(ERR_IGNORE_FILE_CONTRIB).into()); return Err(Error::other(ERR_IGNORE_FILE_CONTRIB).into());
}*/ }*/
} }
}
// Store object metadata for later analysis // Store object metadata for later analysis
object_metadata.insert(entry.name.clone(), file_meta.clone()); object_metadata.insert(entry.name.clone(), file_meta.clone());
@@ -2064,15 +2043,11 @@ impl Scanner {
// object metadata parse failed, submit metadata heal task // object metadata parse failed, submit metadata heal task
let enable_healing = self.config.read().await.enable_healing; let enable_healing = self.config.read().await.enable_healing;
if enable_healing { if enable_healing && let Some(heal_manager) = &self.heal_manager {
if let Some(heal_manager) = &self.heal_manager {
let req = HealRequest::metadata(bucket.to_string(), entry.name.clone()); let req = HealRequest::metadata(bucket.to_string(), entry.name.clone());
match heal_manager.submit_heal_request(req).await { match heal_manager.submit_heal_request(req).await {
Ok(task_id) => { Ok(task_id) => {
warn!( warn!("object metadata parse failed, submit heal task: {} {} / {}", task_id, bucket, entry.name);
"object metadata parse failed, submit heal task: {} {} / {}",
task_id, bucket, entry.name
);
} }
Err(e) => { Err(e) => {
error!( error!(
@@ -2085,7 +2060,6 @@ impl Scanner {
} }
} }
} }
}
// Update metrics // Update metrics
self.metrics.increment_objects_scanned(objects_scanned); self.metrics.increment_objects_scanned(objects_scanned);
@@ -2190,19 +2164,16 @@ impl Scanner {
// the delete marker, but we keep it conservative here. // the delete marker, but we keep it conservative here.
let mut has_latest_delete_marker = false; let mut has_latest_delete_marker = false;
for &disk_idx in locations { for &disk_idx in locations {
if let Some(bucket_map) = all_disk_objects.get(disk_idx) { if let Some(bucket_map) = all_disk_objects.get(disk_idx)
if let Some(file_map) = bucket_map.get(bucket) { && let Some(file_map) = bucket_map.get(bucket)
if let Some(fm) = file_map.get(object_name) { && let Some(fm) = file_map.get(object_name)
if let Some(first_ver) = fm.versions.first() { && let Some(first_ver) = fm.versions.first()
if first_ver.header.version_type == VersionType::Delete { && first_ver.header.version_type == VersionType::Delete
{
has_latest_delete_marker = true; has_latest_delete_marker = true;
break; break;
} }
} }
}
}
}
}
if has_latest_delete_marker { if has_latest_delete_marker {
debug!( debug!(
"Object {}/{} is a delete marker on some disk(s), skipping heal for missing parts", "Object {}/{} is a delete marker on some disk(s), skipping heal for missing parts",
@@ -2248,8 +2219,7 @@ impl Scanner {
// submit heal task // submit heal task
let enable_healing = self.config.read().await.enable_healing; let enable_healing = self.config.read().await.enable_healing;
if enable_healing { if enable_healing && let Some(heal_manager) = &self.heal_manager {
if let Some(heal_manager) = &self.heal_manager {
use crate::heal::{HealPriority, HealRequest}; use crate::heal::{HealPriority, HealRequest};
let req = HealRequest::new( let req = HealRequest::new(
crate::heal::HealType::Object { crate::heal::HealType::Object {
@@ -2273,18 +2243,17 @@ impl Scanner {
} }
} }
} }
}
// Step 3: Deep scan EC verification // Step 3: Deep scan EC verification
let config = self.config.read().await; let config = self.config.read().await;
if config.scan_mode == ScanMode::Deep { if config.scan_mode == ScanMode::Deep
if let Err(e) = self.verify_object_integrity(bucket, object_name).await { && let Err(e) = self.verify_object_integrity(bucket, object_name).await
{
objects_with_ec_issues += 1; objects_with_ec_issues += 1;
warn!("Object integrity verification failed for object {}/{}: {}", bucket, object_name, e); warn!("Object integrity verification failed for object {}/{}: {}", bucket, object_name, e);
} }
} }
} }
}
info!( info!(
"Analysis complete: {} objects need healing, {} objects have EC issues", "Analysis complete: {} objects need healing, {} objects have EC issues",
@@ -2293,11 +2262,11 @@ impl Scanner {
// Step 4: Collect data usage statistics if enabled // Step 4: Collect data usage statistics if enabled
let config = self.config.read().await; let config = self.config.read().await;
if config.enable_data_usage_stats { if config.enable_data_usage_stats
if let Err(e) = self.collect_data_usage_statistics(all_disk_objects).await { && let Err(e) = self.collect_data_usage_statistics(all_disk_objects).await
{
error!("Failed to collect data usage statistics: {}", e); error!("Failed to collect data usage statistics: {}", e);
} }
}
drop(config); drop(config);
Ok(()) Ok(())
@@ -2526,23 +2495,21 @@ impl Scanner {
info!("Starting legacy scan loop for backward compatibility"); info!("Starting legacy scan loop for backward compatibility");
loop { loop {
if let Some(token) = get_ahm_services_cancel_token() { if let Some(token) = get_ahm_services_cancel_token()
if token.is_cancelled() { && token.is_cancelled()
{
info!("Cancellation requested, exiting legacy scan loop"); info!("Cancellation requested, exiting legacy scan loop");
break; break;
} }
}
let (enable_data_usage_stats, scan_interval) = { let (enable_data_usage_stats, scan_interval) = {
let config = self.config.read().await; let config = self.config.read().await;
(config.enable_data_usage_stats, config.scan_interval) (config.enable_data_usage_stats, config.scan_interval)
}; };
if enable_data_usage_stats { if enable_data_usage_stats && let Err(e) = self.collect_and_persist_data_usage().await {
if let Err(e) = self.collect_and_persist_data_usage().await {
warn!("Background data usage collection failed: {}", e); warn!("Background data usage collection failed: {}", e);
} }
}
// Update local stats in aggregator after latest scan // Update local stats in aggregator after latest scan
let local_stats = self.node_scanner.get_stats_summary().await; let local_stats = self.node_scanner.get_stats_summary().await;
@@ -2656,11 +2623,11 @@ mod tests {
// create temp dir as 4 disks // create temp dir as 4 disks
let test_base_dir = test_dir.unwrap_or("/tmp/rustfs_ahm_test"); let test_base_dir = test_dir.unwrap_or("/tmp/rustfs_ahm_test");
let temp_dir = std::path::PathBuf::from(test_base_dir); let temp_dir = std::path::PathBuf::from(test_base_dir);
if temp_dir.exists() { if temp_dir.exists()
if let Err(e) = fs::remove_dir_all(&temp_dir) { && let Err(e) = fs::remove_dir_all(&temp_dir)
{
panic!("Failed to remove test directory: {e}"); panic!("Failed to remove test directory: {e}");
} }
}
if let Err(e) = fs::create_dir_all(&temp_dir) { if let Err(e) = fs::create_dir_all(&temp_dir) {
panic!("Failed to create test directory: {e}"); panic!("Failed to create test directory: {e}");
} }
+3 -3
View File
@@ -305,13 +305,13 @@ fn compute_object_usage(bucket: &str, object: &str, file_meta: &FileMeta) -> Res
has_live_object = true; has_live_object = true;
versions_count = versions_count.saturating_add(1); versions_count = versions_count.saturating_add(1);
if latest_file_info.is_none() { if latest_file_info.is_none()
if let Ok(info) = file_meta.into_fileinfo(bucket, object, "", false, false) { && let Ok(info) = file_meta.into_fileinfo(bucket, object, "", false, false)
{
latest_file_info = Some(info); latest_file_info = Some(info);
} }
} }
} }
}
VersionType::Delete => { VersionType::Delete => {
delete_markers_count = delete_markers_count.saturating_add(1); delete_markers_count = delete_markers_count.saturating_add(1);
versions_count = versions_count.saturating_add(1); versions_count = versions_count.saturating_add(1);
+3 -3
View File
@@ -112,11 +112,11 @@ impl LocalStatsManager {
/// create new local stats manager /// create new local stats manager
pub fn new(node_id: &str, data_dir: &Path) -> Self { pub fn new(node_id: &str, data_dir: &Path) -> Self {
// ensure data directory exists // ensure data directory exists
if !data_dir.exists() { if !data_dir.exists()
if let Err(e) = std::fs::create_dir_all(data_dir) { && let Err(e) = std::fs::create_dir_all(data_dir)
{
error!("create stats data directory failed {:?}: {}", data_dir, e); error!("create stats data directory failed {:?}: {}", data_dir, e);
} }
}
let stats_file = data_dir.join(format!("scanner_stats_{node_id}.json")); let stats_file = data_dir.join(format!("scanner_stats_{node_id}.json"));
let backup_file = data_dir.join(format!("scanner_stats_{node_id}.backup")); let backup_file = data_dir.join(format!("scanner_stats_{node_id}.backup"));
+3 -3
View File
@@ -436,11 +436,11 @@ impl NodeScanner {
/// create a new node scanner /// create a new node scanner
pub fn new(node_id: String, config: NodeScannerConfig) -> Self { pub fn new(node_id: String, config: NodeScannerConfig) -> Self {
// Ensure data directory exists // Ensure data directory exists
if !config.data_dir.exists() { if !config.data_dir.exists()
if let Err(e) = std::fs::create_dir_all(&config.data_dir) { && let Err(e) = std::fs::create_dir_all(&config.data_dir)
{
error!("create data directory failed {:?}: {}", config.data_dir, e); error!("create data directory failed {:?}: {}", config.data_dir, e);
} }
}
let stats_manager = Arc::new(LocalStatsManager::new(&node_id, &config.data_dir)); let stats_manager = Arc::new(LocalStatsManager::new(&node_id, &config.data_dir));
+3 -3
View File
@@ -327,8 +327,9 @@ impl DecentralizedStatsAggregator {
); );
// Check cache validity if timestamp is not initial value (UNIX_EPOCH) // Check cache validity if timestamp is not initial value (UNIX_EPOCH)
if cache_timestamp != SystemTime::UNIX_EPOCH { if cache_timestamp != SystemTime::UNIX_EPOCH
if let Ok(elapsed) = now.duration_since(cache_timestamp) { && let Ok(elapsed) = now.duration_since(cache_timestamp)
{
if elapsed < cache_ttl { if elapsed < cache_ttl {
if let Some(cached) = self.cached_stats.read().await.as_ref() { if let Some(cached) = self.cached_stats.read().await.as_ref() {
debug!("Returning cached aggregated stats, remaining TTL: {:?}", cache_ttl - elapsed); debug!("Returning cached aggregated stats, remaining TTL: {:?}", cache_ttl - elapsed);
@@ -338,7 +339,6 @@ impl DecentralizedStatsAggregator {
debug!("Cache expired: elapsed={:?} >= ttl={:?}", elapsed, cache_ttl); debug!("Cache expired: elapsed={:?} >= ttl={:?}", elapsed, cache_ttl);
} }
} }
}
// cache expired, re-aggregate // cache expired, re-aggregate
info!("cache expired, start re-aggregating stats data"); info!("cache expired, start re-aggregating stats data");
+3 -3
View File
@@ -421,8 +421,9 @@ mod serial_tests {
} }
}; };
if let Some(lmdb_env) = GLOBAL_LMDB_ENV.get() { if let Some(lmdb_env) = GLOBAL_LMDB_ENV.get()
if let Some(lmdb) = GLOBAL_LMDB_DB.get() { && let Some(lmdb) = GLOBAL_LMDB_DB.get()
{
let mut wtxn = lmdb_env.write_txn().unwrap(); let mut wtxn = lmdb_env.write_txn().unwrap();
/*if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await { /*if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await {
@@ -501,7 +502,6 @@ mod serial_tests {
//drop(iter); //drop(iter);
wtxn.commit().unwrap(); wtxn.commit().unwrap();
} }
}
println!("Lifecycle cache test completed"); println!("Lifecycle cache test completed");
} }
@@ -415,8 +415,8 @@ mod serial_tests {
.await; .await;
println!("Pending expiry tasks: {pending}"); println!("Pending expiry tasks: {pending}");
if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await { if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await
if let Ok(object_info) = ecstore && let Ok(object_info) = ecstore
.get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default()) .get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
.await .await
{ {
@@ -438,7 +438,6 @@ mod serial_tests {
expired = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(2)).await; expired = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(2)).await;
} }
}
if !expired { if !expired {
println!("❌ Object was not deleted by lifecycle processing"); println!("❌ Object was not deleted by lifecycle processing");
@@ -550,8 +549,8 @@ mod serial_tests {
.await; .await;
println!("Pending expiry tasks: {pending}"); println!("Pending expiry tasks: {pending}");
if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await { if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await
if let Ok(obj_info) = ecstore && let Ok(obj_info) = ecstore
.get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default()) .get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
.await .await
{ {
@@ -577,7 +576,6 @@ mod serial_tests {
); );
} }
} }
}
if !deleted { if !deleted {
println!("❌ Object was not deleted by lifecycle processing"); println!("❌ Object was not deleted by lifecycle processing");
+3 -3
View File
@@ -204,12 +204,12 @@ impl TargetFactory for MQTTTargetFactory {
if !std::path::Path::new(&queue_dir).is_absolute() { if !std::path::Path::new(&queue_dir).is_absolute() {
return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string())); return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string()));
} }
if let Some(qos_str) = config.lookup(MQTT_QOS) { if let Some(qos_str) = config.lookup(MQTT_QOS)
if qos_str == "0" { && qos_str == "0"
{
warn!("Using queue_dir with QoS 0 may result in event loss"); warn!("Using queue_dir with QoS 0 may result in event loss");
} }
} }
}
Ok(()) Ok(())
} }
+7 -8
View File
@@ -138,14 +138,13 @@ impl AuditRegistry {
format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}") format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}")
.to_uppercase(); .to_uppercase();
for (key, value) in &all_env { for (key, value) in &all_env {
if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false) { if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false)
if let Some(id) = key.strip_prefix(&enable_prefix) { && let Some(id) = key.strip_prefix(&enable_prefix)
if !id.is_empty() { && !id.is_empty()
{
instance_ids_from_env.insert(id.to_lowercase()); instance_ids_from_env.insert(id.to_lowercase());
} }
} }
}
}
// 3.2. Parse all relevant environment variable configurations // 3.2. Parse all relevant environment variable configurations
// 3.2.1. Build environment variable prefixes such as 'RUSTFS_AUDIT_WEBHOOK_' // 3.2.1. Build environment variable prefixes such as 'RUSTFS_AUDIT_WEBHOOK_'
@@ -292,11 +291,11 @@ impl AuditRegistry {
for section in sections { for section in sections {
let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new(); let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
// Add default item // Add default item
if let Some(default_kvs) = section_defaults.get(&section) { if let Some(default_kvs) = section_defaults.get(&section)
if !default_kvs.is_empty() { && !default_kvs.is_empty()
{
section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone()); section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
} }
}
// Add successful instance item // Add successful instance item
if let Some(instances) = successes_by_section.get(&section) { if let Some(instances) = successes_by_section.get(&section) {
+3 -3
View File
@@ -573,11 +573,11 @@ impl AuditSystem {
} }
// Remove existing target if present // Remove existing target if present
if let Some(old_target) = registry.remove_target(&target_id) { if let Some(old_target) = registry.remove_target(&target_id)
if let Err(e) = old_target.close().await { && let Err(e) = old_target.close().await
{
error!(target_id = %target_id, error = %e, "Failed to close old target during upsert"); error!(target_id = %target_id, error = %e, "Failed to close old target during upsert");
} }
}
registry.add_target(target_id.clone(), target); registry.add_target(target_id.clone(), target);
info!(target_id = %target_id, "Target upserted"); info!(target_id = %target_id, "Target upserted");
+7 -8
View File
@@ -605,14 +605,13 @@ impl DataUsageCache {
pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> { pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> {
let want = hash.key(); let want = hash.key();
if let Some(last_index) = want.rfind('/') { if let Some(last_index) = want.rfind('/')
if let Some(v) = self.find(&want[0..last_index]) { && let Some(v) = self.find(&want[0..last_index])
if v.children.contains(&want) { && v.children.contains(&want)
{
let found = hash_path(&want[0..last_index]); let found = hash_path(&want[0..last_index]);
return Some(found); return Some(found);
} }
}
}
for (k, v) in self.cache.iter() { for (k, v) in self.cache.iter() {
if v.children.contains(&want) { if v.children.contains(&want) {
@@ -1150,13 +1149,13 @@ impl DataUsageInfo {
self.buckets_count = self.buckets_usage.len() as u64; self.buckets_count = self.buckets_usage.len() as u64;
// Update last update time // Update last update time
if let Some(other_update) = other.last_update { if let Some(other_update) = other.last_update
if self.last_update.is_none() || other_update > self.last_update.unwrap() { && (self.last_update.is_none() || other_update > self.last_update.unwrap())
{
self.last_update = Some(other_update); self.last_update = Some(other_update);
} }
} }
} }
}
impl BucketUsageInfo { impl BucketUsageInfo {
/// Create a new BucketUsageInfo /// Create a new BucketUsageInfo
+8 -10
View File
@@ -403,12 +403,12 @@ fn lc_get_prefix(rule: &LifecycleRule) -> String {
} else if let Some(filter) = &rule.filter { } else if let Some(filter) = &rule.filter {
if let Some(p) = &filter.prefix { if let Some(p) = &filter.prefix {
return p.to_string(); return p.to_string();
} else if let Some(and) = &filter.and { } else if let Some(and) = &filter.and
if let Some(p) = &and.prefix { && let Some(p) = &and.prefix
{
return p.to_string(); return p.to_string();
} }
} }
}
"".into() "".into()
} }
@@ -475,10 +475,11 @@ pub fn rep_has_active_rules(config: &ReplicationConfiguration, prefix: &str, rec
{ {
continue; continue;
} }
if !prefix.is_empty() { if !prefix.is_empty()
if let Some(filter) = &rule.filter { && let Some(filter) = &rule.filter
if let Some(r_prefix) = &filter.prefix { && let Some(r_prefix) = &filter.prefix
if !r_prefix.is_empty() { && !r_prefix.is_empty()
{
// incoming prefix must be in rule prefix // incoming prefix must be in rule prefix
if !recursive && !prefix.starts_with(r_prefix) { if !recursive && !prefix.starts_with(r_prefix) {
continue; continue;
@@ -489,9 +490,6 @@ pub fn rep_has_active_rules(config: &ReplicationConfiguration, prefix: &str, rec
continue; continue;
} }
} }
}
}
}
return true; return true;
} }
false false
+6 -6
View File
@@ -466,23 +466,23 @@ impl Metrics {
// Lifetime operations // Lifetime operations
for i in 0..Metric::Last as usize { for i in 0..Metric::Last as usize {
let count = self.operations[i].load(Ordering::Relaxed); let count = self.operations[i].load(Ordering::Relaxed);
if count > 0 { if count > 0
if let Some(metric) = Metric::from_index(i) { && let Some(metric) = Metric::from_index(i)
{
metrics.life_time_ops.insert(metric.as_str().to_string(), count); metrics.life_time_ops.insert(metric.as_str().to_string(), count);
} }
} }
}
// Last minute statistics for realtime metrics // Last minute statistics for realtime metrics
for i in 0..Metric::LastRealtime as usize { for i in 0..Metric::LastRealtime as usize {
let last_min = self.latency[i].total().await; let last_min = self.latency[i].total().await;
if last_min.n > 0 { if last_min.n > 0
if let Some(_metric) = Metric::from_index(i) { && let Some(_metric) = Metric::from_index(i)
{
// Convert to madmin TimedAction format if needed // Convert to madmin TimedAction format if needed
// This would require implementing the conversion // This would require implementing the conversion
} }
} }
}
metrics metrics
} }
+3 -3
View File
@@ -178,12 +178,12 @@ impl RustFSTestEnvironment {
info!("Cleaning up any existing RustFS processes"); info!("Cleaning up any existing RustFS processes");
let output = Command::new("pkill").args(["-f", "rustfs"]).output(); let output = Command::new("pkill").args(["-f", "rustfs"]).output();
if let Ok(output) = output { if let Ok(output) = output
if output.status.success() { && output.status.success()
{
info!("Killed existing RustFS processes"); info!("Killed existing RustFS processes");
sleep(Duration::from_millis(1000)).await; sleep(Duration::from_millis(1000)).await;
} }
}
Ok(()) Ok(())
} }
+3 -3
View File
@@ -406,13 +406,13 @@ impl VaultTestEnvironment {
let port_check = TcpStream::connect(VAULT_ADDRESS).await.is_ok(); let port_check = TcpStream::connect(VAULT_ADDRESS).await.is_ok();
if port_check { if port_check {
// Additional check by making a health request // Additional check by making a health request
if let Ok(response) = reqwest::get(&format!("{VAULT_URL}/v1/sys/health")).await { if let Ok(response) = reqwest::get(&format!("{VAULT_URL}/v1/sys/health")).await
if response.status().is_success() { && response.status().is_success()
{
info!("Vault server is ready after {} seconds", i); info!("Vault server is ready after {} seconds", i);
return Ok(()); return Ok(());
} }
} }
}
if i == 29 { if i == 29 {
return Err("Vault server failed to become ready".into()); return Err("Vault server failed to become ready".into());
@@ -498,8 +498,9 @@ impl BucketTargetSys {
bucket: bucket.to_string(), bucket: bucket.to_string(),
})?; })?;
if arn.arn_type == BucketTargetType::ReplicationService { if arn.arn_type == BucketTargetType::ReplicationService
if let Ok((config, _)) = get_replication_config(bucket).await { && let Ok((config, _)) = get_replication_config(bucket).await
{
for rule in config.filter_target_arns(&ObjectOpts { for rule in config.filter_target_arns(&ObjectOpts {
op_type: ReplicationType::All, op_type: ReplicationType::All,
..Default::default() ..Default::default()
@@ -514,7 +515,6 @@ impl BucketTargetSys {
} }
} }
} }
}
{ {
let mut targets_map = self.targets_map.write().await; let mut targets_map = self.targets_map.write().await;
@@ -691,8 +691,9 @@ impl BucketTargetSys {
} }
// Add new targets // Add new targets
if let Some(new_targets) = targets { if let Some(new_targets) = targets
if !new_targets.is_empty() { && !new_targets.is_empty()
{
for target in &new_targets.targets { for target in &new_targets.targets {
if let Ok(client) = self.get_remote_target_client_internal(target).await { if let Ok(client) = self.get_remote_target_client_internal(target).await {
arn_remotes_map.insert( arn_remotes_map.insert(
@@ -708,7 +709,6 @@ impl BucketTargetSys {
targets_map.insert(bucket.to_string(), new_targets.targets.clone()); targets_map.insert(bucket.to_string(), new_targets.targets.clone());
} }
} }
}
pub async fn set(&self, bucket: &str, meta: &BucketMetadata) { pub async fn set(&self, bucket: &str, meta: &BucketMetadata) {
let Some(config) = &meta.bucket_target_config else { let Some(config) = &meta.bucket_target_config else {
@@ -31,11 +31,11 @@ impl BucketObjectLockSys {
} }
pub async fn get(bucket: &str) -> Option<DefaultRetention> { pub async fn get(bucket: &str) -> Option<DefaultRetention> {
if let Ok(object_lock_config) = get_object_lock_config(bucket).await { if let Ok(object_lock_config) = get_object_lock_config(bucket).await
if let Some(object_lock_rule) = object_lock_config.0.rule { && let Some(object_lock_rule) = object_lock_config.0.rule
{
return object_lock_rule.default_retention; return object_lock_rule.default_retention;
} }
}
None None
} }
} }
+10 -12
View File
@@ -55,13 +55,13 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
if !has_arn { if !has_arn {
has_arn = true; has_arn = true;
} }
if let Some(status) = &rule.existing_object_replication { if let Some(status) = &rule.existing_object_replication
if status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED) { && status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED)
{
return (true, true); return (true, true);
} }
} }
} }
}
(has_arn, false) (has_arn, false)
} }
@@ -86,13 +86,12 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
continue; continue;
} }
if let Some(status) = &rule.existing_object_replication { if let Some(status) = &rule.existing_object_replication
if obj.existing_object && obj.existing_object
&& status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED) && status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
{ {
continue; continue;
} }
}
if !obj.name.starts_with(rule.prefix()) { if !obj.name.starts_with(rule.prefix()) {
continue; continue;
@@ -145,13 +144,12 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
continue; continue;
} }
if let Some(status) = &rule.existing_object_replication { if let Some(status) = &rule.existing_object_replication
if obj.existing_object && obj.existing_object
&& status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED) && status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
{ {
return false; return false;
} }
}
if obj.op_type == ReplicationType::Delete { if obj.op_type == ReplicationType::Delete {
if obj.version_id.is_some() { if obj.version_id.is_some() {
@@ -186,8 +184,9 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
continue; continue;
} }
if let Some(filter) = &rule.filter { if let Some(filter) = &rule.filter
if let Some(filter_prefix) = &filter.prefix { && let Some(filter_prefix) = &filter.prefix
{
if !prefix.is_empty() && !filter_prefix.is_empty() { if !prefix.is_empty() && !filter_prefix.is_empty() {
// The provided prefix must fall within the rule prefix // The provided prefix must fall within the rule prefix
if !recursive && !prefix.starts_with(filter_prefix) { if !recursive && !prefix.starts_with(filter_prefix) {
@@ -200,7 +199,6 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
continue; continue;
} }
} }
}
return true; return true;
} }
false false
@@ -512,8 +512,9 @@ impl<S: StorageAPI> ReplicationPool<S> {
if !lrg_workers.is_empty() { if !lrg_workers.is_empty() {
let index = (hash as usize) % lrg_workers.len(); let index = (hash as usize) % lrg_workers.len();
if let Some(worker) = lrg_workers.get(index) { if let Some(worker) = lrg_workers.get(index)
if worker.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_err() { && worker.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_err()
{
// Queue to MRF if worker is busy // Queue to MRF if worker is busy
let _ = self.mrf_save_tx.try_send(ri.to_mrf_entry()); let _ = self.mrf_save_tx.try_send(ri.to_mrf_entry());
@@ -528,7 +529,6 @@ impl<S: StorageAPI> ReplicationPool<S> {
} }
} }
} }
}
return; return;
} }
@@ -539,8 +539,9 @@ impl<S: StorageAPI> ReplicationPool<S> {
_ => self.get_worker_ch(&ri.bucket, &ri.name, ri.size).await, _ => self.get_worker_ch(&ri.bucket, &ri.name, ri.size).await,
}; };
if let Some(channel) = ch { if let Some(channel) = ch
if channel.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_err() { && channel.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_err()
{
// Queue to MRF if all workers are busy // Queue to MRF if all workers are busy
let _ = self.mrf_save_tx.try_send(ri.to_mrf_entry()); let _ = self.mrf_save_tx.try_send(ri.to_mrf_entry());
@@ -554,9 +555,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
info!("Warning: Unable to keep up with incoming traffic"); info!("Warning: Unable to keep up with incoming traffic");
} }
ReplicationPriority::Slow => { ReplicationPriority::Slow => {
info!( info!("Warning: Unable to keep up with incoming traffic - recommend increasing replication priority to auto");
"Warning: Unable to keep up with incoming traffic - recommend increasing replication priority to auto"
);
} }
ReplicationPriority::Auto => { ReplicationPriority::Auto => {
let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT); let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT);
@@ -584,7 +583,6 @@ impl<S: StorageAPI> ReplicationPool<S> {
} }
} }
} }
}
/// Queues a replica delete task /// Queues a replica delete task
pub async fn queue_replica_delete_task(&self, doi: DeletedObjectReplicationInfo) { pub async fn queue_replica_delete_task(&self, doi: DeletedObjectReplicationInfo) {
@@ -593,8 +591,9 @@ impl<S: StorageAPI> ReplicationPool<S> {
_ => self.get_worker_ch(&doi.bucket, &doi.delete_object.object_name, 0).await, _ => self.get_worker_ch(&doi.bucket, &doi.delete_object.object_name, 0).await,
}; };
if let Some(channel) = ch { if let Some(channel) = ch
if channel.try_send(ReplicationOperation::Delete(Box::new(doi.clone()))).is_err() { && channel.try_send(ReplicationOperation::Delete(Box::new(doi.clone()))).is_err()
{
let _ = self.mrf_save_tx.try_send(doi.to_mrf_entry()); let _ = self.mrf_save_tx.try_send(doi.to_mrf_entry());
let priority = self.priority.read().await.clone(); let priority = self.priority.read().await.clone();
@@ -605,9 +604,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
info!("Warning: Unable to keep up with incoming deletes"); info!("Warning: Unable to keep up with incoming deletes");
} }
ReplicationPriority::Slow => { ReplicationPriority::Slow => {
info!( info!("Warning: Unable to keep up with incoming deletes - recommend increasing replication priority to auto");
"Warning: Unable to keep up with incoming deletes - recommend increasing replication priority to auto"
);
} }
ReplicationPriority::Auto => { ReplicationPriority::Auto => {
let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT); let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT);
@@ -622,7 +619,6 @@ impl<S: StorageAPI> ReplicationPool<S> {
} }
} }
} }
}
/// Queues an MRF save operation /// Queues an MRF save operation
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) { async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
@@ -242,11 +242,10 @@ impl ReplicationResyncer {
if let Some(last_update) = status.last_update { if let Some(last_update) = status.last_update
if last_update > *last_update_times.get(bucket).unwrap_or(&OffsetDateTime::UNIX_EPOCH) { && last_update > *last_update_times.get(bucket).unwrap_or(&OffsetDateTime::UNIX_EPOCH) {
update = true; update = true;
} }
}
if update { if update {
if let Err(err) = save_resync_status(bucket, status, api.clone()).await { if let Err(err) = save_resync_status(bucket, status, api.clone()).await {
@@ -345,14 +344,13 @@ impl ReplicationResyncer {
return; return;
}; };
if !heal { if !heal
if let Err(e) = self && let Err(e) = self
.mark_status(ResyncStatusType::ResyncStarted, opts.clone(), storage.clone()) .mark_status(ResyncStatusType::ResyncStarted, opts.clone(), storage.clone())
.await .await
{ {
error!("Failed to mark resync status: {}", e); error!("Failed to mark resync status: {}", e);
} }
}
let (tx, mut rx) = tokio::sync::mpsc::channel(100); let (tx, mut rx) = tokio::sync::mpsc::channel(100);
@@ -1463,21 +1461,18 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
Some(version_id.to_string()) Some(version_id.to_string())
}; };
if dobj.delete_object.delete_marker_version_id.is_some() { if dobj.delete_object.delete_marker_version_id.is_some()
if let Err(e) = tgt_client && let Err(e) = tgt_client
.head_object(&tgt_client.bucket, &dobj.delete_object.object_name, version_id.clone()) .head_object(&tgt_client.bucket, &dobj.delete_object.object_name, version_id.clone())
.await .await
&& let SdkError::ServiceError(service_err) = &e
&& !service_err.err().is_not_found()
{ {
if let SdkError::ServiceError(service_err) = &e {
if !service_err.err().is_not_found() {
rinfo.replication_status = ReplicationStatusType::Failed; rinfo.replication_status = ReplicationStatusType::Failed;
rinfo.error = Some(e.to_string()); rinfo.error = Some(e.to_string());
return rinfo; return rinfo;
}
}
}; };
}
match tgt_client match tgt_client
.remove_object( .remove_object(
@@ -49,8 +49,9 @@ impl ExponentialMovingAverage {
pub fn update_exponential_moving_average(&self, now: SystemTime) { pub fn update_exponential_moving_average(&self, now: SystemTime) {
if let Ok(mut last_update_guard) = self.last_update.try_lock() { if let Ok(mut last_update_guard) = self.last_update.try_lock() {
let last_update = *last_update_guard; let last_update = *last_update_guard;
if let Ok(duration) = now.duration_since(last_update) { if let Ok(duration) = now.duration_since(last_update)
if duration.as_secs() > 0 { && duration.as_secs() > 0
{
let decay = (-duration.as_secs_f64() / 60.0).exp(); // 1 minute decay let decay = (-duration.as_secs_f64() / 60.0).exp(); // 1 minute decay
let current_value = f64::from_bits(self.value.load(AtomicOrdering::Relaxed)); let current_value = f64::from_bits(self.value.load(AtomicOrdering::Relaxed));
self.value.store((current_value * decay).to_bits(), AtomicOrdering::Relaxed); self.value.store((current_value * decay).to_bits(), AtomicOrdering::Relaxed);
@@ -58,7 +59,6 @@ impl ExponentialMovingAverage {
} }
} }
} }
}
pub fn merge(&self, other: &ExponentialMovingAverage) -> Self { pub fn merge(&self, other: &ExponentialMovingAverage) -> Self {
let now = SystemTime::now(); let now = SystemTime::now();
@@ -757,11 +757,11 @@ impl ReplicationStats {
/// Check if bucket replication statistics have usage /// Check if bucket replication statistics have usage
pub fn has_replication_usage(&self, bucket: &str) -> bool { pub fn has_replication_usage(&self, bucket: &str) -> bool {
if let Ok(cache) = self.cache.try_read() { if let Ok(cache) = self.cache.try_read()
if let Some(stats) = cache.get(bucket) { && let Some(stats) = cache.get(bucket)
{
return stats.has_replication_usage(); return stats.has_replication_usage();
} }
}
false false
} }
+8 -6
View File
@@ -37,11 +37,12 @@ impl VersioningApi for VersioningConfiguration {
return true; return true;
} }
if let Some(exclude_folders) = self.exclude_folders { if let Some(exclude_folders) = self.exclude_folders
if exclude_folders && prefix.ends_with('/') { && exclude_folders
&& prefix.ends_with('/')
{
return false; return false;
} }
}
if let Some(ref excluded_prefixes) = self.excluded_prefixes { if let Some(ref excluded_prefixes) = self.excluded_prefixes {
for p in excluded_prefixes.iter() { for p in excluded_prefixes.iter() {
@@ -67,11 +68,12 @@ impl VersioningApi for VersioningConfiguration {
return false; return false;
} }
if let Some(exclude_folders) = self.exclude_folders { if let Some(exclude_folders) = self.exclude_folders
if exclude_folders && prefix.ends_with('/') { && exclude_folders
&& prefix.ends_with('/')
{
return true; return true;
} }
}
if let Some(ref excluded_prefixes) = self.excluded_prefixes { if let Some(ref excluded_prefixes) = self.excluded_prefixes {
for p in excluded_prefixes.iter() { for p in excluded_prefixes.iter() {
@@ -308,13 +308,12 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
// Break if all at EOF or error. // Break if all at EOF or error.
if at_eof + has_err == readers.len() { if at_eof + has_err == readers.len() {
if has_err > 0 { if has_err > 0
if let Some(finished_fn) = opts.finished.as_ref() { && let Some(finished_fn) = opts.finished.as_ref()
if has_err > 0 { && has_err > 0
{
finished_fn(&errs).await; finished_fn(&errs).await;
} }
}
}
// error!("list_path_raw: at_eof + has_err == readers.len() break {:?}", &errs); // error!("list_path_raw: at_eof + has_err == readers.len() break {:?}", &errs);
break; break;
+1 -1
View File
@@ -161,7 +161,7 @@ impl TransitionClient {
async fn private_new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> { async fn private_new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
let endpoint_url = get_endpoint_url(endpoint, opts.secure)?; let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
let _ = rustls::crypto::ring::default_provider().install_default(); let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
let scheme = endpoint_url.scheme(); let scheme = endpoint_url.scheme();
let client; let client;
let tls = if let Some(store) = load_root_store_from_tls_path() { let tls = if let Some(store) = load_root_store_from_tls_path() {
+4 -3
View File
@@ -211,12 +211,13 @@ async fn apply_dynamic_config_for_sub_sys<S: StorageAPI>(cfg: &mut Config, api:
for (i, count) in set_drive_counts.iter().enumerate() { for (i, count) in set_drive_counts.iter().enumerate() {
match storageclass::lookup_config(&kvs, *count) { match storageclass::lookup_config(&kvs, *count) {
Ok(res) => { Ok(res) => {
if i == 0 && GLOBAL_STORAGE_CLASS.get().is_none() { if i == 0
if let Err(r) = GLOBAL_STORAGE_CLASS.set(res) { && GLOBAL_STORAGE_CLASS.get().is_none()
&& let Err(r) = GLOBAL_STORAGE_CLASS.set(res)
{
error!("GLOBAL_STORAGE_CLASS.set failed {:?}", r); error!("GLOBAL_STORAGE_CLASS.set failed {:?}", r);
} }
} }
}
Err(err) => { Err(err) => {
error!("init storage class err:{:?}", &err); error!("init storage class err:{:?}", &err);
break; break;
+3 -3
View File
@@ -180,14 +180,14 @@ impl Config {
let mut default = HashMap::new(); let mut default = HashMap::new();
default.insert(DEFAULT_DELIMITER.to_owned(), v.clone()); default.insert(DEFAULT_DELIMITER.to_owned(), v.clone());
self.0.insert(k.clone(), default); self.0.insert(k.clone(), default);
} else if !self.0[k].contains_key(DEFAULT_DELIMITER) { } else if !self.0[k].contains_key(DEFAULT_DELIMITER)
if let Some(m) = self.0.get_mut(k) { && let Some(m) = self.0.get_mut(k)
{
m.insert(DEFAULT_DELIMITER.to_owned(), v.clone()); m.insert(DEFAULT_DELIMITER.to_owned(), v.clone());
} }
} }
} }
} }
}
pub fn unmarshal(data: &[u8]) -> Result<Config> { pub fn unmarshal(data: &[u8]) -> Result<Config> {
let m: HashMap<String, HashMap<String, KVS>> = serde_json::from_slice(data)?; let m: HashMap<String, HashMap<String, KVS>> = serde_json::from_slice(data)?;
+15 -19
View File
@@ -65,19 +65,17 @@ lazy_static::lazy_static! {
/// Store data usage info to backend storage /// Store data usage info to backend storage
pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> { pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
// Prevent older data from overwriting newer persisted stats // Prevent older data from overwriting newer persisted stats
if let Ok(buf) = read_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH).await { if let Ok(buf) = read_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH).await
if let Ok(existing) = serde_json::from_slice::<DataUsageInfo>(&buf) { && let Ok(existing) = serde_json::from_slice::<DataUsageInfo>(&buf)
if let (Some(new_ts), Some(existing_ts)) = (data_usage_info.last_update, existing.last_update) { && let (Some(new_ts), Some(existing_ts)) = (data_usage_info.last_update, existing.last_update)
if new_ts <= existing_ts { && new_ts <= existing_ts
{
info!( info!(
"Skip persisting data usage: incoming last_update {:?} <= existing {:?}", "Skip persisting data usage: incoming last_update {:?} <= existing {:?}",
new_ts, existing_ts new_ts, existing_ts
); );
return Ok(()); return Ok(());
} }
}
}
}
let data = let data =
serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?; serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
@@ -149,13 +147,13 @@ pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsa
// Handle replication info // Handle replication info
for (bucket, bui) in &data_usage_info.buckets_usage { for (bucket, bui) in &data_usage_info.buckets_usage {
if bui.replicated_size_v1 > 0 if (bui.replicated_size_v1 > 0
|| bui.replication_failed_count_v1 > 0 || bui.replication_failed_count_v1 > 0
|| bui.replication_failed_size_v1 > 0 || bui.replication_failed_size_v1 > 0
|| bui.replication_pending_count_v1 > 0 || bui.replication_pending_count_v1 > 0)
&& let Ok((cfg, _)) = get_replication_config(bucket).await
&& !cfg.role.is_empty()
{ {
if let Ok((cfg, _)) = get_replication_config(bucket).await {
if !cfg.role.is_empty() {
data_usage_info.replication_info.insert( data_usage_info.replication_info.insert(
cfg.role.clone(), cfg.role.clone(),
BucketTargetUsageInfo { BucketTargetUsageInfo {
@@ -169,19 +167,17 @@ pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsa
); );
} }
} }
}
}
Ok(data_usage_info) Ok(data_usage_info)
} }
/// Aggregate usage information from local disk snapshots. /// Aggregate usage information from local disk snapshots.
fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapshot, latest_update: &mut Option<SystemTime>) { fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapshot, latest_update: &mut Option<SystemTime>) {
if let Some(update) = snapshot.last_update { if let Some(update) = snapshot.last_update
if latest_update.is_none_or(|current| update > current) { && latest_update.is_none_or(|current| update > current)
{
*latest_update = Some(update); *latest_update = Some(update);
} }
}
snapshot.recompute_totals(); snapshot.recompute_totals();
@@ -255,12 +251,12 @@ pub async fn aggregate_local_snapshots(store: Arc<ECStore>) -> Result<(Vec<DiskU
); );
// Best-effort cleanup so next scan can rebuild a fresh snapshot instead of repeatedly failing // Best-effort cleanup so next scan can rebuild a fresh snapshot instead of repeatedly failing
let snapshot_file = snapshot_path(root.as_path(), &disk_id); let snapshot_file = snapshot_path(root.as_path(), &disk_id);
if let Err(remove_err) = fs::remove_file(&snapshot_file).await { if let Err(remove_err) = fs::remove_file(&snapshot_file).await
if remove_err.kind() != std::io::ErrorKind::NotFound { && remove_err.kind() != std::io::ErrorKind::NotFound
{
warn!("Failed to remove corrupted snapshot {:?}: {}", snapshot_file, remove_err); warn!("Failed to remove corrupted snapshot {:?}: {}", snapshot_file, remove_err);
} }
} }
}
if let Ok(Some(mut snapshot)) = snapshot_result { if let Ok(Some(mut snapshot)) = snapshot_result {
status.last_update = snapshot.last_update; status.last_update = snapshot.last_update;
+86 -91
View File
@@ -288,17 +288,17 @@ impl LocalDisk {
let path = path_join(&[trash.clone(), name.into()]); let path = path_join(&[trash.clone(), name.into()]);
if file_type.is_dir() { if file_type.is_dir() {
if let Err(e) = tokio::fs::remove_dir_all(path).await { if let Err(e) = tokio::fs::remove_dir_all(path).await
if e.kind() != ErrorKind::NotFound { && e.kind() != ErrorKind::NotFound
{
return Err(e.into()); return Err(e.into());
} }
} } else if let Err(e) = tokio::fs::remove_file(path).await
} else if let Err(e) = tokio::fs::remove_file(path).await { && e.kind() != ErrorKind::NotFound
if e.kind() != ErrorKind::NotFound { {
return Err(e.into()); return Err(e.into());
} }
} }
}
Ok(()) Ok(())
} }
@@ -684,14 +684,12 @@ impl LocalDisk {
Err(err) => { Err(err) => {
if err == Error::FileNotFound if err == Error::FileNotFound
&& !skip_access_checks(volume_dir.as_ref().to_string_lossy().to_string().as_str()) && !skip_access_checks(volume_dir.as_ref().to_string_lossy().to_string().as_str())
&& let Err(e) = access(volume_dir.as_ref()).await
&& e.kind() == ErrorKind::NotFound
{ {
if let Err(e) = access(volume_dir.as_ref()).await {
if e.kind() == ErrorKind::NotFound {
// warn!("read_metadata_with_dmtime os err {:?}", &aerr); // warn!("read_metadata_with_dmtime os err {:?}", &aerr);
return Err(DiskError::VolumeNotFound); return Err(DiskError::VolumeNotFound);
} }
}
}
Err(err) Err(err)
} }
@@ -763,14 +761,14 @@ impl LocalDisk {
let mut f = match super::fs::open_file(file_path.as_ref(), O_RDONLY).await { let mut f = match super::fs::open_file(file_path.as_ref(), O_RDONLY).await {
Ok(f) => f, Ok(f) => f,
Err(e) => { Err(e) => {
if e.kind() == ErrorKind::NotFound && !skip_access_checks(volume) { if e.kind() == ErrorKind::NotFound
if let Err(er) = access(volume_dir.as_ref()).await { && !skip_access_checks(volume)
if er.kind() == ErrorKind::NotFound { && let Err(er) = access(volume_dir.as_ref()).await
&& er.kind() == ErrorKind::NotFound
{
warn!("read_all_data_with_dmtime os err {:?}", &er); warn!("read_all_data_with_dmtime os err {:?}", &er);
return Err(DiskError::VolumeNotFound); return Err(DiskError::VolumeNotFound);
} }
}
}
return Err(to_file_error(e).into()); return Err(to_file_error(e).into());
} }
@@ -828,10 +826,10 @@ impl LocalDisk {
let _ = fm.data.remove(vec![vid, dir]); let _ = fm.data.remove(vec![vid, dir]);
let dir_path = self.get_object_path(volume, format!("{path}/{dir}").as_str())?; let dir_path = self.get_object_path(volume, format!("{path}/{dir}").as_str())?;
if let Err(err) = self.move_to_trash(&dir_path, true, false).await { if let Err(err) = self.move_to_trash(&dir_path, true, false).await
if !(err == DiskError::FileNotFound || err == DiskError::VolumeNotFound) { && !(err == DiskError::FileNotFound || err == DiskError::VolumeNotFound)
{
return Err(err); return Err(err);
}
}; };
} }
} }
@@ -1051,12 +1049,12 @@ impl LocalDisk {
continue; continue;
} }
if let Some(forward) = &forward { if let Some(forward) = &forward
if &entry < forward { && &entry < forward
{
*item = "".to_owned(); *item = "".to_owned();
continue; continue;
} }
}
if entry.ends_with(SLASH_SEPARATOR) { if entry.ends_with(SLASH_SEPARATOR) {
if entry.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) { if entry.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) {
@@ -1133,11 +1131,11 @@ impl LocalDisk {
}) })
.await?; .await?;
if opts.recursive { if opts.recursive
if let Err(er) = Box::pin(self.scan_dir(pop, prefix.clone(), opts, out, objs_returned)).await { && let Err(er) = Box::pin(self.scan_dir(pop, prefix.clone(), opts, out, objs_returned)).await
{
error!("scan_dir err {:?}", er); error!("scan_dir err {:?}", er);
} }
}
dir_stack.pop(); dir_stack.pop();
} }
@@ -1200,12 +1198,12 @@ impl LocalDisk {
}) })
.await?; .await?;
if opts.recursive { if opts.recursive
if let Err(er) = Box::pin(self.scan_dir(dir, prefix.clone(), opts, out, objs_returned)).await { && let Err(er) = Box::pin(self.scan_dir(dir, prefix.clone(), opts, out, objs_returned)).await
{
warn!("scan_dir err {:?}", &er); warn!("scan_dir err {:?}", &er);
} }
} }
}
Ok(()) Ok(())
} }
@@ -1345,24 +1343,24 @@ impl DiskAPI for LocalDisk {
if format_info.file_info.is_some() && id.is_some() { if format_info.file_info.is_some() && id.is_some() {
// check last check time // check last check time
if let Some(last_check) = format_info.last_check { if let Some(last_check) = format_info.last_check
if last_check.unix_timestamp() + 1 < OffsetDateTime::now_utc().unix_timestamp() { && last_check.unix_timestamp() + 1 < OffsetDateTime::now_utc().unix_timestamp()
{
return Ok(id); return Ok(id);
} }
} }
}
let file_meta = self.check_format_json().await?; let file_meta = self.check_format_json().await?;
if let Some(file_info) = &format_info.file_info { if let Some(file_info) = &format_info.file_info
if super::fs::same_file(&file_meta, file_info) { && super::fs::same_file(&file_meta, file_info)
{
let mut format_info = self.format_info.write().await; let mut format_info = self.format_info.write().await;
format_info.last_check = Some(OffsetDateTime::now_utc()); format_info.last_check = Some(OffsetDateTime::now_utc());
drop(format_info); drop(format_info);
return Ok(id); return Ok(id);
} }
}
debug!("get_disk_id: read format.json"); debug!("get_disk_id: read format.json");
@@ -1420,11 +1418,11 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
let volume_dir = self.get_bucket_path(volume)?; let volume_dir = self.get_bucket_path(volume)?;
if !skip_access_checks(volume) { if !skip_access_checks(volume)
if let Err(e) = access(&volume_dir).await { && let Err(e) = access(&volume_dir).await
{
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
} }
}
let file_path = volume_dir.join(Path::new(&path)); let file_path = volume_dir.join(Path::new(&path));
check_path_length(file_path.to_string_lossy().to_string().as_str())?; check_path_length(file_path.to_string_lossy().to_string().as_str())?;
@@ -1438,11 +1436,11 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> { async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
let volume_dir = self.get_bucket_path(volume)?; let volume_dir = self.get_bucket_path(volume)?;
if !skip_access_checks(volume) { if !skip_access_checks(volume)
if let Err(e) = access(&volume_dir).await { && let Err(e) = access(&volume_dir).await
{
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
} }
}
let mut resp = CheckPartsResp { let mut resp = CheckPartsResp {
results: vec![0; fi.parts.len()], results: vec![0; fi.parts.len()],
@@ -1466,8 +1464,9 @@ impl DiskAPI for LocalDisk {
.await .await
.err(); .err();
resp.results[i] = conv_part_err_to_int(&err); resp.results[i] = conv_part_err_to_int(&err);
if resp.results[i] == CHECK_PART_UNKNOWN { if resp.results[i] == CHECK_PART_UNKNOWN
if let Some(err) = err { && let Some(err) = err
{
error!("verify_file: failed to bitrot verify file: {:?}, error: {:?}", &part_path, &err); error!("verify_file: failed to bitrot verify file: {:?}, error: {:?}", &part_path, &err);
if err == DiskError::FileAccessDenied { if err == DiskError::FileAccessDenied {
continue; continue;
@@ -1475,7 +1474,6 @@ impl DiskAPI for LocalDisk {
info!("part unknown, disk: {}, path: {:?}", self.to_string(), part_path); info!("part unknown, disk: {}, path: {:?}", self.to_string(), part_path);
} }
} }
}
Ok(resp) Ok(resp)
} }
@@ -1572,14 +1570,13 @@ impl DiskAPI for LocalDisk {
let e: DiskError = to_file_error(err).into(); let e: DiskError = to_file_error(err).into();
if e == DiskError::FileNotFound { if e == DiskError::FileNotFound {
if !skip_access_checks(volume) { if !skip_access_checks(volume)
if let Err(err) = access(&volume_dir).await { && let Err(err) = access(&volume_dir).await
if err.kind() == ErrorKind::NotFound { && err.kind() == ErrorKind::NotFound
{
resp.results[i] = CHECK_PART_VOLUME_NOT_FOUND; resp.results[i] = CHECK_PART_VOLUME_NOT_FOUND;
continue; continue;
} }
}
}
resp.results[i] = CHECK_PART_FILE_NOT_FOUND; resp.results[i] = CHECK_PART_FILE_NOT_FOUND;
} else { } else {
error!("check_parts: failed to stat file: {:?}, error: {:?}", &file_path, &e); error!("check_parts: failed to stat file: {:?}, error: {:?}", &file_path, &e);
@@ -1634,12 +1631,12 @@ impl DiskAPI for LocalDisk {
} }
}; };
if let Some(meta) = meta_op { if let Some(meta) = meta_op
if !meta.is_dir() { && !meta.is_dir()
{
warn!("rename_part src is not dir {:?}", &src_file_path); warn!("rename_part src is not dir {:?}", &src_file_path);
return Err(DiskError::FileAccessDenied); return Err(DiskError::FileAccessDenied);
} }
}
remove_std(&dst_file_path).map_err(to_file_error)?; remove_std(&dst_file_path).map_err(to_file_error)?;
} }
@@ -1695,11 +1692,11 @@ impl DiskAPI for LocalDisk {
} }
}; };
if let Some(meta) = meta_op { if let Some(meta) = meta_op
if !meta.is_dir() { && !meta.is_dir()
{
return Err(DiskError::FileAccessDenied); return Err(DiskError::FileAccessDenied);
} }
}
remove(&dst_file_path).await.map_err(to_file_error)?; remove(&dst_file_path).await.map_err(to_file_error)?;
} }
@@ -1814,12 +1811,12 @@ impl DiskAPI for LocalDisk {
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> { async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
if !origvolume.is_empty() { if !origvolume.is_empty() {
let origvolume_dir = self.get_bucket_path(origvolume)?; let origvolume_dir = self.get_bucket_path(origvolume)?;
if !skip_access_checks(origvolume) { if !skip_access_checks(origvolume)
if let Err(e) = access(origvolume_dir).await { && let Err(e) = access(origvolume_dir).await
{
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
} }
} }
}
let volume_dir = self.get_bucket_path(volume)?; let volume_dir = self.get_bucket_path(volume)?;
let dir_path_abs = volume_dir.join(Path::new(&dir_path.trim_start_matches(SLASH_SEPARATOR))); let dir_path_abs = volume_dir.join(Path::new(&dir_path.trim_start_matches(SLASH_SEPARATOR)));
@@ -1827,11 +1824,12 @@ impl DiskAPI for LocalDisk {
let entries = match os::read_dir(&dir_path_abs, count).await { let entries = match os::read_dir(&dir_path_abs, count).await {
Ok(res) => res, Ok(res) => res,
Err(e) => { Err(e) => {
if e.kind() == std::io::ErrorKind::NotFound && !skip_access_checks(volume) { if e.kind() == std::io::ErrorKind::NotFound
if let Err(e) = access(&volume_dir).await { && !skip_access_checks(volume)
&& let Err(e) = access(&volume_dir).await
{
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
} }
}
return Err(to_file_error(e).into()); return Err(to_file_error(e).into());
} }
@@ -1845,11 +1843,11 @@ impl DiskAPI for LocalDisk {
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> { async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
let volume_dir = self.get_bucket_path(&opts.bucket)?; let volume_dir = self.get_bucket_path(&opts.bucket)?;
if !skip_access_checks(&opts.bucket) { if !skip_access_checks(&opts.bucket)
if let Err(e) = access(&volume_dir).await { && let Err(e) = access(&volume_dir).await
{
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
} }
}
let mut wr = wr; let mut wr = wr;
@@ -1909,20 +1907,20 @@ impl DiskAPI for LocalDisk {
dst_path: &str, dst_path: &str,
) -> Result<RenameDataResp> { ) -> Result<RenameDataResp> {
let src_volume_dir = self.get_bucket_path(src_volume)?; let src_volume_dir = self.get_bucket_path(src_volume)?;
if !skip_access_checks(src_volume) { if !skip_access_checks(src_volume)
if let Err(e) = super::fs::access_std(&src_volume_dir) { && let Err(e) = super::fs::access_std(&src_volume_dir)
{
info!("access checks failed, src_volume_dir: {:?}, err: {}", src_volume_dir, e.to_string()); info!("access checks failed, src_volume_dir: {:?}, err: {}", src_volume_dir, e.to_string());
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
} }
}
let dst_volume_dir = self.get_bucket_path(dst_volume)?; let dst_volume_dir = self.get_bucket_path(dst_volume)?;
if !skip_access_checks(dst_volume) { if !skip_access_checks(dst_volume)
if let Err(e) = super::fs::access_std(&dst_volume_dir) { && let Err(e) = super::fs::access_std(&dst_volume_dir)
{
info!("access checks failed, dst_volume_dir: {:?}, err: {}", dst_volume_dir, e.to_string()); info!("access checks failed, dst_volume_dir: {:?}, err: {}", dst_volume_dir, e.to_string());
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into()); return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
} }
}
// xl.meta path // xl.meta path
let src_file_path = src_volume_dir.join(Path::new(format!("{}/{}", &src_path, STORAGE_FORMAT_FILE).as_str())); let src_file_path = src_volume_dir.join(Path::new(format!("{}/{}", &src_path, STORAGE_FORMAT_FILE).as_str()));
@@ -1973,20 +1971,19 @@ impl DiskAPI for LocalDisk {
let mut xlmeta = FileMeta::new(); let mut xlmeta = FileMeta::new();
if let Some(dst_buf) = has_dst_buf.as_ref() { if let Some(dst_buf) = has_dst_buf.as_ref()
if FileMeta::is_xl2_v1_format(dst_buf) { && FileMeta::is_xl2_v1_format(dst_buf)
if let Ok(nmeta) = FileMeta::load(dst_buf) { && let Ok(nmeta) = FileMeta::load(dst_buf)
{
xlmeta = nmeta xlmeta = nmeta
} }
}
}
let mut skip_parent = dst_volume_dir.clone(); let mut skip_parent = dst_volume_dir.clone();
if has_dst_buf.as_ref().is_some() { if has_dst_buf.as_ref().is_some()
if let Some(parent) = dst_file_path.parent() { && let Some(parent) = dst_file_path.parent()
{
skip_parent = parent.to_path_buf(); skip_parent = parent.to_path_buf();
} }
}
// TODO: Healing // TODO: Healing
@@ -2017,8 +2014,7 @@ impl DiskAPI for LocalDisk {
.await?; .await?;
if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() { if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() {
let no_inline = fi.data.is_none() && fi.size > 0; let no_inline = fi.data.is_none() && fi.size > 0;
if no_inline { if no_inline && let Err(err) = rename_all(&src_data_path, &dst_data_path, &skip_parent).await {
if let Err(err) = rename_all(&src_data_path, &dst_data_path, &skip_parent).await {
let _ = self.delete_file(&dst_volume_dir, dst_data_path, false, false).await; let _ = self.delete_file(&dst_volume_dir, dst_data_path, false, false).await;
info!( info!(
"rename all failed src_data_path: {:?}, dst_data_path: {:?}, err: {:?}", "rename all failed src_data_path: {:?}, dst_data_path: {:?}, err: {:?}",
@@ -2027,12 +2023,11 @@ impl DiskAPI for LocalDisk {
return Err(err); return Err(err);
} }
} }
}
if let Some(old_data_dir) = has_old_data_dir { if let Some(old_data_dir) = has_old_data_dir {
// preserve current xl.meta inside the oldDataDir. // preserve current xl.meta inside the oldDataDir.
if let Some(dst_buf) = has_dst_buf { if let Some(dst_buf) = has_dst_buf
if let Err(err) = self && let Err(err) = self
.write_all_private( .write_all_private(
dst_volume, dst_volume,
format!("{}/{}/{}", &dst_path, &old_data_dir.to_string(), STORAGE_FORMAT_FILE).as_str(), format!("{}/{}/{}", &dst_path, &old_data_dir.to_string(), STORAGE_FORMAT_FILE).as_str(),
@@ -2046,7 +2041,6 @@ impl DiskAPI for LocalDisk {
return Err(err); return Err(err);
} }
} }
}
if let Err(err) = rename_all(&src_file_path, &dst_file_path, &skip_parent).await { if let Err(err) = rename_all(&src_file_path, &dst_file_path, &skip_parent).await {
if let Some((_, dst_data_path)) = has_data_dir_path.as_ref() { if let Some((_, dst_data_path)) = has_data_dir_path.as_ref() {
@@ -2075,12 +2069,12 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> { async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
for vol in volumes { for vol in volumes {
if let Err(e) = self.make_volume(vol).await { if let Err(e) = self.make_volume(vol).await
if e != DiskError::VolumeExists { && e != DiskError::VolumeExists
{
error!("local disk make volumes failed: {e}"); error!("local disk make volumes failed: {e}");
return Err(e); return Err(e);
} }
}
// TODO: health check // TODO: health check
} }
Ok(()) Ok(())
@@ -2313,12 +2307,13 @@ impl DiskAPI for LocalDisk {
let old_path = file_path.join(Path::new(uuid.to_string().as_str())); let old_path = file_path.join(Path::new(uuid.to_string().as_str()));
check_path_length(old_path.to_string_lossy().as_ref())?; check_path_length(old_path.to_string_lossy().as_ref())?;
if let Err(err) = self.move_to_trash(&old_path, true, false).await { if let Err(err) = self.move_to_trash(&old_path, true, false).await
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound { && err != DiskError::FileNotFound
&& err != DiskError::VolumeNotFound
{
return Err(err); return Err(err);
} }
} }
}
if !meta.versions.is_empty() { if !meta.versions.is_empty() {
let buf = meta.marshal_msg()?; let buf = meta.marshal_msg()?;
@@ -2328,14 +2323,14 @@ impl DiskAPI for LocalDisk {
} }
// opts.undo_write && opts.old_data_dir.is_some_and(f) // opts.undo_write && opts.old_data_dir.is_some_and(f)
if let Some(old_data_dir) = opts.old_data_dir { if let Some(old_data_dir) = opts.old_data_dir
if opts.undo_write { && opts.undo_write
{
let src_path = let src_path =
file_path.join(Path::new(format!("{old_data_dir}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE_BACKUP}").as_str())); file_path.join(Path::new(format!("{old_data_dir}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE_BACKUP}").as_str()));
let dst_path = file_path.join(Path::new(format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str())); let dst_path = file_path.join(Path::new(format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str()));
return rename_all(src_path, dst_path, file_path).await; return rename_all(src_path, dst_path, file_path).await;
} }
}
self.delete_file(&volume_dir, &xl_path, true, false).await self.delete_file(&volume_dir, &xl_path, true, false).await
} }
+7 -8
View File
@@ -147,12 +147,12 @@ async fn reliable_rename(
dst_file_path: impl AsRef<Path>, dst_file_path: impl AsRef<Path>,
base_dir: impl AsRef<Path>, base_dir: impl AsRef<Path>,
) -> io::Result<()> { ) -> io::Result<()> {
if let Some(parent) = dst_file_path.as_ref().parent() { if let Some(parent) = dst_file_path.as_ref().parent()
if !file_exists(parent) { && !file_exists(parent)
{
// info!("reliable_rename reliable_mkdir_all parent: {:?}", parent); // info!("reliable_rename reliable_mkdir_all parent: {:?}", parent);
reliable_mkdir_all(parent, base_dir.as_ref()).await?; reliable_mkdir_all(parent, base_dir.as_ref()).await?;
} }
}
let mut i = 0; let mut i = 0;
loop { loop {
@@ -190,13 +190,12 @@ pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Pat
if e.kind() == io::ErrorKind::NotFound && i == 0 { if e.kind() == io::ErrorKind::NotFound && i == 0 {
i += 1; i += 1;
if let Some(base_parent) = base_dir.parent() { if let Some(base_parent) = base_dir.parent()
if let Some(c) = base_parent.components().next() { && let Some(c) = base_parent.components().next()
if c != Component::RootDir { && c != Component::RootDir
{
base_dir = base_parent base_dir = base_parent
} }
}
}
continue; continue;
} }
+2 -2
View File
@@ -318,7 +318,7 @@ fn get_divisible_size(total_sizes: &[usize]) -> usize {
fn possible_set_counts(set_size: usize) -> Vec<usize> { fn possible_set_counts(set_size: usize) -> Vec<usize> {
let mut ss = Vec::new(); let mut ss = Vec::new();
for s in SET_SIZES { for s in SET_SIZES {
if set_size % s == 0 { if set_size.is_multiple_of(s) {
ss.push(s); ss.push(s);
} }
} }
@@ -340,7 +340,7 @@ fn common_set_drive_count(divisible_size: usize, set_counts: &[usize]) -> usize
let mut prev_d = divisible_size / set_counts[0]; let mut prev_d = divisible_size / set_counts[0];
let mut set_size = 0; let mut set_size = 0;
for &cnt in set_counts { for &cnt in set_counts {
if divisible_size % cnt == 0 { if divisible_size.is_multiple_of(cnt) {
let d = divisible_size / cnt; let d = divisible_size / cnt;
if d <= prev_d { if d <= prev_d {
prev_d = d; prev_d = d;
+4 -5
View File
@@ -266,13 +266,12 @@ impl Erasure {
let (mut shards, errs) = reader.read().await; let (mut shards, errs) = reader.read().await;
if ret_err.is_none() { if ret_err.is_none()
if let (_, Some(err)) = reduce_errs(&errs, &[]) { && let (_, Some(err)) = reduce_errs(&errs, &[])
if err == Error::FileNotFound || err == Error::FileCorrupt { && (err == Error::FileNotFound || err == Error::FileCorrupt)
{
ret_err = Some(err.into()); ret_err = Some(err.into());
} }
}
}
if !reader.can_decode(&shards) { if !reader.can_decode(&shards) {
error!("erasure decode can_decode errs: {:?}", &errs); error!("erasure decode can_decode errs: {:?}", &errs);
+3 -3
View File
@@ -150,11 +150,11 @@ impl Erasure {
} }
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
// Check if the inner error is a checksum mismatch - if so, propagate it // Check if the inner error is a checksum mismatch - if so, propagate it
if let Some(inner) = e.get_ref() { if let Some(inner) = e.get_ref()
if rustfs_rio::is_checksum_mismatch(inner) { && rustfs_rio::is_checksum_mismatch(inner)
{
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())); return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()));
} }
}
break; break;
} }
Err(e) => { Err(e) => {
+1 -1
View File
@@ -45,7 +45,7 @@ impl super::Erasure {
let start_block = 0; let start_block = 0;
let mut end_block = total_length / self.block_size; let mut end_block = total_length / self.block_size;
if total_length % self.block_size != 0 { if !total_length.is_multiple_of(self.block_size) {
end_block += 1; end_block += 1;
} }
+32 -32
View File
@@ -244,11 +244,13 @@ impl PoolMeta {
} }
pub fn decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> { pub fn decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> {
if let Some(pool) = self.pools.get_mut(idx) { if let Some(pool) = self.pools.get_mut(idx) {
if let Some(ref info) = pool.decommission { if let Some(ref info) = pool.decommission
if !info.complete && !info.failed && !info.canceled { && !info.complete
&& !info.failed
&& !info.canceled
{
return Err(StorageError::DecommissionAlreadyRunning); return Err(StorageError::DecommissionAlreadyRunning);
} }
}
let now = OffsetDateTime::now_utc(); let now = OffsetDateTime::now_utc();
pool.last_update = now; pool.last_update = now;
@@ -273,14 +275,14 @@ impl PoolMeta {
pub fn pending_buckets(&self, idx: usize) -> Vec<DecomBucketInfo> { pub fn pending_buckets(&self, idx: usize) -> Vec<DecomBucketInfo> {
let mut list = Vec::new(); let mut list = Vec::new();
if let Some(pool) = self.pools.get(idx) { if let Some(pool) = self.pools.get(idx)
if let Some(ref info) = pool.decommission { && let Some(ref info) = pool.decommission
{
for bk in info.queued_buckets.iter() { for bk in info.queued_buckets.iter() {
let (name, prefix) = path2_bucket_object(bk); let (name, prefix) = path2_bucket_object(bk);
list.push(DecomBucketInfo { name, prefix }); list.push(DecomBucketInfo { name, prefix });
} }
} }
}
list list
} }
@@ -306,8 +308,9 @@ impl PoolMeta {
} }
pub fn count_item(&mut self, idx: usize, size: usize, failed: bool) { pub fn count_item(&mut self, idx: usize, size: usize, failed: bool) {
if let Some(pool) = self.pools.get_mut(idx) { if let Some(pool) = self.pools.get_mut(idx)
if let Some(info) = pool.decommission.as_mut() { && let Some(info) = pool.decommission.as_mut()
{
if failed { if failed {
info.items_decommission_failed += 1; info.items_decommission_failed += 1;
info.bytes_failed += size; info.bytes_failed += size;
@@ -317,20 +320,19 @@ impl PoolMeta {
} }
} }
} }
}
pub fn track_current_bucket_object(&mut self, idx: usize, bucket: String, object: String) { pub fn track_current_bucket_object(&mut self, idx: usize, bucket: String, object: String) {
if self.pools.get(idx).is_none_or(|v| v.decommission.is_none()) { if self.pools.get(idx).is_none_or(|v| v.decommission.is_none()) {
return; return;
} }
if let Some(pool) = self.pools.get_mut(idx) { if let Some(pool) = self.pools.get_mut(idx)
if let Some(info) = pool.decommission.as_mut() { && let Some(info) = pool.decommission.as_mut()
{
info.object = object; info.object = object;
info.bucket = bucket; info.bucket = bucket;
} }
} }
}
pub async fn update_after(&mut self, idx: usize, pools: Vec<Arc<Sets>>, duration: Duration) -> Result<bool> { pub async fn update_after(&mut self, idx: usize, pools: Vec<Arc<Sets>>, duration: Duration) -> Result<bool> {
if self.pools.get(idx).is_none_or(|v| v.decommission.is_none()) { if self.pools.get(idx).is_none_or(|v| v.decommission.is_none()) {
@@ -407,13 +409,13 @@ impl PoolMeta {
if specified_pools.len() == remembered_pools.len() { if specified_pools.len() == remembered_pools.len() {
for (k, pi) in remembered_pools.iter() { for (k, pi) in remembered_pools.iter() {
if let Some(pos) = specified_pools.get(k) { if let Some(pos) = specified_pools.get(k)
if *pos != pi.position { && *pos != pi.position
{
update = true; // Pool order changed, allow the update. update = true; // Pool order changed, allow the update.
} }
} }
} }
}
if !update { if !update {
update = specified_pools.len() != remembered_pools.len(); update = specified_pools.len() != remembered_pools.len();
@@ -640,12 +642,14 @@ impl ECStore {
pub async fn is_decommission_running(&self) -> bool { pub async fn is_decommission_running(&self) -> bool {
let pool_meta = self.pool_meta.read().await; let pool_meta = self.pool_meta.read().await;
for pool in pool_meta.pools.iter() { for pool in pool_meta.pools.iter() {
if let Some(ref info) = pool.decommission { if let Some(ref info) = pool.decommission
if !info.complete && !info.failed && !info.canceled { && !info.complete
&& !info.failed
&& !info.canceled
{
return true; return true;
} }
} }
}
false false
} }
@@ -850,8 +854,8 @@ impl ECStore {
decommissioned += 1; decommissioned += 1;
} }
if decommissioned == fivs.versions.len() { if decommissioned == fivs.versions.len()
if let Err(err) = set && let Err(err) = set
.delete_object( .delete_object(
bucket.as_str(), bucket.as_str(),
&encode_dir_object(&entry.name), &encode_dir_object(&entry.name),
@@ -866,7 +870,6 @@ impl ECStore {
{ {
error!("decommission_pool: delete_object err {:?}", &err); error!("decommission_pool: delete_object err {:?}", &err);
} }
}
{ {
let mut pool_meta = self.pool_meta.write().await; let mut pool_meta = self.pool_meta.write().await;
@@ -879,12 +882,10 @@ impl ECStore {
.unwrap_or_default(); .unwrap_or_default();
drop(pool_meta); drop(pool_meta);
if ok { if ok && let Some(notification_sys) = get_global_notification_sys() {
if let Some(notification_sys) = get_global_notification_sys() {
notification_sys.reload_pool_meta().await; notification_sys.reload_pool_meta().await;
} }
} }
}
warn!("decommission_pool: decommission_entry done {} {}", &bucket, &entry.name); warn!("decommission_pool: decommission_entry done {} {}", &bucket, &entry.name);
} }
@@ -1080,12 +1081,12 @@ impl ECStore {
{ {
let mut pool_meta = self.pool_meta.write().await; let mut pool_meta = self.pool_meta.write().await;
if pool_meta.bucket_done(idx, bucket.to_string()) { if pool_meta.bucket_done(idx, bucket.to_string())
if let Err(err) = pool_meta.save(self.pools.clone()).await { && let Err(err) = pool_meta.save(self.pools.clone()).await
{
error!("decom pool_meta.save err {:?}", err); error!("decom pool_meta.save err {:?}", err);
} }
} }
}
continue; continue;
} }
@@ -1100,11 +1101,11 @@ impl ECStore {
{ {
let mut pool_meta = self.pool_meta.write().await; let mut pool_meta = self.pool_meta.write().await;
if pool_meta.bucket_done(idx, bucket.to_string()) { if pool_meta.bucket_done(idx, bucket.to_string())
if let Err(err) = pool_meta.save(self.pools.clone()).await { && let Err(err) = pool_meta.save(self.pools.clone()).await
{
error!("decom pool_meta.save err {:?}", err); error!("decom pool_meta.save err {:?}", err);
} }
}
warn!("decommission: decommission_pool bucket_done {}", &bucket.name); warn!("decommission: decommission_pool bucket_done {}", &bucket.name);
} }
@@ -1138,13 +1139,12 @@ impl ECStore {
if let Err(err) = self if let Err(err) = self
.make_bucket(bk.to_string_lossy().to_string().as_str(), &MakeBucketOptions::default()) .make_bucket(bk.to_string_lossy().to_string().as_str(), &MakeBucketOptions::default())
.await .await
&& !is_err_bucket_exists(&err)
{ {
if !is_err_bucket_exists(&err) {
error!("decommission: make bucket failed: {err}"); error!("decommission: make bucket failed: {err}");
return Err(err); return Err(err);
} }
} }
}
let mut pool_meta = self.pool_meta.write().await; let mut pool_meta = self.pool_meta.write().await;
for idx in indices.iter() { for idx in indices.iter() {
+18 -18
View File
@@ -380,11 +380,11 @@ impl ECStore {
#[tracing::instrument(skip(self, fi))] #[tracing::instrument(skip(self, fi))]
pub async fn update_pool_stats(&self, pool_index: usize, bucket: String, fi: &FileInfo) -> Result<()> { pub async fn update_pool_stats(&self, pool_index: usize, bucket: String, fi: &FileInfo) -> Result<()> {
let mut rebalance_meta = self.rebalance_meta.write().await; let mut rebalance_meta = self.rebalance_meta.write().await;
if let Some(meta) = rebalance_meta.as_mut() { if let Some(meta) = rebalance_meta.as_mut()
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) { && let Some(pool_stat) = meta.pool_stats.get_mut(pool_index)
{
pool_stat.update(bucket, fi); pool_stat.update(bucket, fi);
} }
}
Ok(()) Ok(())
} }
@@ -394,8 +394,9 @@ impl ECStore {
info!("next_rebal_bucket: pool_index: {}", pool_index); info!("next_rebal_bucket: pool_index: {}", pool_index);
let rebalance_meta = self.rebalance_meta.read().await; let rebalance_meta = self.rebalance_meta.read().await;
info!("next_rebal_bucket: rebalance_meta: {:?}", rebalance_meta); info!("next_rebal_bucket: rebalance_meta: {:?}", rebalance_meta);
if let Some(meta) = rebalance_meta.as_ref() { if let Some(meta) = rebalance_meta.as_ref()
if let Some(pool_stat) = meta.pool_stats.get(pool_index) { && let Some(pool_stat) = meta.pool_stats.get(pool_index)
{
if pool_stat.info.status == RebalStatus::Completed || !pool_stat.participating { if pool_stat.info.status == RebalStatus::Completed || !pool_stat.participating {
info!("next_rebal_bucket: pool_index: {} completed or not participating", pool_index); info!("next_rebal_bucket: pool_index: {} completed or not participating", pool_index);
return Ok(None); return Ok(None);
@@ -408,7 +409,6 @@ impl ECStore {
info!("next_rebal_bucket: pool_index: {} bucket: {}", pool_index, pool_stat.buckets[0]); info!("next_rebal_bucket: pool_index: {} bucket: {}", pool_index, pool_stat.buckets[0]);
return Ok(Some(pool_stat.buckets[0].clone())); return Ok(Some(pool_stat.buckets[0].clone()));
} }
}
info!("next_rebal_bucket: pool_index: {} None", pool_index); info!("next_rebal_bucket: pool_index: {} None", pool_index);
Ok(None) Ok(None)
@@ -417,8 +417,9 @@ impl ECStore {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
pub async fn bucket_rebalance_done(&self, pool_index: usize, bucket: String) -> Result<()> { pub async fn bucket_rebalance_done(&self, pool_index: usize, bucket: String) -> Result<()> {
let mut rebalance_meta = self.rebalance_meta.write().await; let mut rebalance_meta = self.rebalance_meta.write().await;
if let Some(meta) = rebalance_meta.as_mut() { if let Some(meta) = rebalance_meta.as_mut()
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) { && let Some(pool_stat) = meta.pool_stats.get_mut(pool_index)
{
info!("bucket_rebalance_done: buckets {:?}", &pool_stat.buckets); info!("bucket_rebalance_done: buckets {:?}", &pool_stat.buckets);
// Use retain to filter out buckets slated for removal // Use retain to filter out buckets slated for removal
@@ -440,7 +441,6 @@ impl ECStore {
info!("bucket_rebalance_done: bucket {} not found", bucket); info!("bucket_rebalance_done: bucket {} not found", bucket);
} }
} }
}
info!("bucket_rebalance_done: bucket {} not found", bucket); info!("bucket_rebalance_done: bucket {} not found", bucket);
Ok(()) Ok(())
} }
@@ -492,11 +492,11 @@ impl ECStore {
#[tracing::instrument(skip(self))] #[tracing::instrument(skip(self))]
pub async fn stop_rebalance(self: &Arc<Self>) -> Result<()> { pub async fn stop_rebalance(self: &Arc<Self>) -> Result<()> {
let rebalance_meta = self.rebalance_meta.read().await; let rebalance_meta = self.rebalance_meta.read().await;
if let Some(meta) = rebalance_meta.as_ref() { if let Some(meta) = rebalance_meta.as_ref()
if let Some(cancel_tx) = meta.cancel.as_ref() { && let Some(cancel_tx) = meta.cancel.as_ref()
{
cancel_tx.cancel(); cancel_tx.cancel();
} }
}
Ok(()) Ok(())
} }
@@ -690,8 +690,9 @@ impl ECStore {
async fn check_if_rebalance_done(&self, pool_index: usize) -> bool { async fn check_if_rebalance_done(&self, pool_index: usize) -> bool {
let mut rebalance_meta = self.rebalance_meta.write().await; let mut rebalance_meta = self.rebalance_meta.write().await;
if let Some(meta) = rebalance_meta.as_mut() { if let Some(meta) = rebalance_meta.as_mut()
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) { && let Some(pool_stat) = meta.pool_stats.get_mut(pool_index)
{
// Check if the pool's rebalance status is already completed // Check if the pool's rebalance status is already completed
if pool_stat.info.status == RebalStatus::Completed { if pool_stat.info.status == RebalStatus::Completed {
info!("check_if_rebalance_done: pool {} is already completed", pool_index); info!("check_if_rebalance_done: pool {} is already completed", pool_index);
@@ -709,7 +710,6 @@ impl ECStore {
return true; return true;
} }
} }
}
false false
} }
@@ -1102,12 +1102,12 @@ impl ECStore {
pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> { pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
// TODO: lock // TODO: lock
let mut meta = RebalanceMeta::new(); let mut meta = RebalanceMeta::new();
if let Err(err) = meta.load(self.pools[0].clone()).await { if let Err(err) = meta.load(self.pools[0].clone()).await
if err != Error::ConfigNotFound { && err != Error::ConfigNotFound
{
info!("save_rebalance_stats: load err: {:?}", err); info!("save_rebalance_stats: load err: {:?}", err);
return Err(err); return Err(err);
} }
}
match opt { match opt {
RebalSaveOpt::Stats => { RebalSaveOpt::Stats => {
+3 -3
View File
@@ -66,15 +66,15 @@ impl PeerRestClient {
let mut remote = Vec::with_capacity(hosts.len()); let mut remote = Vec::with_capacity(hosts.len());
let mut all = vec![None; hosts.len()]; let mut all = vec![None; hosts.len()];
for (i, hs_host) in hosts.iter().enumerate() { for (i, hs_host) in hosts.iter().enumerate() {
if let Some(host) = hs_host { if let Some(host) = hs_host
if let Some(grid_host) = eps.find_grid_hosts_from_peer(host) { && let Some(grid_host) = eps.find_grid_hosts_from_peer(host)
{
let client = PeerRestClient::new(host.clone(), grid_host); let client = PeerRestClient::new(host.clone(), grid_host);
all[i] = Some(client.clone()); all[i] = Some(client.clone());
remote.push(Some(client)); remote.push(Some(client));
} }
} }
}
if all.len() != remote.len() + 1 { if all.len() != remote.len() + 1 {
warn!("Expected number of all hosts ({}) to be remote +1 ({})", all.len(), remote.len()); warn!("Expected number of all hosts ({}) to be remote +1 ({})", all.len(), remote.len());
+6 -6
View File
@@ -101,12 +101,12 @@ impl S3PeerSys {
for pool_idx in 0..self.pools_count { for pool_idx in 0..self.pools_count {
let mut per_pool_errs = vec![None; self.clients.len()]; let mut per_pool_errs = vec![None; self.clients.len()];
for (i, client) in self.clients.iter().enumerate() { for (i, client) in self.clients.iter().enumerate() {
if let Some(v) = client.get_pools() { if let Some(v) = client.get_pools()
if v.contains(&pool_idx) { && v.contains(&pool_idx)
{
per_pool_errs[i] = errs[i].clone(); per_pool_errs[i] = errs[i].clone();
} }
} }
}
let qu = per_pool_errs.len() / 2; let qu = per_pool_errs.len() / 2;
pool_errs.push(reduce_write_quorum_errs(&per_pool_errs, BUCKET_OP_IGNORED_ERRS, qu)); pool_errs.push(reduce_write_quorum_errs(&per_pool_errs, BUCKET_OP_IGNORED_ERRS, qu));
} }
@@ -136,12 +136,12 @@ impl S3PeerSys {
for pool_idx in 0..self.pools_count { for pool_idx in 0..self.pools_count {
let mut per_pool_errs = vec![None; self.clients.len()]; let mut per_pool_errs = vec![None; self.clients.len()];
for (i, client) in self.clients.iter().enumerate() { for (i, client) in self.clients.iter().enumerate() {
if let Some(v) = client.get_pools() { if let Some(v) = client.get_pools()
if v.contains(&pool_idx) { && v.contains(&pool_idx)
{
per_pool_errs[i] = errs[i].clone(); per_pool_errs[i] = errs[i].clone();
} }
} }
}
let qu = per_pool_errs.len() / 2; let qu = per_pool_errs.len() / 2;
if let Some(pool_err) = reduce_write_quorum_errs(&per_pool_errs, BUCKET_OP_IGNORED_ERRS, qu) { if let Some(pool_err) = reduce_write_quorum_errs(&per_pool_errs, BUCKET_OP_IGNORED_ERRS, qu) {
tracing::error!("heal_bucket per_pool_errs: {per_pool_errs:?}"); tracing::error!("heal_bucket per_pool_errs: {per_pool_errs:?}");
+60 -62
View File
@@ -266,12 +266,12 @@ impl SetDisks {
let mut new_disk = Vec::with_capacity(disks.len()); let mut new_disk = Vec::with_capacity(disks.len());
for disk in disks.iter() { for disk in disks.iter() {
if let Some(d) = disk { if let Some(d) = disk
if d.is_online().await { && d.is_online().await
{
new_disk.push(disk.clone()); new_disk.push(disk.clone());
} }
} }
}
let mut rng = rand::rng(); let mut rng = rand::rng();
@@ -1417,9 +1417,10 @@ impl SetDisks {
let mut valid_obj_map = HashMap::new(); let mut valid_obj_map = HashMap::new();
for (i, op_hash) in meta_hashes.iter().enumerate() { for (i, op_hash) in meta_hashes.iter().enumerate() {
if let Some(hash) = op_hash { if let Some(hash) = op_hash
if let Some(max_hash) = max_val { && let Some(max_hash) = max_val
if hash == max_hash { && hash == max_hash
{
if metas[i].is_valid() && !found { if metas[i].is_valid() && !found {
found_fi = Some(metas[i].clone()); found_fi = Some(metas[i].clone());
found = true; found = true;
@@ -1433,8 +1434,6 @@ impl SetDisks {
*valid_obj_map.entry(props).or_insert(0) += 1; *valid_obj_map.entry(props).or_insert(0) += 1;
} }
} }
}
}
if found { if found {
let mut fi = found_fi.unwrap(); let mut fi = found_fi.unwrap();
@@ -3572,19 +3571,19 @@ impl SetDisks {
let mut offline = 0; let mut offline = 0;
for (i, err) in errs.iter().enumerate() { for (i, err) in errs.iter().enumerate() {
let mut found = false; let mut found = false;
if let Some(err) = err { if let Some(err) = err
if err == &DiskError::DiskNotFound { && err == &DiskError::DiskNotFound
{
found = true; found = true;
} }
}
for p in data_errs_by_part { for p in data_errs_by_part {
if let Some(v) = p.1.get(i) { if let Some(v) = p.1.get(i)
if *v == CHECK_PART_DISK_NOT_FOUND { && *v == CHECK_PART_DISK_NOT_FOUND
{
found = true; found = true;
break; break;
} }
} }
}
if found { if found {
offline += 1; offline += 1;
@@ -3838,11 +3837,11 @@ impl ObjectIO for SetDisks {
None None
}; };
if let Some(http_preconditions) = opts.http_preconditions.clone() { if let Some(http_preconditions) = opts.http_preconditions.clone()
if let Some(err) = self.check_write_precondition(bucket, object, opts).await { && let Some(err) = self.check_write_precondition(bucket, object, opts).await
{
return Err(err); return Err(err);
} }
}
let mut user_defined = opts.user_defined.clone(); let mut user_defined = opts.user_defined.clone();
@@ -4002,17 +4001,17 @@ impl ObjectIO for SetDisks {
} }
} }
if fi.checksum.is_none() { if fi.checksum.is_none()
if let Some(content_hash) = data.as_hash_reader().content_hash() { && let Some(content_hash) = data.as_hash_reader().content_hash()
{
fi.checksum = Some(content_hash.to_bytes(&[])); fi.checksum = Some(content_hash.to_bytes(&[]));
} }
}
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS) { if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS)
if sc == storageclass::STANDARD { && sc == storageclass::STANDARD
{
let _ = user_defined.remove(AMZ_STORAGE_CLASS); let _ = user_defined.remove(AMZ_STORAGE_CLASS);
} }
}
let mod_time = if let Some(mod_time) = opts.mod_time { let mod_time = if let Some(mod_time) = opts.mod_time {
Some(mod_time) Some(mod_time)
@@ -4062,13 +4061,13 @@ impl ObjectIO for SetDisks {
self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?; self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?;
for (i, op_disk) in online_disks.iter().enumerate() { for (i, op_disk) in online_disks.iter().enumerate() {
if let Some(disk) = op_disk { if let Some(disk) = op_disk
if disk.is_online().await { && disk.is_online().await
{
fi = parts_metadatas[i].clone(); fi = parts_metadatas[i].clone();
break; break;
} }
} }
}
fi.replication_state_internal = Some(opts.put_replication_state()); fi.replication_state_internal = Some(opts.put_replication_state());
@@ -5568,11 +5567,11 @@ impl StorageAPI for SetDisks {
user_defined.insert("etag".to_owned(), etag.clone()); user_defined.insert("etag".to_owned(), etag.clone());
} }
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS) { if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS)
if sc == storageclass::STANDARD { && sc == storageclass::STANDARD
{
let _ = user_defined.remove(AMZ_STORAGE_CLASS); let _ = user_defined.remove(AMZ_STORAGE_CLASS);
} }
}
let sc_parity_drives = { let sc_parity_drives = {
if let Some(sc) = GLOBAL_STORAGE_CLASS.get() { if let Some(sc) = GLOBAL_STORAGE_CLASS.get() {
@@ -5620,11 +5619,11 @@ impl StorageAPI for SetDisks {
// TODO: get content-type // TODO: get content-type
} }
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS) { if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS)
if sc == storageclass::STANDARD { && sc == storageclass::STANDARD
{
let _ = user_defined.remove(AMZ_STORAGE_CLASS); let _ = user_defined.remove(AMZ_STORAGE_CLASS);
} }
}
if let Some(checksum) = &opts.want_checksum { if let Some(checksum) = &opts.want_checksum {
user_defined.insert(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM.to_string(), checksum.checksum_type.to_string()); user_defined.insert(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM.to_string(), checksum.checksum_type.to_string());
@@ -5925,15 +5924,15 @@ impl StorageAPI for SetDisks {
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default())); return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
} }
if checksum_type.full_object_requested() { if checksum_type.full_object_requested()
if let Err(err) = checksum.add_part(&cs, ext_part.actual_size) { && let Err(err) = checksum.add_part(&cs, ext_part.actual_size)
{
error!( error!(
"complete_multipart_upload checksum add_part failed part_id={}, bucket={}, object={}", "complete_multipart_upload checksum add_part failed part_id={}, bucket={}, object={}",
p.part_num, bucket, object p.part_num, bucket, object
); );
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default())); return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
} }
}
checksum_combined.extend_from_slice(cs.raw.as_slice()); checksum_combined.extend_from_slice(cs.raw.as_slice());
} }
@@ -6112,13 +6111,13 @@ impl StorageAPI for SetDisks {
}); });
for (i, op_disk) in online_disks.iter().enumerate() { for (i, op_disk) in online_disks.iter().enumerate() {
if let Some(disk) = op_disk { if let Some(disk) = op_disk
if disk.is_online().await { && disk.is_online().await
{
fi = parts_metadatas[i].clone(); fi = parts_metadatas[i].clone();
break; break;
} }
} }
}
fi.is_latest = true; fi.is_latest = true;
@@ -6210,8 +6209,8 @@ impl StorageAPI for SetDisks {
let _write_lock_guard = if !opts.no_lock { let _write_lock_guard = if !opts.no_lock {
let key = rustfs_lock::fast_lock::types::ObjectKey::new(bucket, object); let key = rustfs_lock::fast_lock::types::ObjectKey::new(bucket, object);
let mut skip_lock = false; let mut skip_lock = false;
if let Some(lock_info) = self.fast_lock_manager.get_lock_info(&key) { if let Some(lock_info) = self.fast_lock_manager.get_lock_info(&key)
if lock_info.owner.as_ref() == self.locker_owner.as_str() && lock_info.owner.as_ref() == self.locker_owner.as_str()
&& matches!(lock_info.mode, rustfs_lock::fast_lock::types::LockMode::Exclusive) && matches!(lock_info.mode, rustfs_lock::fast_lock::types::LockMode::Exclusive)
{ {
debug!( debug!(
@@ -6220,7 +6219,6 @@ impl StorageAPI for SetDisks {
); );
skip_lock = true; skip_lock = true;
} }
}
if skip_lock { if skip_lock {
None None
} else { } else {
@@ -6563,8 +6561,9 @@ async fn disks_with_all_parts(
if err.is_some() { if err.is_some() {
let part_err = conv_part_err_to_int(err); let part_err = conv_part_err_to_int(err);
for p in 0..latest_meta.parts.len() { for p in 0..latest_meta.parts.len() {
if let Some(vec) = data_errs_by_part.get_mut(&p) { if let Some(vec) = data_errs_by_part.get_mut(&p)
if index < vec.len() { && index < vec.len()
{
info!( info!(
"data_errs_by_part: copy meta errors to part errors: object_name={}, index: {index}, part: {p}, part_err: {part_err}", "data_errs_by_part: copy meta errors to part errors: object_name={}, index: {index}, part: {p}, part_err: {part_err}",
object_name object_name
@@ -6574,7 +6573,6 @@ async fn disks_with_all_parts(
} }
} }
} }
}
// Check data for each disk // Check data for each disk
for (index, disk) in online_disks.iter().enumerate() { for (index, disk) in online_disks.iter().enumerate() {
@@ -6609,8 +6607,9 @@ async fn disks_with_all_parts(
.await .await
.err(); .err();
if let Some(vec) = data_errs_by_part.get_mut(&0) { if let Some(vec) = data_errs_by_part.get_mut(&0)
if index < vec.len() { && index < vec.len()
{
vec[index] = conv_part_err_to_int(&verify_err.map(|e| e.into())); vec[index] = conv_part_err_to_int(&verify_err.map(|e| e.into()));
info!( info!(
"data_errs_by_part:bitrot check result: object_name={}, index: {index}, result: {}", "data_errs_by_part:bitrot check result: object_name={}, index: {index}, result: {}",
@@ -6618,7 +6617,6 @@ async fn disks_with_all_parts(
); );
} }
} }
}
continue; continue;
} }
@@ -6654,8 +6652,9 @@ async fn disks_with_all_parts(
// Update dataErrsByPart for all parts // Update dataErrsByPart for all parts
for p in 0..latest_meta.parts.len() { for p in 0..latest_meta.parts.len() {
if let Some(vec) = data_errs_by_part.get_mut(&p) { if let Some(vec) = data_errs_by_part.get_mut(&p)
if index < vec.len() { && index < vec.len()
{
if verify_err.is_some() { if verify_err.is_some() {
info!( info!(
"data_errs_by_part: verify_err: object_name={}, index: {index}, part: {p}, verify_err: {verify_err:?}", "data_errs_by_part: verify_err: object_name={}, index: {index}, part: {p}, verify_err: {verify_err:?}",
@@ -6684,13 +6683,13 @@ async fn disks_with_all_parts(
} }
} }
} }
}
// Build dataErrsByDisk from dataErrsByPart // Build dataErrsByDisk from dataErrsByPart
for (part, disks) in data_errs_by_part.iter() { for (part, disks) in data_errs_by_part.iter() {
for (disk_idx, disk_err) in disks.iter().enumerate() { for (disk_idx, disk_err) in disks.iter().enumerate() {
if let Some(vec) = data_errs_by_disk.get_mut(&disk_idx) { if let Some(vec) = data_errs_by_disk.get_mut(&disk_idx)
if *part < vec.len() { && *part < vec.len()
{
vec[*part] = *disk_err; vec[*part] = *disk_err;
info!( info!(
"data_errs_by_disk: update data_errs_by_disk: object_name={}, part: {part}, disk_idx: {disk_idx}, disk_err: {disk_err}", "data_errs_by_disk: update data_errs_by_disk: object_name={}, part: {part}, disk_idx: {disk_idx}, disk_err: {disk_err}",
@@ -6699,7 +6698,6 @@ async fn disks_with_all_parts(
} }
} }
} }
}
// Calculate available_disks based on meta_errs and data_errs_by_disk // Calculate available_disks based on meta_errs and data_errs_by_disk
for (i, disk) in online_disks.iter().enumerate() { for (i, disk) in online_disks.iter().enumerate() {
@@ -6738,11 +6736,11 @@ pub fn should_heal_object_on_disk(
meta: &FileInfo, meta: &FileInfo,
latest_meta: &FileInfo, latest_meta: &FileInfo,
) -> (bool, Option<DiskError>) { ) -> (bool, Option<DiskError>) {
if let Some(err) = err { if let Some(err) = err
if err == &DiskError::FileNotFound || err == &DiskError::FileVersionNotFound || err == &DiskError::FileCorrupt { && (err == &DiskError::FileNotFound || err == &DiskError::FileVersionNotFound || err == &DiskError::FileCorrupt)
{
return (true, Some(err.clone())); return (true, Some(err.clone()));
} }
}
if latest_meta.volume != meta.volume if latest_meta.volume != meta.volume
|| latest_meta.name != meta.name || latest_meta.name != meta.name
@@ -6906,16 +6904,16 @@ pub fn e_tag_matches(etag: &str, condition: &str) -> bool {
pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<String>, if_match: Option<String>) -> bool { pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<String>, if_match: Option<String>) -> bool {
match &oi.etag { match &oi.etag {
Some(etag) => { Some(etag) => {
if let Some(if_none_match) = if_none_match { if let Some(if_none_match) = if_none_match
if e_tag_matches(etag, &if_none_match) { && e_tag_matches(etag, &if_none_match)
{
return true; return true;
} }
} if let Some(if_match) = if_match
if let Some(if_match) = if_match { && !e_tag_matches(etag, &if_match)
if !e_tag_matches(etag, &if_match) { {
return true; return true;
} }
}
false false
} }
// If we can't obtain the etag of the object, perevent the write only when we have at least one condition // If we can't obtain the etag of the object, perevent the write only when we have at least one condition
+13 -14
View File
@@ -491,13 +491,13 @@ impl StorageAPI for Sets {
let cp_src_dst_same = path_join_buf(&[src_bucket, src_object]) == path_join_buf(&[dst_bucket, dst_object]); let cp_src_dst_same = path_join_buf(&[src_bucket, src_object]) == path_join_buf(&[dst_bucket, dst_object]);
if cp_src_dst_same { if cp_src_dst_same {
if let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id) { if let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id)
if src_vid == dst_vid { && src_vid == dst_vid
{
return src_set return src_set
.copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts) .copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
.await; .await;
} }
}
if !dst_opts.versioned && src_opts.version_id.is_none() { if !dst_opts.versioned && src_opts.version_id.is_none() {
return src_set return src_set
@@ -823,11 +823,11 @@ impl StorageAPI for Sets {
Ok((m, n)) => (m, n), Ok((m, n)) => (m, n),
Err(_) => continue, Err(_) => continue,
}; };
if let Some(set) = self.disk_set.get(m) { if let Some(set) = self.disk_set.get(m)
if let Some(Some(disk)) = set.disks.read().await.get(n) { && let Some(Some(disk)) = set.disks.read().await.get(n)
{
let _ = disk.close().await; let _ = disk.close().await;
} }
}
if let Some(Some(disk)) = disks.get(index) { if let Some(Some(disk)) = disks.get(index) {
self.disk_set[m].renew_disk(&disk.endpoint()).await; self.disk_set[m].renew_disk(&disk.endpoint()).await;
@@ -980,8 +980,9 @@ fn new_heal_format_sets(
let mut current_disks_info = vec![vec![DiskInfo::default(); set_drive_count]; set_count]; let mut current_disks_info = vec![vec![DiskInfo::default(); set_drive_count]; set_count];
for (i, set) in ref_format.erasure.sets.iter().enumerate() { for (i, set) in ref_format.erasure.sets.iter().enumerate() {
for j in 0..set.len() { for j in 0..set.len() {
if let Some(Some(err)) = errs.get(i * set_drive_count + j) { if let Some(Some(err)) = errs.get(i * set_drive_count + j)
if *err == DiskError::UnformattedDisk { && *err == DiskError::UnformattedDisk
{
let mut fm = FormatV3::new(set_count, set_drive_count); let mut fm = FormatV3::new(set_count, set_drive_count);
fm.id = ref_format.id; fm.id = ref_format.id;
fm.format = ref_format.format.clone(); fm.format = ref_format.format.clone();
@@ -992,16 +993,14 @@ fn new_heal_format_sets(
fm.erasure.distribution_algo = ref_format.erasure.distribution_algo.clone(); fm.erasure.distribution_algo = ref_format.erasure.distribution_algo.clone();
new_formats[i][j] = Some(fm); new_formats[i][j] = Some(fm);
} }
} if let (Some(format), None) = (&formats[i * set_drive_count + j], &errs[i * set_drive_count + j])
if let (Some(format), None) = (&formats[i * set_drive_count + j], &errs[i * set_drive_count + j]) { && let Some(info) = &format.disk_info
if let Some(info) = &format.disk_info { && !info.endpoint.is_empty()
if !info.endpoint.is_empty() { {
current_disks_info[i][j] = info.clone(); current_disks_info[i][j] = info.clone();
} }
} }
} }
}
}
(new_formats, current_disks_info) (new_formats, current_disks_info)
} }
+25 -24
View File
@@ -243,11 +243,11 @@ impl ECStore {
}); });
// Only set it when the global deployment ID is not yet configured // Only set it when the global deployment ID is not yet configured
if let Some(dep_id) = deployment_id { if let Some(dep_id) = deployment_id
if get_global_deployment_id().is_none() { && get_global_deployment_id().is_none()
{
set_global_deployment_id(dep_id); set_global_deployment_id(dep_id);
} }
}
let wait_sec = 5; let wait_sec = 5;
let mut exit_count = 0; let mut exit_count = 0;
@@ -768,11 +768,11 @@ impl ECStore {
def_pool = pinfo.clone(); def_pool = pinfo.clone();
has_def_pool = true; has_def_pool = true;
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-deletes.html // https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-deletes.html
if is_err_object_not_found(err) { if is_err_object_not_found(err)
if let Err(err) = opts.precondition_check(&pinfo.object_info) { && let Err(err) = opts.precondition_check(&pinfo.object_info)
{
return Err(err.clone()); return Err(err.clone());
} }
}
if !is_err_object_not_found(err) && !is_err_version_not_found(err) { if !is_err_object_not_found(err) && !is_err_version_not_found(err) {
return Err(err.clone()); return Err(err.clone());
@@ -885,14 +885,15 @@ impl ECStore {
return Ok((obj, res.idx)); return Ok((obj, res.idx));
} }
if let Some(err) = res.err { if let Some(err) = res.err
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) { && !is_err_object_not_found(&err)
&& !is_err_version_not_found(&err)
{
return Err(err); return Err(err);
} }
// TODO: delete marker // TODO: delete marker
} }
}
let object = decode_dir_object(object); let object = decode_dir_object(object);
@@ -918,13 +919,13 @@ impl ECStore {
let mut derrs = Vec::new(); let mut derrs = Vec::new();
for pe in errs.iter() { for pe in errs.iter() {
if let Some(err) = &pe.err { if let Some(err) = &pe.err
if err == &StorageError::ErasureWriteQuorum { && err == &StorageError::ErasureWriteQuorum
{
objs.push(None); objs.push(None);
derrs.push(Some(StorageError::ErasureWriteQuorum)); derrs.push(Some(StorageError::ErasureWriteQuorum));
continue; continue;
} }
}
if let Some(idx) = pe.index { if let Some(idx) = pe.index {
match self.pools[idx].delete_object(bucket, object, opts.clone()).await { match self.pools[idx].delete_object(bucket, object, opts.clone()).await {
@@ -1226,13 +1227,13 @@ impl StorageAPI for ECStore {
#[instrument(skip(self))] #[instrument(skip(self))]
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> { async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
if !is_meta_bucketname(bucket) { if !is_meta_bucketname(bucket)
if let Err(err) = check_valid_bucket_name_strict(bucket) { && let Err(err) = check_valid_bucket_name_strict(bucket)
{
return Err(StorageError::BucketNameInvalid(err.to_string())); return Err(StorageError::BucketNameInvalid(err.to_string()));
} }
// TODO: nslock // TODO: nslock
}
if let Err(err) = self.peer_sys.make_bucket(bucket, opts).await { if let Err(err) = self.peer_sys.make_bucket(bucket, opts).await {
let err = to_object_err(err.into(), vec![bucket]); let err = to_object_err(err.into(), vec![bucket]);
@@ -1427,13 +1428,13 @@ impl StorageAPI for ECStore {
let pool_idx = self.get_pool_idx_no_lock(src_bucket, &src_object, src_info.size).await?; let pool_idx = self.get_pool_idx_no_lock(src_bucket, &src_object, src_info.size).await?;
if cp_src_dst_same { if cp_src_dst_same {
if let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id) { if let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id)
if src_vid == dst_vid { && src_vid == dst_vid
{
return self.pools[pool_idx] return self.pools[pool_idx]
.copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, dst_opts) .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, dst_opts)
.await; .await;
} }
}
if !dst_opts.versioned && src_opts.version_id.is_none() { if !dst_opts.versioned && src_opts.version_id.is_none() {
return self.pools[pool_idx] return self.pools[pool_idx]
@@ -2433,14 +2434,14 @@ fn check_list_multipart_args(
check_list_objs_args(bucket, prefix, key_marker)?; check_list_objs_args(bucket, prefix, key_marker)?;
if let Some(upload_id_marker) = upload_id_marker { if let Some(upload_id_marker) = upload_id_marker {
if let Some(key_marker) = key_marker { if let Some(key_marker) = key_marker
if key_marker.ends_with('/') { && key_marker.ends_with('/')
{
return Err(StorageError::InvalidUploadIDKeyCombination( return Err(StorageError::InvalidUploadIDKeyCombination(
upload_id_marker.to_string(), upload_id_marker.to_string(),
key_marker.to_string(), key_marker.to_string(),
)); ));
} }
}
if let Err(_e) = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id_marker.as_bytes()) { if let Err(_e) = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id_marker.as_bytes()) {
return Err(StorageError::MalformedUploadID(upload_id_marker.to_owned())); return Err(StorageError::MalformedUploadID(upload_id_marker.to_owned()));
@@ -2510,12 +2511,12 @@ pub async fn get_disk_infos(disks: &[Option<DiskStore>]) -> Vec<Option<DiskInfo>
let opts = &DiskInfoOptions::default(); let opts = &DiskInfoOptions::default();
let mut res = vec![None; disks.len()]; let mut res = vec![None; disks.len()];
for (idx, disk_op) in disks.iter().enumerate() { for (idx, disk_op) in disks.iter().enumerate() {
if let Some(disk) = disk_op { if let Some(disk) = disk_op
if let Ok(info) = disk.disk_info(opts).await { && let Ok(info) = disk.disk_info(opts).await
{
res[idx] = Some(info); res[idx] = Some(info);
} }
} }
}
res res
} }
+38 -43
View File
@@ -144,11 +144,11 @@ impl GetObjectReader {
) -> Result<(Self, usize, i64)> { ) -> Result<(Self, usize, i64)> {
let mut rs = rs; let mut rs = rs;
if let Some(part_number) = opts.part_number { if let Some(part_number) = opts.part_number
if rs.is_none() { && rs.is_none()
{
rs = HTTPRangeSpec::from_object_info(oi, part_number); rs = HTTPRangeSpec::from_object_info(oi, part_number);
} }
}
// TODO:Encrypted // TODO:Encrypted
@@ -462,33 +462,31 @@ impl ObjectOptions {
pub fn precondition_check(&self, obj_info: &ObjectInfo) -> Result<()> { pub fn precondition_check(&self, obj_info: &ObjectInfo) -> Result<()> {
let has_valid_mod_time = obj_info.mod_time.is_some_and(|t| t != OffsetDateTime::UNIX_EPOCH); let has_valid_mod_time = obj_info.mod_time.is_some_and(|t| t != OffsetDateTime::UNIX_EPOCH);
if let Some(part_number) = self.part_number { if let Some(part_number) = self.part_number
if part_number > 1 && !obj_info.parts.is_empty() { && part_number > 1
&& !obj_info.parts.is_empty()
{
let part_found = obj_info.parts.iter().any(|pi| pi.number == part_number); let part_found = obj_info.parts.iter().any(|pi| pi.number == part_number);
if !part_found { if !part_found {
return Err(Error::InvalidPartNumber(part_number)); return Err(Error::InvalidPartNumber(part_number));
} }
} }
}
if let Some(pre) = &self.http_preconditions { if let Some(pre) = &self.http_preconditions {
if let Some(if_none_match) = &pre.if_none_match { if let Some(if_none_match) = &pre.if_none_match
if let Some(etag) = &obj_info.etag { && let Some(etag) = &obj_info.etag
if is_etag_equal(etag, if_none_match) { && is_etag_equal(etag, if_none_match)
{
return Err(Error::NotModified); return Err(Error::NotModified);
} }
}
}
if has_valid_mod_time { if has_valid_mod_time
if let Some(if_modified_since) = &pre.if_modified_since { && let Some(if_modified_since) = &pre.if_modified_since
if let Some(mod_time) = &obj_info.mod_time { && let Some(mod_time) = &obj_info.mod_time
if !is_modified_since(mod_time, if_modified_since) { && !is_modified_since(mod_time, if_modified_since)
{
return Err(Error::NotModified); return Err(Error::NotModified);
} }
}
}
}
if let Some(if_match) = &pre.if_match { if let Some(if_match) = &pre.if_match {
if let Some(etag) = &obj_info.etag { if let Some(etag) = &obj_info.etag {
@@ -499,16 +497,15 @@ impl ObjectOptions {
return Err(Error::PreconditionFailed); return Err(Error::PreconditionFailed);
} }
} }
if has_valid_mod_time && pre.if_match.is_none() { if has_valid_mod_time
if let Some(if_unmodified_since) = &pre.if_unmodified_since { && pre.if_match.is_none()
if let Some(mod_time) = &obj_info.mod_time { && let Some(if_unmodified_since) = &pre.if_unmodified_since
if is_modified_since(mod_time, if_unmodified_since) { && let Some(mod_time) = &obj_info.mod_time
&& is_modified_since(mod_time, if_unmodified_since)
{
return Err(Error::PreconditionFailed); return Err(Error::PreconditionFailed);
} }
} }
}
}
}
Ok(()) Ok(())
} }
@@ -698,13 +695,13 @@ impl ObjectInfo {
} }
if self.is_compressed() { if self.is_compressed() {
if let Some(size_str) = self.user_defined.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size")) { if let Some(size_str) = self.user_defined.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size"))
if !size_str.is_empty() { && !size_str.is_empty()
{
// Todo: deal with error // Todo: deal with error
let size = size_str.parse::<i64>().map_err(|e| std::io::Error::other(e.to_string()))?; let size = size_str.parse::<i64>().map_err(|e| std::io::Error::other(e.to_string()))?;
return Ok(size); return Ok(size);
} }
}
let mut actual_size = 0; let mut actual_size = 0;
self.parts.iter().for_each(|part| { self.parts.iter().for_each(|part| {
actual_size += part.actual_size; actual_size += part.actual_size;
@@ -881,16 +878,17 @@ impl ObjectInfo {
continue; continue;
} }
if entry.is_dir() { if entry.is_dir()
if let Some(delimiter) = &delimiter { && let Some(delimiter) = &delimiter
if let Some(idx) = { && let Some(idx) = {
let remaining = if entry.name.starts_with(prefix) { let remaining = if entry.name.starts_with(prefix) {
&entry.name[prefix.len()..] &entry.name[prefix.len()..]
} else { } else {
entry.name.as_str() entry.name.as_str()
}; };
remaining.find(delimiter.as_str()) remaining.find(delimiter.as_str())
} { }
{
let idx = prefix.len() + idx + delimiter.len(); let idx = prefix.len() + idx + delimiter.len();
if let Some(curr_prefix) = entry.name.get(0..idx) { if let Some(curr_prefix) = entry.name.get(0..idx) {
if curr_prefix == prev_prefix { if curr_prefix == prev_prefix {
@@ -908,8 +906,6 @@ impl ObjectInfo {
} }
} }
} }
}
}
objects objects
} }
@@ -966,16 +962,17 @@ impl ObjectInfo {
continue; continue;
} }
if entry.is_dir() { if entry.is_dir()
if let Some(delimiter) = &delimiter { && let Some(delimiter) = &delimiter
if let Some(idx) = { && let Some(idx) = {
let remaining = if entry.name.starts_with(prefix) { let remaining = if entry.name.starts_with(prefix) {
&entry.name[prefix.len()..] &entry.name[prefix.len()..]
} else { } else {
entry.name.as_str() entry.name.as_str()
}; };
remaining.find(delimiter.as_str()) remaining.find(delimiter.as_str())
} { }
{
let idx = prefix.len() + idx + delimiter.len(); let idx = prefix.len() + idx + delimiter.len();
if let Some(curr_prefix) = entry.name.get(0..idx) { if let Some(curr_prefix) = entry.name.get(0..idx) {
if curr_prefix == prev_prefix { if curr_prefix == prev_prefix {
@@ -993,8 +990,6 @@ impl ObjectInfo {
} }
} }
} }
}
}
objects objects
} }
@@ -1026,11 +1021,11 @@ impl ObjectInfo {
} }
pub fn decrypt_checksums(&self, part: usize, _headers: &HeaderMap) -> Result<(HashMap<String, String>, bool)> { pub fn decrypt_checksums(&self, part: usize, _headers: &HeaderMap) -> Result<(HashMap<String, String>, bool)> {
if part > 0 { if part > 0
if let Some(checksums) = self.parts.iter().find(|p| p.number == part).and_then(|p| p.checksums.clone()) { && let Some(checksums) = self.parts.iter().find(|p| p.number == part).and_then(|p| p.checksums.clone())
{
return Ok((checksums, true)); return Ok((checksums, true));
} }
}
// TODO: decrypt checksums // TODO: decrypt checksums
+32 -30
View File
@@ -302,11 +302,11 @@ impl ECStore {
..Default::default() ..Default::default()
}); });
if let Some(err) = list_result.err.clone() { if let Some(err) = list_result.err.clone()
if err != rustfs_filemeta::Error::Unexpected { && err != rustfs_filemeta::Error::Unexpected
{
return Err(to_object_err(err.into(), vec![bucket, prefix])); return Err(to_object_err(err.into(), vec![bucket, prefix]));
} }
}
if let Some(result) = list_result.entries.as_mut() { if let Some(result) = list_result.entries.as_mut() {
result.forward_past(opts.marker); result.forward_past(opts.marker);
@@ -418,11 +418,11 @@ impl ECStore {
}, },
}; };
if let Some(err) = list_result.err.clone() { if let Some(err) = list_result.err.clone()
if err != rustfs_filemeta::Error::Unexpected { && err != rustfs_filemeta::Error::Unexpected
{
return Err(to_object_err(err.into(), vec![bucket, prefix])); return Err(to_object_err(err.into(), vec![bucket, prefix]));
} }
}
if let Some(result) = list_result.entries.as_mut() { if let Some(result) = list_result.entries.as_mut() {
result.forward_past(opts.marker); result.forward_past(opts.marker);
@@ -509,11 +509,12 @@ impl ECStore {
let mut o = o.clone(); let mut o = o.clone();
o.marker = o.marker.filter(|v| v >= &o.prefix); o.marker = o.marker.filter(|v| v >= &o.prefix);
if let Some(marker) = &o.marker { if let Some(marker) = &o.marker
if !o.prefix.is_empty() && !marker.starts_with(&o.prefix) { && !o.prefix.is_empty()
&& !marker.starts_with(&o.prefix)
{
return Err(Error::Unexpected); return Err(Error::Unexpected);
} }
}
if o.limit == 0 { if o.limit == 0 {
return Err(Error::Unexpected); return Err(Error::Unexpected);
@@ -817,12 +818,12 @@ impl ECStore {
let value = tx2.clone(); let value = tx2.clone();
let resolver = resolver.clone(); let resolver = resolver.clone();
async move { async move {
if let Some(entry) = entries.resolve(resolver) { if let Some(entry) = entries.resolve(resolver)
if let Err(err) = value.send(entry).await { && let Err(err) = value.send(entry).await
{
error!("list_path send fail {:?}", err); error!("list_path send fail {:?}", err);
} }
} }
}
}) })
})), })),
finished: None, finished: None,
@@ -986,21 +987,22 @@ async fn gather_results(
continue; continue;
} }
if let Some(marker) = &opts.marker { if let Some(marker) = &opts.marker
if &entry.name < marker { && &entry.name < marker
{
continue; continue;
} }
}
if !entry.name.starts_with(&opts.prefix) { if !entry.name.starts_with(&opts.prefix) {
continue; continue;
} }
if let Some(separator) = &opts.separator { if let Some(separator) = &opts.separator
if !opts.recursive && !entry.is_in_dir(&opts.prefix, separator) { && !opts.recursive
&& !entry.is_in_dir(&opts.prefix, separator)
{
continue; continue;
} }
}
if !opts.incl_deleted && entry.is_object() && entry.is_latest_delete_marker() && !entry.is_object_dir() { if !opts.incl_deleted && entry.is_object() && entry.is_latest_delete_marker() && !entry.is_object_dir() {
continue; continue;
@@ -1200,29 +1202,29 @@ async fn merge_entry_channels(
} }
} }
if let Some(xl) = has_xl.as_mut() { if let Some(xl) = has_xl.as_mut()
if !versions.is_empty() { && !versions.is_empty()
{
xl.versions = merge_file_meta_versions(read_quorum, true, 0, &versions); xl.versions = merge_file_meta_versions(read_quorum, true, 0, &versions);
if let Ok(meta) = xl.marshal_msg() { if let Ok(meta) = xl.marshal_msg()
if let Some(b) = best.as_mut() { && let Some(b) = best.as_mut()
{
b.metadata = meta; b.metadata = meta;
b.cached = Some(xl.clone()); b.cached = Some(xl.clone());
} }
} }
} }
}
}
to_merge.clear(); to_merge.clear();
} }
if let Some(best_entry) = &best { if let Some(best_entry) = &best
if best_entry.name > last { && best_entry.name > last
{
out_channel.send(best_entry.clone()).await.map_err(Error::other)?; out_channel.send(best_entry.clone()).await.map_err(Error::other)?;
last = best_entry.name.clone(); last = best_entry.name.clone();
} }
}
select_from(&mut in_channels, best_idx, &mut top, &mut n_done).await?; select_from(&mut in_channels, best_idx, &mut top, &mut n_done).await?;
} }
@@ -1307,12 +1309,12 @@ impl SetDisks {
let value = tx2.clone(); let value = tx2.clone();
let resolver = resolver.clone(); let resolver = resolver.clone();
async move { async move {
if let Some(entry) = entries.resolve(resolver) { if let Some(entry) = entries.resolve(resolver)
if let Err(err) = value.send(entry).await { && let Err(err) = value.send(entry).await
{
error!("list_path send fail {:?}", err); error!("list_path send fail {:?}", err);
} }
} }
}
}) })
})), })),
finished: None, finished: None,
+3 -3
View File
@@ -635,10 +635,10 @@ fn parse_restore_obj_status(restore_hdr: &str) -> Result<RestoreStatus> {
} }
pub fn is_restored_object_on_disk(meta: &HashMap<String, String>) -> bool { pub fn is_restored_object_on_disk(meta: &HashMap<String, String>) -> bool {
if let Some(restore_hdr) = meta.get(X_AMZ_RESTORE.as_str()) { if let Some(restore_hdr) = meta.get(X_AMZ_RESTORE.as_str())
if let Ok(restore_status) = parse_restore_obj_status(restore_hdr) { && let Ok(restore_status) = parse_restore_obj_status(restore_hdr)
{
return restore_status.on_disk(); return restore_status.on_disk();
} }
}
false false
} }
+14 -16
View File
@@ -575,15 +575,14 @@ impl FileMeta {
let mod_time = version.get_mod_time(); let mod_time = version.get_mod_time();
for (idx, exist) in self.versions.iter().enumerate() { for (idx, exist) in self.versions.iter().enumerate() {
if let Some(ref ex_mt) = exist.header.mod_time { if let Some(ref ex_mt) = exist.header.mod_time
if let Some(ref in_md) = mod_time { && let Some(ref in_md) = mod_time
if ex_mt <= in_md { && ex_mt <= in_md
{
self.versions.insert(idx, FileMetaShallowVersion::try_from(version)?); self.versions.insert(idx, FileMetaShallowVersion::try_from(version)?);
return Ok(()); return Ok(());
} }
} }
}
}
Err(Error::other("add_version failed")) Err(Error::other("add_version failed"))
// if !ver.valid() { // if !ver.valid() {
@@ -657,8 +656,9 @@ impl FileMeta {
} }
if fi.deleted { if fi.deleted {
if !fi.delete_marker_replication_status().is_empty() { if !fi.delete_marker_replication_status().is_empty()
if let Some(delete_marker) = ventry.delete_marker.as_mut() { && let Some(delete_marker) = ventry.delete_marker.as_mut()
{
if fi.delete_marker_replication_status() == ReplicationStatusType::Replica { if fi.delete_marker_replication_status() == ReplicationStatusType::Replica {
delete_marker.meta_sys.insert( delete_marker.meta_sys.insert(
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-status"), format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-status"),
@@ -700,10 +700,10 @@ impl FileMeta {
); );
} }
} }
}
if !fi.version_purge_status().is_empty() { if !fi.version_purge_status().is_empty()
if let Some(delete_marker) = ventry.delete_marker.as_mut() { && let Some(delete_marker) = ventry.delete_marker.as_mut()
{
delete_marker.meta_sys.insert( delete_marker.meta_sys.insert(
VERSION_PURGE_STATUS_KEY.to_string(), VERSION_PURGE_STATUS_KEY.to_string(),
fi.replication_state_internal fi.replication_state_internal
@@ -714,7 +714,6 @@ impl FileMeta {
.to_vec(), .to_vec(),
); );
} }
}
if let Some(delete_marker) = ventry.delete_marker.as_mut() { if let Some(delete_marker) = ventry.delete_marker.as_mut() {
for (k, v) in fi for (k, v) in fi
@@ -1917,8 +1916,8 @@ impl MetaObject {
if let Some(status) = self if let Some(status) = self
.meta_sys .meta_sys
.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}")) .get(&format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}"))
&& *status == TRANSITION_COMPLETE.as_bytes().to_vec()
{ {
if *status == TRANSITION_COMPLETE.as_bytes().to_vec() {
let vid = Uuid::parse_str(&fi.tier_free_version_id()); let vid = Uuid::parse_str(&fi.tier_free_version_id());
if let Err(err) = vid { if let Err(err) = vid {
panic!("Invalid Tier Object delete marker versionId {} {}", fi.tier_free_version_id(), err); panic!("Invalid Tier Object delete marker versionId {} {}", fi.tier_free_version_id(), err);
@@ -1953,7 +1952,6 @@ impl MetaObject {
} }
return (free_entry, true); return (free_entry, true);
} }
}
(FileMetaVersion::default(), false) (FileMetaVersion::default(), false)
} }
} }
@@ -3568,8 +3566,9 @@ impl FileMeta {
match version.header.version_type { match version.header.version_type {
VersionType::Object => { VersionType::Object => {
stats.object_versions += 1; stats.object_versions += 1;
if let Ok(ver) = FileMetaVersion::try_from(version.meta.as_slice()) { if let Ok(ver) = FileMetaVersion::try_from(version.meta.as_slice())
if let Some(obj) = &ver.object { && let Some(obj) = &ver.object
{
stats.total_size += obj.size; stats.total_size += obj.size;
if obj.uses_data_dir() { if obj.uses_data_dir() {
stats.versions_with_data_dir += 1; stats.versions_with_data_dir += 1;
@@ -3579,7 +3578,6 @@ impl FileMeta {
} }
} }
} }
}
VersionType::Delete => stats.delete_markers += 1, VersionType::Delete => stats.delete_markers += 1,
VersionType::Legacy => stats.legacy_versions += 1, VersionType::Legacy => stats.legacy_versions += 1,
VersionType::Invalid => stats.invalid_versions += 1, VersionType::Invalid => stats.invalid_versions += 1,
+12 -12
View File
@@ -442,13 +442,13 @@ impl MetaCacheEntriesSorted {
} }
pub fn forward_past(&mut self, marker: Option<String>) { pub fn forward_past(&mut self, marker: Option<String>) {
if let Some(val) = marker { if let Some(val) = marker
if let Some(idx) = self.o.0.iter().flatten().position(|v| v.name > val) { && let Some(idx) = self.o.0.iter().flatten().position(|v| v.name > val)
{
self.o.0 = self.o.0.split_off(idx); self.o.0 = self.o.0.split_off(idx);
} }
} }
} }
}
const METACACHE_STREAM_VERSION: u8 = 2; const METACACHE_STREAM_VERSION: u8 = 2;
@@ -788,14 +788,16 @@ impl<T: Clone + Debug + Send + 'static> Cache<T> {
.duration_since(UNIX_EPOCH) .duration_since(UNIX_EPOCH)
.expect("Time went backwards") .expect("Time went backwards")
.as_secs(); .as_secs();
if now - self.last_update_ms.load(AtomicOrdering::SeqCst) < self.ttl.as_secs() { if now - self.last_update_ms.load(AtomicOrdering::SeqCst) < self.ttl.as_secs()
if let Some(v) = v { && let Some(v) = v
{
return Ok(v); return Ok(v);
} }
}
if self.opts.no_wait && now - self.last_update_ms.load(AtomicOrdering::SeqCst) < self.ttl.as_secs() * 2 { if self.opts.no_wait
if let Some(value) = v { && now - self.last_update_ms.load(AtomicOrdering::SeqCst) < self.ttl.as_secs() * 2
&& let Some(value) = v
{
if self.updating.try_lock().is_ok() { if self.updating.try_lock().is_ok() {
let this = Arc::clone(&self); let this = Arc::clone(&self);
spawn(async move { spawn(async move {
@@ -804,18 +806,16 @@ impl<T: Clone + Debug + Send + 'static> Cache<T> {
} }
return Ok(value); return Ok(value);
} }
}
let _ = self.updating.lock().await; let _ = self.updating.lock().await;
if let (Ok(duration), Some(value)) = ( if let (Ok(duration), Some(value)) = (
SystemTime::now().duration_since(UNIX_EPOCH + Duration::from_secs(self.last_update_ms.load(AtomicOrdering::SeqCst))), SystemTime::now().duration_since(UNIX_EPOCH + Duration::from_secs(self.last_update_ms.load(AtomicOrdering::SeqCst))),
v, v,
) { ) && duration < self.ttl
if duration < self.ttl { {
return Ok(value); return Ok(value);
} }
}
match self.update().await { match self.update().await {
Ok(_) => { Ok(_) => {
+3 -5
View File
@@ -270,15 +270,13 @@ impl ReplicationState {
return repl_status; return repl_status;
} }
if repl_status == ReplicationStatusType::Completed { if repl_status == ReplicationStatusType::Completed
if let (Some(replica_timestamp), Some(replication_timestamp)) = && let (Some(replica_timestamp), Some(replication_timestamp)) =
(self.replica_timestamp, self.replication_timestamp) (self.replica_timestamp, self.replication_timestamp)
&& replica_timestamp > replication_timestamp
{ {
if replica_timestamp > replication_timestamp {
return self.replica_status.clone(); return self.replica_status.clone();
} }
}
}
return repl_status; return repl_status;
} }
+39 -39
View File
@@ -246,8 +246,9 @@ where
} }
let sts_user = has_sts_user.map(|sts| sts.credentials.access_key.clone()); let sts_user = has_sts_user.map(|sts| sts.credentials.access_key.clone());
if let Some(ref sts) = sts_user { if let Some(ref sts) = sts_user
if let Some(plc) = sts_policy_map.get(sts) { && let Some(plc) = sts_policy_map.get(sts)
{
for p in plc.to_slice().iter() { for p in plc.to_slice().iter() {
if !policy_docs_map.contains_key(p) { if !policy_docs_map.contains_key(p) {
let _ = self.api.load_policy_doc(p, &mut policy_docs_map).await; let _ = self.api.load_policy_doc(p, &mut policy_docs_map).await;
@@ -255,7 +256,6 @@ where
} }
} }
} }
}
if let Some(plc) = user_policy_map.get(access_key) { if let Some(plc) = user_policy_map.get(access_key) {
for p in plc.to_slice().iter() { for p in plc.to_slice().iter() {
@@ -635,11 +635,11 @@ where
} }
let users = self.cache.users.load(); let users = self.cache.users.load();
if let Some(x) = users.get(&cred.access_key) { if let Some(x) = users.get(&cred.access_key)
if x.credentials.is_service_account() { && x.credentials.is_service_account()
{
return Err(Error::IAMActionNotAllowed); return Err(Error::IAMActionNotAllowed);
} }
}
let u = UserIdentity::new(cred); let u = UserIdentity::new(cred);
@@ -789,11 +789,11 @@ where
if !policy_present { if !policy_present {
let mut m = HashMap::new(); let mut m = HashMap::new();
if let Err(err) = self.api.load_mapped_policy(name, UserType::Reg, true, &mut m).await { if let Err(err) = self.api.load_mapped_policy(name, UserType::Reg, true, &mut m).await
if !is_err_no_such_policy(&err) { && !is_err_no_such_policy(&err)
{
return Err(err); return Err(err);
} }
}
if let Some(p) = m.get(name) { if let Some(p) = m.get(name) {
Cache::add_or_update(&self.cache.group_policies, name, p, OffsetDateTime::now_utc()); Cache::add_or_update(&self.cache.group_policies, name, p, OffsetDateTime::now_utc());
return Ok((p.to_slice(), p.update_at)); return Ok((p.to_slice(), p.update_at));
@@ -815,11 +815,11 @@ where
Some(p) => p.clone(), Some(p) => p.clone(),
None => { None => {
let mut m = HashMap::new(); let mut m = HashMap::new();
if let Err(err) = self.api.load_mapped_policy(name, UserType::Reg, false, &mut m).await { if let Err(err) = self.api.load_mapped_policy(name, UserType::Reg, false, &mut m).await
if !is_err_no_such_policy(&err) { && !is_err_no_such_policy(&err)
{
return Err(err); return Err(err);
} }
}
if let Some(p) = m.get(name) { if let Some(p) = m.get(name) {
Cache::add_or_update(&self.cache.user_policies, name, p, OffsetDateTime::now_utc()); Cache::add_or_update(&self.cache.user_policies, name, p, OffsetDateTime::now_utc());
p.clone() p.clone()
@@ -828,11 +828,11 @@ where
Some(p) => p.clone(), Some(p) => p.clone(),
None => { None => {
let mut m = HashMap::new(); let mut m = HashMap::new();
if let Err(err) = self.api.load_mapped_policy(name, UserType::Sts, false, &mut m).await { if let Err(err) = self.api.load_mapped_policy(name, UserType::Sts, false, &mut m).await
if !is_err_no_such_policy(&err) { && !is_err_no_such_policy(&err)
{
return Err(err); return Err(err);
} }
}
if let Some(p) = m.get(name) { if let Some(p) = m.get(name) {
Cache::add_or_update(&self.cache.sts_policies, name, p, OffsetDateTime::now_utc()); Cache::add_or_update(&self.cache.sts_policies, name, p, OffsetDateTime::now_utc());
p.clone() p.clone()
@@ -864,11 +864,11 @@ where
Some(p) => p.clone(), Some(p) => p.clone(),
None => { None => {
let mut m = HashMap::new(); let mut m = HashMap::new();
if let Err(err) = self.api.load_mapped_policy(group, UserType::Reg, true, &mut m).await { if let Err(err) = self.api.load_mapped_policy(group, UserType::Reg, true, &mut m).await
if !is_err_no_such_policy(&err) { && !is_err_no_such_policy(&err)
{
return Err(err); return Err(err);
} }
}
if let Some(p) = m.get(group) { if let Some(p) = m.get(group) {
Cache::add_or_update(&self.cache.group_policies, group, p, OffsetDateTime::now_utc()); Cache::add_or_update(&self.cache.group_policies, group, p, OffsetDateTime::now_utc());
p.clone() p.clone()
@@ -910,11 +910,11 @@ where
Some(p) => p.clone(), Some(p) => p.clone(),
None => { None => {
let mut m = HashMap::new(); let mut m = HashMap::new();
if let Err(err) = self.api.load_mapped_policy(group, UserType::Reg, true, &mut m).await { if let Err(err) = self.api.load_mapped_policy(group, UserType::Reg, true, &mut m).await
if !is_err_no_such_policy(&err) { && !is_err_no_such_policy(&err)
{
return Err(err); return Err(err);
} }
}
if let Some(p) = m.get(group) { if let Some(p) = m.get(group) {
Cache::add_or_update(&self.cache.group_policies, group, p, OffsetDateTime::now_utc()); Cache::add_or_update(&self.cache.group_policies, group, p, OffsetDateTime::now_utc());
p.clone() p.clone()
@@ -937,11 +937,11 @@ where
} }
if policy.is_empty() { if policy.is_empty() {
if let Err(err) = self.api.delete_mapped_policy(name, user_type, is_group).await { if let Err(err) = self.api.delete_mapped_policy(name, user_type, is_group).await
if !is_err_no_such_policy(&err) { && !is_err_no_such_policy(&err)
{
return Err(err); return Err(err);
} }
}
if is_group { if is_group {
Cache::delete(&self.cache.group_policies, name, OffsetDateTime::now_utc()); Cache::delete(&self.cache.group_policies, name, OffsetDateTime::now_utc());
@@ -1220,11 +1220,11 @@ where
Cache::delete(&self.cache.user_policies, access_key, OffsetDateTime::now_utc()); Cache::delete(&self.cache.user_policies, access_key, OffsetDateTime::now_utc());
if let Err(err) = self.api.delete_user_identity(access_key, utype).await { if let Err(err) = self.api.delete_user_identity(access_key, utype).await
if !is_err_no_such_user(&err) { && !is_err_no_such_user(&err)
{
return Err(err); return Err(err);
} }
}
if utype == UserType::Sts { if utype == UserType::Sts {
Cache::delete(&self.cache.sts_accounts, access_key, OffsetDateTime::now_utc()); Cache::delete(&self.cache.sts_accounts, access_key, OffsetDateTime::now_utc());
@@ -1532,17 +1532,17 @@ where
} }
if members.is_empty() { if members.is_empty() {
if let Err(err) = self.api.delete_mapped_policy(group, UserType::Reg, true).await { if let Err(err) = self.api.delete_mapped_policy(group, UserType::Reg, true).await
if !is_err_no_such_policy(&err) { && !is_err_no_such_policy(&err)
{
return Err(err); return Err(err);
} }
}
if let Err(err) = self.api.delete_group_info(group).await { if let Err(err) = self.api.delete_group_info(group).await
if !is_err_no_such_group(&err) { && !is_err_no_such_group(&err)
{
return Err(err); return Err(err);
} }
}
Cache::delete(&self.cache.groups, group, OffsetDateTime::now_utc()); Cache::delete(&self.cache.groups, group, OffsetDateTime::now_utc());
Cache::delete(&self.cache.group_policies, group, OffsetDateTime::now_utc()); Cache::delete(&self.cache.group_policies, group, OffsetDateTime::now_utc());
@@ -1691,13 +1691,13 @@ where
let member_of = self.cache.user_group_memberships.load(); let member_of = self.cache.user_group_memberships.load();
if let Some(m) = member_of.get(name) { if let Some(m) = member_of.get(name) {
for group in m.iter() { for group in m.iter() {
if let Err(err) = self.remove_members_from_group(group, vec![name.to_string()], true).await { if let Err(err) = self.remove_members_from_group(group, vec![name.to_string()], true).await
if !is_err_no_such_group(&err) { && !is_err_no_such_group(&err)
{
return Err(err); return Err(err);
} }
} }
} }
}
if user_type == UserType::Reg { if user_type == UserType::Reg {
let users_cache = self.cache.users.load(); let users_cache = self.cache.users.load();
@@ -1859,13 +1859,13 @@ fn filter_policies(cache: &Cache, policy_name: &str, bucket_name: &str) -> (Stri
continue; continue;
} }
if let Some(p) = cache.policy_docs.load().get(&policy) { if let Some(p) = cache.policy_docs.load().get(&policy)
if bucket_name.is_empty() || pollster::block_on(p.policy.match_resource(bucket_name)) { && (bucket_name.is_empty() || pollster::block_on(p.policy.match_resource(bucket_name)))
{
policies.push(policy); policies.push(policy);
to_merge.push(p.policy.clone()); to_merge.push(p.policy.clone());
} }
} }
}
(policies.join(","), Policy::merge_policies(to_merge)) (policies.join(","), Policy::merge_policies(to_merge))
} }
+10 -11
View File
@@ -633,13 +633,13 @@ impl Store for ObjectStore {
if let Some(item) = v.item { if let Some(item) = v.item {
let name = rustfs_utils::path::dir(&item); let name = rustfs_utils::path::dir(&item);
if let Err(err) = self.load_group(&name, m).await { if let Err(err) = self.load_group(&name, m).await
if !is_err_no_such_group(&err) { && !is_err_no_such_group(&err)
{
return Err(err); return Err(err);
} }
} }
} }
}
let _ = ctx.cancel(); let _ = ctx.cancel();
Ok(()) Ok(())
} }
@@ -936,10 +936,10 @@ impl Store for ObjectStore {
let name = item.trim_end_matches(".json"); let name = item.trim_end_matches(".json");
info!("load group policy: {}", name); info!("load group policy: {}", name);
if let Err(err) = self.load_mapped_policy(name, UserType::Reg, true, &mut items_cache).await { if let Err(err) = self.load_mapped_policy(name, UserType::Reg, true, &mut items_cache).await
if !is_err_no_such_policy(&err) { && !is_err_no_such_policy(&err)
{
return Err(Error::other(format!("load group policy failed: {err}"))); return Err(Error::other(format!("load group policy failed: {err}")));
}
}; };
} }
@@ -955,10 +955,10 @@ impl Store for ObjectStore {
for item in item_name_list.iter() { for item in item_name_list.iter() {
let name = rustfs_utils::path::dir(item); let name = rustfs_utils::path::dir(item);
info!("load svc user: {}", name); info!("load svc user: {}", name);
if let Err(err) = self.load_user(&name, UserType::Svc, &mut items_cache).await { if let Err(err) = self.load_user(&name, UserType::Svc, &mut items_cache).await
if !is_err_no_such_user(&err) { && !is_err_no_such_user(&err)
{
return Err(Error::other(format!("load svc user failed: {err}"))); return Err(Error::other(format!("load svc user failed: {err}")));
}
}; };
} }
@@ -969,13 +969,12 @@ impl Store for ObjectStore {
if let Err(err) = self if let Err(err) = self
.load_mapped_policy(&parent, UserType::Sts, false, &mut sts_policies_cache) .load_mapped_policy(&parent, UserType::Sts, false, &mut sts_policies_cache)
.await .await
&& !is_err_no_such_policy(&err)
{ {
if !is_err_no_such_policy(&err) {
return Err(Error::other(format!("load_mapped_policy failed: {err}"))); return Err(Error::other(format!("load_mapped_policy failed: {err}")));
} }
} }
} }
}
// Merge items_cache to user_items_cache // Merge items_cache to user_items_cache
user_items_cache.extend(items_cache); user_items_cache.extend(items_cache);
+27 -27
View File
@@ -203,8 +203,9 @@ impl<T: Store> IamSys<T> {
pub async fn set_policy(&self, name: &str, policy: Policy) -> Result<OffsetDateTime> { pub async fn set_policy(&self, name: &str, policy: Policy) -> Result<OffsetDateTime> {
let updated_at = self.store.set_policy(name, policy).await?; let updated_at = self.store.set_policy(name, policy).await?;
if !self.has_watcher() { if !self.has_watcher()
if let Some(notification_sys) = get_global_notification_sys() { && let Some(notification_sys) = get_global_notification_sys()
{
let resp = notification_sys.load_policy(name).await; let resp = notification_sys.load_policy(name).await;
for r in resp { for r in resp {
if let Some(err) = r.err { if let Some(err) = r.err {
@@ -212,7 +213,6 @@ impl<T: Store> IamSys<T> {
} }
} }
} }
}
Ok(updated_at) Ok(updated_at)
} }
@@ -232,8 +232,10 @@ impl<T: Store> IamSys<T> {
pub async fn delete_user(&self, name: &str, notify: bool) -> Result<()> { pub async fn delete_user(&self, name: &str, notify: bool) -> Result<()> {
self.store.delete_user(name, UserType::Reg).await?; self.store.delete_user(name, UserType::Reg).await?;
if notify && !self.has_watcher() { if notify
if let Some(notification_sys) = get_global_notification_sys() { && !self.has_watcher()
&& let Some(notification_sys) = get_global_notification_sys()
{
let resp = notification_sys.delete_user(name).await; let resp = notification_sys.delete_user(name).await;
for r in resp { for r in resp {
if let Some(err) = r.err { if let Some(err) = r.err {
@@ -241,7 +243,6 @@ impl<T: Store> IamSys<T> {
} }
} }
} }
}
Ok(()) Ok(())
} }
@@ -476,14 +477,13 @@ impl<T: Store> IamSys<T> {
let op_pt = claims.get(&iam_policy_claim_name_sa()); let op_pt = claims.get(&iam_policy_claim_name_sa());
let op_sp = claims.get(SESSION_POLICY_NAME); let op_sp = claims.get(SESSION_POLICY_NAME);
if let (Some(pt), Some(sp)) = (op_pt, op_sp) { if let (Some(pt), Some(sp)) = (op_pt, op_sp)
if pt == EMBEDDED_POLICY_TYPE { && pt == EMBEDDED_POLICY_TYPE
let policy = serde_json::from_slice( {
&base64_simd::URL_SAFE_NO_PAD.decode_to_vec(sp.as_str().unwrap_or_default().as_bytes())?, let policy =
)?; serde_json::from_slice(&base64_simd::URL_SAFE_NO_PAD.decode_to_vec(sp.as_str().unwrap_or_default().as_bytes())?)?;
return Ok((sa, Some(policy))); return Ok((sa, Some(policy)));
} }
}
Ok((sa, None)) Ok((sa, None))
} }
@@ -537,14 +537,13 @@ impl<T: Store> IamSys<T> {
let op_pt = claims.get(&iam_policy_claim_name_sa()); let op_pt = claims.get(&iam_policy_claim_name_sa());
let op_sp = claims.get(SESSION_POLICY_NAME); let op_sp = claims.get(SESSION_POLICY_NAME);
if let (Some(pt), Some(sp)) = (op_pt, op_sp) { if let (Some(pt), Some(sp)) = (op_pt, op_sp)
if pt == EMBEDDED_POLICY_TYPE { && pt == EMBEDDED_POLICY_TYPE
let policy = serde_json::from_slice( {
&base64_simd::URL_SAFE_NO_PAD.decode_to_vec(sp.as_str().unwrap_or_default().as_bytes())?, let policy =
)?; serde_json::from_slice(&base64_simd::URL_SAFE_NO_PAD.decode_to_vec(sp.as_str().unwrap_or_default().as_bytes())?)?;
return Ok((sa, Some(policy))); return Ok((sa, Some(policy)));
} }
}
Ok((sa, None)) Ok((sa, None))
} }
@@ -572,8 +571,10 @@ impl<T: Store> IamSys<T> {
self.store.delete_user(access_key, UserType::Svc).await?; self.store.delete_user(access_key, UserType::Svc).await?;
if notify && !self.has_watcher() { if notify
if let Some(notification_sys) = get_global_notification_sys() { && !self.has_watcher()
&& let Some(notification_sys) = get_global_notification_sys()
{
let resp = notification_sys.delete_service_account(access_key).await; let resp = notification_sys.delete_service_account(access_key).await;
for r in resp { for r in resp {
if let Some(err) = r.err { if let Some(err) = r.err {
@@ -581,7 +582,6 @@ impl<T: Store> IamSys<T> {
} }
} }
} }
}
Ok(()) Ok(())
} }
@@ -651,11 +651,11 @@ impl<T: Store> IamSys<T> {
} }
pub async fn check_key(&self, access_key: &str) -> Result<(Option<UserIdentity>, bool)> { pub async fn check_key(&self, access_key: &str) -> Result<(Option<UserIdentity>, bool)> {
if let Some(sys_cred) = get_global_action_cred() { if let Some(sys_cred) = get_global_action_cred()
if sys_cred.access_key == access_key { && sys_cred.access_key == access_key
{
return Ok((Some(UserIdentity::new(sys_cred)), true)); return Ok((Some(UserIdentity::new(sys_cred)), true));
} }
}
match self.store.get_user(access_key).await { match self.store.get_user(access_key).await {
Some(res) => { Some(res) => {
@@ -725,8 +725,9 @@ impl<T: Store> IamSys<T> {
pub async fn policy_db_set(&self, name: &str, user_type: UserType, is_group: bool, policy: &str) -> Result<OffsetDateTime> { pub async fn policy_db_set(&self, name: &str, user_type: UserType, is_group: bool, policy: &str) -> Result<OffsetDateTime> {
let updated_at = self.store.policy_db_set(name, user_type, is_group, policy).await?; let updated_at = self.store.policy_db_set(name, user_type, is_group, policy).await?;
if !self.has_watcher() { if !self.has_watcher()
if let Some(notification_sys) = get_global_notification_sys() { && let Some(notification_sys) = get_global_notification_sys()
{
let resp = notification_sys.load_policy_mapping(name, user_type.to_u64(), is_group).await; let resp = notification_sys.load_policy_mapping(name, user_type.to_u64(), is_group).await;
for r in resp { for r in resp {
if let Some(err) = r.err { if let Some(err) = r.err {
@@ -734,7 +735,6 @@ impl<T: Store> IamSys<T> {
} }
} }
} }
}
Ok(updated_at) Ok(updated_at)
} }
+11 -13
View File
@@ -452,29 +452,27 @@ impl KmsClient for LocalKmsClient {
} }
let path = entry.path(); let path = entry.path();
if path.extension().is_some_and(|ext| ext == "key") { if path.extension().is_some_and(|ext| ext == "key")
if let Some(stem) = path.file_stem() { && let Some(stem) = path.file_stem()
if let Some(key_id) = stem.to_str() { && let Some(key_id) = stem.to_str()
if let Ok(key_info) = self.describe_key(key_id, None).await { && let Ok(key_info) = self.describe_key(key_id, None).await
{
// Apply filters // Apply filters
if let Some(ref status_filter) = request.status_filter { if let Some(ref status_filter) = request.status_filter
if &key_info.status != status_filter { && &key_info.status != status_filter
{
continue; continue;
} }
} if let Some(ref usage_filter) = request.usage_filter
if let Some(ref usage_filter) = request.usage_filter { && &key_info.usage != usage_filter
if &key_info.usage != usage_filter { {
continue; continue;
} }
}
keys.push(key_info); keys.push(key_info);
count += 1; count += 1;
} }
} }
}
}
}
Ok(ListKeysResponse { Ok(ListKeysResponse {
keys, keys,
+4 -5
View File
@@ -279,9 +279,10 @@ impl KmsConfig {
} }
// Validate TLS configuration if using HTTPS // Validate TLS configuration if using HTTPS
if config.address.starts_with("https://") { if config.address.starts_with("https://")
if let Some(ref tls) = config.tls { && let Some(ref tls) = config.tls
if !tls.skip_verify { && !tls.skip_verify
{
// In production, we should have proper TLS configuration // In production, we should have proper TLS configuration
if tls.ca_cert_path.is_none() && tls.client_cert_path.is_none() { if tls.ca_cert_path.is_none() && tls.client_cert_path.is_none() {
tracing::warn!("Using HTTPS without custom TLS configuration - relying on system CA"); tracing::warn!("Using HTTPS without custom TLS configuration - relying on system CA");
@@ -289,8 +290,6 @@ impl KmsConfig {
} }
} }
} }
}
}
// Validate cache configuration // Validate cache configuration
if self.enable_cache && self.cache_config.max_keys == 0 { if self.enable_cache && self.cache_config.max_keys == 0 {
+3 -3
View File
@@ -74,8 +74,9 @@ impl KmsManager {
// Check cache first if enabled // Check cache first if enabled
if self.config.enable_cache { if self.config.enable_cache {
let cache = self.cache.read().await; let cache = self.cache.read().await;
if let Some(cached_key) = cache.get_data_key(&request.key_id).await { if let Some(cached_key) = cache.get_data_key(&request.key_id).await
if cached_key.key_spec == request.key_spec { && cached_key.key_spec == request.key_spec
{
return Ok(GenerateDataKeyResponse { return Ok(GenerateDataKeyResponse {
key_id: request.key_id.clone(), key_id: request.key_id.clone(),
plaintext_key: cached_key.plaintext.clone(), plaintext_key: cached_key.plaintext.clone(),
@@ -83,7 +84,6 @@ impl KmsManager {
}); });
} }
} }
}
// Generate new data key from backend // Generate new data key from backend
let response = self.backend.generate_data_key(request).await?; let response = self.backend.generate_data_key(request).await?;
+3 -3
View File
@@ -104,8 +104,9 @@ mod tests {
let response = client.acquire_exclusive(&request).await; let response = client.acquire_exclusive(&request).await;
assert!(response.is_ok()); assert!(response.is_ok());
if let Ok(response) = response { if let Ok(response) = response
if response.success { && response.success
{
let lock_info = response.lock_info.unwrap(); let lock_info = response.lock_info.unwrap();
// Test status check // Test status check
@@ -120,4 +121,3 @@ mod tests {
} }
} }
} }
}
+3 -3
View File
@@ -953,12 +953,12 @@ mod tests {
// Wait for all operations to complete // Wait for all operations to complete
let mut successful_operations = 0; let mut successful_operations = 0;
for handle in handles { for handle in handles {
if let Ok(success) = handle.await { if let Ok(success) = handle.await
if success { && success
{
successful_operations += 1; successful_operations += 1;
} }
} }
}
// We expect most operations to succeed with the new timeouts and optimizations // We expect most operations to succeed with the new timeouts and optimizations
let total_operations = concurrent_queries + 5; let total_operations = concurrent_queries + 5;
+3 -3
View File
@@ -414,13 +414,13 @@ impl Default for FastObjectLockManager {
impl Drop for FastObjectLockManager { impl Drop for FastObjectLockManager {
fn drop(&mut self) { fn drop(&mut self) {
// Note: We can't use async in Drop, so we just abort the cleanup task // Note: We can't use async in Drop, so we just abort the cleanup task
if let Ok(handle_guard) = self.cleanup_handle.try_read() { if let Ok(handle_guard) = self.cleanup_handle.try_read()
if let Some(handle) = handle_guard.as_ref() { && let Some(handle) = handle_guard.as_ref()
{
handle.abort(); handle.abort();
} }
} }
} }
}
impl Clone for FastObjectLockManager { impl Clone for FastObjectLockManager {
fn clone(&self) -> Self { fn clone(&self) -> Self {
+6 -6
View File
@@ -69,11 +69,11 @@ impl LockShard {
/// Try fast path only (without fallback to slow path) /// Try fast path only (without fallback to slow path)
pub fn try_fast_path_only(&self, request: &ObjectLockRequest) -> bool { pub fn try_fast_path_only(&self, request: &ObjectLockRequest) -> bool {
// Early check to avoid unnecessary lock contention // Early check to avoid unnecessary lock contention
if let Some(state) = self.objects.read().get(&request.key) { if let Some(state) = self.objects.read().get(&request.key)
if !state.atomic_state.is_fast_path_available(request.mode) { && !state.atomic_state.is_fast_path_available(request.mode)
{
return false; return false;
} }
}
self.try_fast_path(request).is_some() self.try_fast_path(request).is_some()
} }
@@ -441,8 +441,9 @@ impl LockShard {
/// Get lock information for monitoring /// Get lock information for monitoring
pub fn get_lock_info(&self, key: &ObjectKey) -> Option<crate::fast_lock::types::ObjectLockInfo> { pub fn get_lock_info(&self, key: &ObjectKey) -> Option<crate::fast_lock::types::ObjectLockInfo> {
let objects = self.objects.read(); let objects = self.objects.read();
if let Some(state) = objects.get(key) { if let Some(state) = objects.get(key)
if let Some(mode) = state.current_mode() { && let Some(mode) = state.current_mode()
{
let (owner, acquired_at, lock_timeout) = match mode { let (owner, acquired_at, lock_timeout) = match mode {
LockMode::Exclusive => { LockMode::Exclusive => {
let current_owner = state.current_owner.read(); let current_owner = state.current_owner.read();
@@ -471,7 +472,6 @@ impl LockShard {
priority, priority,
}); });
} }
}
None None
} }
+3 -3
View File
@@ -165,12 +165,12 @@ impl NamespaceLock {
let mut successful_clients = Vec::new(); let mut successful_clients = Vec::new();
for (idx, res) in results { for (idx, res) in results {
if let Ok(resp) = res { if let Ok(resp) = res
if resp.success { && resp.success
{
successful_clients.push(idx); successful_clients.push(idx);
} }
} }
}
if successful_clients.len() >= self.quorum { if successful_clients.len() >= self.quorum {
let resp = LockResponse::success( let resp = LockResponse::success(
+3 -3
View File
@@ -204,12 +204,12 @@ impl TargetFactory for MQTTTargetFactory {
if !std::path::Path::new(&queue_dir).is_absolute() { if !std::path::Path::new(&queue_dir).is_absolute() {
return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string())); return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string()));
} }
if let Some(qos_str) = config.lookup(MQTT_QOS) { if let Some(qos_str) = config.lookup(MQTT_QOS)
if qos_str == "0" { && qos_str == "0"
{
warn!("Using queue_dir with QoS 0 may result in event loss"); warn!("Using queue_dir with QoS 0 may result in event loss");
} }
} }
}
Ok(()) Ok(())
} }
+3 -3
View File
@@ -345,12 +345,12 @@ impl TargetList {
pub async fn clear_targets_only(&mut self) { pub async fn clear_targets_only(&mut self) {
let target_ids_to_clear: Vec<TargetID> = self.targets.keys().cloned().collect(); let target_ids_to_clear: Vec<TargetID> = self.targets.keys().cloned().collect();
for id in target_ids_to_clear { for id in target_ids_to_clear {
if let Some(target_arc) = self.targets.remove(&id) { if let Some(target_arc) = self.targets.remove(&id)
if let Err(e) = target_arc.close().await { && let Err(e) = target_arc.close().await
{
error!("Failed to close target {} during clear: {}", id, e); error!("Failed to close target {} during clear: {}", id, e);
} }
} }
}
self.targets.clear(); self.targets.clear();
} }
+7 -8
View File
@@ -119,14 +119,13 @@ impl TargetRegistry {
format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}") format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}")
.to_uppercase(); .to_uppercase();
for (key, value) in &all_env { for (key, value) in &all_env {
if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false) { if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false)
if let Some(id) = key.strip_prefix(&enable_prefix) { && let Some(id) = key.strip_prefix(&enable_prefix)
if !id.is_empty() { && !id.is_empty()
{
instance_ids_from_env.insert(id.to_lowercase()); instance_ids_from_env.insert(id.to_lowercase());
} }
} }
}
}
// 3.2. Parse all relevant environment variable configurations // 3.2. Parse all relevant environment variable configurations
// 3.2.1. Build environment variable prefixes such as 'RUSTFS_NOTIFY_WEBHOOK_' // 3.2.1. Build environment variable prefixes such as 'RUSTFS_NOTIFY_WEBHOOK_'
@@ -273,11 +272,11 @@ impl TargetRegistry {
for section in sections { for section in sections {
let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new(); let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
// Add default item // Add default item
if let Some(default_kvs) = section_defaults.get(&section) { if let Some(default_kvs) = section_defaults.get(&section)
if !default_kvs.is_empty() { && !default_kvs.is_empty()
{
section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone()); section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
} }
}
// Add successful instance item // Add successful instance item
if let Some(instances) = successes_by_section.get(&section) { if let Some(instances) = successes_by_section.get(&section) {
+6 -6
View File
@@ -24,18 +24,19 @@ pub fn new_pattern(prefix: Option<&str>, suffix: Option<&str>) -> String {
let mut pattern = String::new(); let mut pattern = String::new();
// Process the prefix part // Process the prefix part
if let Some(p) = prefix { if let Some(p) = prefix
if !p.is_empty() { && !p.is_empty()
{
pattern.push_str(p); pattern.push_str(p);
if !p.ends_with('*') { if !p.ends_with('*') {
pattern.push('*'); pattern.push('*');
} }
} }
}
// Process the suffix part // Process the suffix part
if let Some(s) = suffix { if let Some(s) = suffix
if !s.is_empty() { && !s.is_empty()
{
let mut s_to_append = s.to_string(); let mut s_to_append = s.to_string();
if !s.starts_with('*') { if !s.starts_with('*') {
s_to_append.insert(0, '*'); s_to_append.insert(0, '*');
@@ -49,7 +50,6 @@ pub fn new_pattern(prefix: Option<&str>, suffix: Option<&str>) -> String {
pattern.push_str(&s_to_append); pattern.push_str(&s_to_append);
} }
} }
}
// Replace "**" with "*" // Replace "**" with "*"
pattern = pattern.replace("**", "*"); pattern = pattern.replace("**", "*");
+9 -9
View File
@@ -86,22 +86,22 @@ impl std::fmt::Debug for OtelGuard {
impl Drop for OtelGuard { impl Drop for OtelGuard {
fn drop(&mut self) { fn drop(&mut self) {
if let Some(provider) = self.tracer_provider.take() { if let Some(provider) = self.tracer_provider.take()
if let Err(err) = provider.shutdown() { && let Err(err) = provider.shutdown()
{
eprintln!("Tracer shutdown error: {err:?}"); eprintln!("Tracer shutdown error: {err:?}");
} }
}
if let Some(provider) = self.meter_provider.take() { if let Some(provider) = self.meter_provider.take()
if let Err(err) = provider.shutdown() { && let Err(err) = provider.shutdown()
{
eprintln!("Meter shutdown error: {err:?}"); eprintln!("Meter shutdown error: {err:?}");
} }
} if let Some(provider) = self.logger_provider.take()
if let Some(provider) = self.logger_provider.take() { && let Err(err) = provider.shutdown()
if let Err(err) = provider.shutdown() { {
eprintln!("Logger shutdown error: {err:?}"); eprintln!("Logger shutdown error: {err:?}");
} }
}
if let Some(handle) = self.flexi_logger_handles.take() { if let Some(handle) = self.flexi_logger_handles.take() {
handle.shutdown(); handle.shutdown();
+3 -3
View File
@@ -122,13 +122,13 @@ impl Resource {
// Apply condition substitutions // Apply condition substitutions
if !conditions.is_empty() { if !conditions.is_empty() {
for key in KeyName::COMMON_KEYS { for key in KeyName::COMMON_KEYS {
if let Some(rvalue) = conditions.get(key.name()) { if let Some(rvalue) = conditions.get(key.name())
if matches!(rvalue.first().map(|c| !c.is_empty()), Some(true)) { && matches!(rvalue.first().map(|c| !c.is_empty()), Some(true))
{
resolved_pattern = resolved_pattern.replace(&key.var_name(), &rvalue[0]); resolved_pattern = resolved_pattern.replace(&key.var_name(), &rvalue[0]);
} }
} }
} }
}
let cp = path::clean(resource); let cp = path::clean(resource);
if cp != "." && cp == resolved_pattern.as_str() { if cp != "." && cp == resolved_pattern.as_str() {
+13 -13
View File
@@ -387,7 +387,7 @@ impl Checksum {
// Ensure we don't divide by 0 // Ensure we don't divide by 0
let raw_len = self.checksum_type.raw_byte_len(); let raw_len = self.checksum_type.raw_byte_len();
if raw_len == 0 || parts.len() % raw_len != 0 { if raw_len == 0 || !parts.len().is_multiple_of(raw_len) {
checksums = 0; checksums = 0;
} else if !parts.is_empty() { } else if !parts.is_empty() {
checksums = (parts.len() / raw_len) as i32; checksums = (parts.len() / raw_len) as i32;
@@ -506,8 +506,9 @@ pub fn get_content_checksum(headers: &HeaderMap) -> Result<Option<Checksum>, std
for header in trailing_headers { for header in trailing_headers {
let mut duplicates = false; let mut duplicates = false;
for &checksum_type in crate::checksum::BASE_CHECKSUM_TYPES { for &checksum_type in crate::checksum::BASE_CHECKSUM_TYPES {
if let Some(key) = checksum_type.key() { if let Some(key) = checksum_type.key()
if header.eq_ignore_ascii_case(key) { && header.eq_ignore_ascii_case(key)
{
duplicates = result.is_some(); duplicates = result.is_some();
result = Some(Checksum { result = Some(Checksum {
checksum_type: ChecksumType(checksum_type.0 | ChecksumType::TRAILING.0), checksum_type: ChecksumType(checksum_type.0 | ChecksumType::TRAILING.0),
@@ -517,7 +518,6 @@ pub fn get_content_checksum(headers: &HeaderMap) -> Result<Option<Checksum>, std
}); });
} }
} }
}
if duplicates { if duplicates {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid checksum")); return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid checksum"));
} }
@@ -567,22 +567,23 @@ fn get_content_checksum_direct(headers: &HeaderMap) -> (ChecksumType, String) {
checksum_type = ChecksumType(checksum_type.0 | ChecksumType::FULL_OBJECT.0); checksum_type = ChecksumType(checksum_type.0 | ChecksumType::FULL_OBJECT.0);
} }
if checksum_type.is_set() { if checksum_type.is_set()
if let Some(key) = checksum_type.key() { && let Some(key) = checksum_type.key()
{
if let Some(value) = headers.get(key).and_then(|v| v.to_str().ok()) { if let Some(value) = headers.get(key).and_then(|v| v.to_str().ok()) {
return (checksum_type, value.to_string()); return (checksum_type, value.to_string());
} else { } else {
return (ChecksumType::NONE, String::new()); return (ChecksumType::NONE, String::new());
} }
} }
}
return (checksum_type, String::new()); return (checksum_type, String::new());
} }
// Check individual checksum headers // Check individual checksum headers
for &ct in crate::checksum::BASE_CHECKSUM_TYPES { for &ct in crate::checksum::BASE_CHECKSUM_TYPES {
if let Some(key) = ct.key() { if let Some(key) = ct.key()
if let Some(value) = headers.get(key).and_then(|v| v.to_str().ok()) { && let Some(value) = headers.get(key).and_then(|v| v.to_str().ok())
{
// If already set, invalid // If already set, invalid
if checksum_type != ChecksumType::NONE { if checksum_type != ChecksumType::NONE {
return (ChecksumType::INVALID, String::new()); return (ChecksumType::INVALID, String::new());
@@ -598,7 +599,6 @@ fn get_content_checksum_direct(headers: &HeaderMap) -> (ChecksumType, String) {
return (checksum_type, value.to_string()); return (checksum_type, value.to_string());
} }
} }
}
(checksum_type, String::new()) (checksum_type, String::new())
} }
@@ -965,11 +965,11 @@ fn gf2_matrix_times(mat: &[u64], mut vec: u64) -> u64 {
let mut mat_iter = mat.iter(); let mut mat_iter = mat.iter();
while vec != 0 { while vec != 0 {
if vec & 1 != 0 { if vec & 1 != 0
if let Some(&m) = mat_iter.next() { && let Some(&m) = mat_iter.next()
{
sum ^= m; sum ^= m;
} }
}
vec >>= 1; vec >>= 1;
mat_iter.next(); mat_iter.next();
} }
+22 -23
View File
@@ -178,13 +178,12 @@ impl HashReader {
)); ));
} }
if let Some(checksum) = existing_hash_reader.checksum() { if let Some(checksum) = existing_hash_reader.checksum()
if let Some(ref md5) = md5hex { && let Some(ref md5) = md5hex
if checksum != md5 { && checksum != md5
{
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "HashReader checksum mismatch")); return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "HashReader checksum mismatch"));
} }
}
}
if existing_hash_reader.size() > 0 && size > 0 && existing_hash_reader.size() != size { if existing_hash_reader.size() > 0 && size > 0 && existing_hash_reader.size() != size {
return Err(std::io::Error::new( return Err(std::io::Error::new(
@@ -359,16 +358,16 @@ impl HashReader {
} }
if checksum.checksum_type.trailing() { if checksum.checksum_type.trailing() {
if let Some(trailer) = self.trailer_s3s.as_ref() { if let Some(trailer) = self.trailer_s3s.as_ref()
if let Some(Some(checksum_str)) = trailer.read(|headers| { && let Some(Some(checksum_str)) = trailer.read(|headers| {
checksum checksum
.checksum_type .checksum_type
.key() .key()
.and_then(|key| headers.get(key).and_then(|value| value.to_str().ok().map(|s| s.to_string()))) .and_then(|key| headers.get(key).and_then(|value| value.to_str().ok().map(|s| s.to_string())))
}) { })
{
map.insert(checksum.checksum_type.to_string(), checksum_str); map.insert(checksum.checksum_type.to_string(), checksum_str);
} }
}
return map; return map;
} }
@@ -450,20 +449,20 @@ impl AsyncRead for HashReader {
if filled > 0 { if filled > 0 {
// Update SHA256 hasher // Update SHA256 hasher
if let Some(hasher) = this.content_sha256_hasher { if let Some(hasher) = this.content_sha256_hasher
if let Err(e) = hasher.write_all(data) { && let Err(e) = hasher.write_all(data)
{
error!("SHA256 hasher write error, error={:?}", e); error!("SHA256 hasher write error, error={:?}", e);
return Poll::Ready(Err(std::io::Error::other(e))); return Poll::Ready(Err(std::io::Error::other(e)));
} }
}
// Update content hasher // Update content hasher
if let Some(hasher) = this.content_hasher { if let Some(hasher) = this.content_hasher
if let Err(e) = hasher.write_all(data) { && let Err(e) = hasher.write_all(data)
{
return Poll::Ready(Err(std::io::Error::other(e))); return Poll::Ready(Err(std::io::Error::other(e)));
} }
} }
}
if filled == 0 && !*this.checksum_on_finish { if filled == 0 && !*this.checksum_on_finish {
// check SHA256 // check SHA256
@@ -477,13 +476,15 @@ impl AsyncRead for HashReader {
// check content hasher // check content hasher
if let (Some(hasher), Some(expected_content_hash)) = (this.content_hasher, this.content_hash) { if let (Some(hasher), Some(expected_content_hash)) = (this.content_hasher, this.content_hash) {
if expected_content_hash.checksum_type.trailing() { if expected_content_hash.checksum_type.trailing()
if let Some(trailer) = this.trailer_s3s.as_ref() { && let Some(trailer) = this.trailer_s3s.as_ref()
if let Some(Some(checksum_str)) = trailer.read(|headers| { && let Some(Some(checksum_str)) = trailer.read(|headers| {
expected_content_hash.checksum_type.key().and_then(|key| { expected_content_hash
headers.get(key).and_then(|value| value.to_str().ok().map(|s| s.to_string())) .checksum_type
.key()
.and_then(|key| headers.get(key).and_then(|value| value.to_str().ok().map(|s| s.to_string())))
}) })
}) { {
expected_content_hash.encoded = checksum_str; expected_content_hash.encoded = checksum_str;
expected_content_hash.raw = general_purpose::STANDARD expected_content_hash.raw = general_purpose::STANDARD
.decode(&expected_content_hash.encoded) .decode(&expected_content_hash.encoded)
@@ -493,8 +494,6 @@ impl AsyncRead for HashReader {
return Poll::Ready(Err(std::io::Error::other("Content hash mismatch"))); return Poll::Ready(Err(std::io::Error::other("Content hash mismatch")));
} }
} }
}
}
let content_hash = hasher.finalize(); let content_hash = hasher.finalize();
@@ -175,11 +175,11 @@ impl SimpleQueryDispatcher {
.clone() .clone()
.map(|e| e.as_bytes().first().copied().unwrap_or_default()), .map(|e| e.as_bytes().first().copied().unwrap_or_default()),
); );
if let Some(delimiter) = csv.field_delimiter.as_ref() { if let Some(delimiter) = csv.field_delimiter.as_ref()
if delimiter.len() == 1 { && delimiter.len() == 1
{
file_format = file_format.with_delimiter(delimiter.as_bytes()[0]); file_format = file_format.with_delimiter(delimiter.as_bytes()[0]);
} }
}
// TODO waiting for processing @junxiang Mu // TODO waiting for processing @junxiang Mu
// if csv.file_header_info.is_some() {} // if csv.file_header_info.is_some() {}
match csv.file_header_info.as_ref() { match csv.file_header_info.as_ref() {
+3 -3
View File
@@ -25,11 +25,11 @@ pub fn get_host_addr(req: &request::Request<Body>) -> String {
} else { } else {
req_host = uri.host().unwrap().to_string(); req_host = uri.host().unwrap().to_string();
} }
if let Some(host) = host { if let Some(host) = host
if req_host != *host.to_str().unwrap() { && req_host != *host.to_str().unwrap()
{
return (*host.to_str().unwrap()).to_string(); return (*host.to_str().unwrap()).to_string();
} }
}
/*if req.uri_ref().unwrap().host().is_some() { /*if req.uri_ref().unwrap().host().is_some() {
return req.uri_ref().unwrap().host().unwrap(); return req.uri_ref().unwrap().host().unwrap();
}*/ }*/
+3 -3
View File
@@ -97,12 +97,12 @@ pub fn parse_key(s: &str) -> Key {
} }
// Number of batch items parsed // Number of batch items parsed
if let Some(colon_pos) = name.find(':') { if let Some(colon_pos) = name.find(':')
if let Ok(count) = name[..colon_pos].parse::<usize>() { && let Ok(count) = name[..colon_pos].parse::<usize>()
{
item_count = count; item_count = count;
name = name[colon_pos + 1..].to_string(); name = name[colon_pos + 1..].to_string();
} }
}
// Resolve extension // Resolve extension
if let Some(dot_pos) = name.rfind('.') { if let Some(dot_pos) = name.rfind('.') {
+11 -10
View File
@@ -225,17 +225,20 @@ where
match tokio::time::timeout(DEFAULT_CONNECTION_TIMEOUT, async { match tokio::time::timeout(DEFAULT_CONNECTION_TIMEOUT, async {
while !self.connected.load(Ordering::SeqCst) { while !self.connected.load(Ordering::SeqCst) {
if let Some(handle) = self.bg_task_manager.init_cell.get() { if let Some(handle) = self.bg_task_manager.init_cell.get()
if handle.is_finished() && !self.connected.load(Ordering::SeqCst) { && handle.is_finished()
&& !self.connected.load(Ordering::SeqCst)
{
error!(target_id = %self.id, "MQTT background task exited prematurely before connection was established."); error!(target_id = %self.id, "MQTT background task exited prematurely before connection was established.");
return Err(TargetError::Network("MQTT background task exited prematurely".to_string())); return Err(TargetError::Network("MQTT background task exited prematurely".to_string()));
} }
}
tokio::time::sleep(Duration::from_millis(100)).await; tokio::time::sleep(Duration::from_millis(100)).await;
} }
debug!(target_id = %self.id, "MQTT target connected successfully."); debug!(target_id = %self.id, "MQTT target connected successfully.");
Ok(()) Ok(())
}).await { })
.await
{
Ok(Ok(_)) => { Ok(Ok(_)) => {
info!(target_id = %self.id, "MQTT target initialized and connected."); info!(target_id = %self.id, "MQTT target initialized and connected.");
Ok(()) Ok(())
@@ -243,9 +246,7 @@ where
Ok(Err(e)) => Err(e), Ok(Err(e)) => Err(e),
Err(_) => { Err(_) => {
error!(target_id = %self.id, "Timeout waiting for MQTT connection after task spawn."); error!(target_id = %self.id, "Timeout waiting for MQTT connection after task spawn.");
Err(TargetError::Network( Err(TargetError::Network("Timeout waiting for MQTT connection".to_string()))
"Timeout waiting for MQTT connection".to_string(),
))
} }
} }
} }
@@ -470,12 +471,12 @@ where
debug!(target_id = %self.id, "Checking if MQTT target is active."); debug!(target_id = %self.id, "Checking if MQTT target is active.");
if self.client.lock().await.is_none() && !self.connected.load(Ordering::SeqCst) { if self.client.lock().await.is_none() && !self.connected.load(Ordering::SeqCst) {
// Check if the background task is running and has not panicked // Check if the background task is running and has not panicked
if let Some(handle) = self.bg_task_manager.init_cell.get() { if let Some(handle) = self.bg_task_manager.init_cell.get()
if handle.is_finished() { && handle.is_finished()
{
error!(target_id = %self.id, "MQTT background task has finished, possibly due to an error. Target is not active."); error!(target_id = %self.id, "MQTT background task has finished, possibly due to an error. Target is not active.");
return Err(TargetError::Network("MQTT background task terminated".to_string())); return Err(TargetError::Network("MQTT background task terminated".to_string()));
} }
}
debug!(target_id = %self.id, "MQTT client not yet initialized or task not running/connected."); debug!(target_id = %self.id, "MQTT client not yet initialized or task not running/connected.");
return Err(TargetError::Configuration( return Err(TargetError::Configuration(
"MQTT client not available or not initialized/connected".to_string(), "MQTT client not available or not initialized/connected".to_string(),
+1 -1
View File
@@ -292,7 +292,7 @@ pub fn create_multi_cert_resolver(
for (domain, (certs, key)) in cert_key_pairs { for (domain, (certs, key)) in cert_key_pairs {
// create a signature // create a signature
let signing_key = rustls::crypto::ring::sign::any_supported_type(&key) let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key)
.map_err(|e| certs_error(format!("unsupported private key types:{domain}, err:{e:?}")))?; .map_err(|e| certs_error(format!("unsupported private key types:{domain}, err:{e:?}")))?;
// create a CertifiedKey // create a CertifiedKey
+25 -26
View File
@@ -56,20 +56,21 @@ fn is_xff_header_enabled() -> bool {
/// ///
pub fn get_source_scheme(headers: &HeaderMap) -> Option<String> { pub fn get_source_scheme(headers: &HeaderMap) -> Option<String> {
// Retrieve the scheme from X-Forwarded-Proto. // Retrieve the scheme from X-Forwarded-Proto.
if let Some(proto) = headers.get(X_FORWARDED_PROTO) { if let Some(proto) = headers.get(X_FORWARDED_PROTO)
if let Ok(proto_str) = proto.to_str() { && let Ok(proto_str) = proto.to_str()
{
return Some(proto_str.to_lowercase()); return Some(proto_str.to_lowercase());
} }
}
if let Some(proto) = headers.get(X_FORWARDED_SCHEME) { if let Some(proto) = headers.get(X_FORWARDED_SCHEME)
if let Ok(proto_str) = proto.to_str() { && let Ok(proto_str) = proto.to_str()
{
return Some(proto_str.to_lowercase()); return Some(proto_str.to_lowercase());
} }
}
if let Some(forwarded) = headers.get(FORWARDED) { if let Some(forwarded) = headers.get(FORWARDED)
if let Ok(forwarded_str) = forwarded.to_str() { && let Ok(forwarded_str) = forwarded.to_str()
{
// match should contain at least two elements if the protocol was // match should contain at least two elements if the protocol was
// specified in the Forwarded header. The first element will always be // specified in the Forwarded header. The first element will always be
// the 'for=', which we ignore, subsequently we proceed to look for // the 'for=', which we ignore, subsequently we proceed to look for
@@ -77,18 +78,17 @@ pub fn get_source_scheme(headers: &HeaderMap) -> Option<String> {
// we simply ignore the values and return empty. This is in line // we simply ignore the values and return empty. This is in line
// with the approach we took for returning first ip from multiple // with the approach we took for returning first ip from multiple
// params. // params.
if let Some(for_match) = FOR_REGEX.captures(forwarded_str) { if let Some(for_match) = FOR_REGEX.captures(forwarded_str)
if for_match.len() > 1 { && for_match.len() > 1
{
let remaining = &for_match[2]; let remaining = &for_match[2];
if let Some(proto_match) = PROTO_REGEX.captures(remaining) { if let Some(proto_match) = PROTO_REGEX.captures(remaining)
if proto_match.len() > 1 { && proto_match.len() > 1
{
return Some(proto_match[2].to_lowercase()); return Some(proto_match[2].to_lowercase());
} }
} }
} }
}
}
}
None None
} }
@@ -105,9 +105,10 @@ pub fn get_source_scheme(headers: &HeaderMap) -> Option<String> {
pub fn get_source_ip_from_headers(headers: &HeaderMap) -> Option<String> { pub fn get_source_ip_from_headers(headers: &HeaderMap) -> Option<String> {
let mut addr = None; let mut addr = None;
if is_xff_header_enabled() { if is_xff_header_enabled()
if let Some(forwarded_for) = headers.get(X_FORWARDED_FOR) { && let Some(forwarded_for) = headers.get(X_FORWARDED_FOR)
if let Ok(forwarded_str) = forwarded_for.to_str() { && let Ok(forwarded_str) = forwarded_for.to_str()
{
// Only grab the first (client) address. Note that '192.168.0.1, // Only grab the first (client) address. Note that '192.168.0.1,
// 10.1.1.1' is a valid key for X-Forwarded-For where addresses after // 10.1.1.1' is a valid key for X-Forwarded-For where addresses after
// the first may represent forwarding proxies earlier in the chain. // the first may represent forwarding proxies earlier in the chain.
@@ -115,8 +116,6 @@ pub fn get_source_ip_from_headers(headers: &HeaderMap) -> Option<String> {
let end = first_comma.unwrap_or(forwarded_str.len()); let end = first_comma.unwrap_or(forwarded_str.len());
addr = Some(forwarded_str[..end].to_string()); addr = Some(forwarded_str[..end].to_string());
} }
}
}
if addr.is_none() { if addr.is_none() {
if let Some(real_ip) = headers.get(X_REAL_IP) { if let Some(real_ip) = headers.get(X_REAL_IP) {
@@ -125,15 +124,17 @@ pub fn get_source_ip_from_headers(headers: &HeaderMap) -> Option<String> {
// request). // request).
addr = Some(real_ip_str.to_string()); addr = Some(real_ip_str.to_string());
} }
} else if let Some(forwarded) = headers.get(FORWARDED) { } else if let Some(forwarded) = headers.get(FORWARDED)
if let Ok(forwarded_str) = forwarded.to_str() { && let Ok(forwarded_str) = forwarded.to_str()
{
// match should contain at least two elements if the protocol was // match should contain at least two elements if the protocol was
// specified in the Forwarded header. The first element will always be // specified in the Forwarded header. The first element will always be
// the 'for=' capture, which we ignore. In the case of multiple IP // the 'for=' capture, which we ignore. In the case of multiple IP
// addresses (for=8.8.8.8, 8.8.4.4, 172.16.1.20 is valid) we only // addresses (for=8.8.8.8, 8.8.4.4, 172.16.1.20 is valid) we only
// extract the first, which should be the client IP. // extract the first, which should be the client IP.
if let Some(for_match) = FOR_REGEX.captures(forwarded_str) { if let Some(for_match) = FOR_REGEX.captures(forwarded_str)
if for_match.len() > 1 { && for_match.len() > 1
{
// IPv6 addresses in Forwarded headers are quoted-strings. We strip // IPv6 addresses in Forwarded headers are quoted-strings. We strip
// these quotes. // these quotes.
let ip = for_match[1].trim_matches('"'); let ip = for_match[1].trim_matches('"');
@@ -141,8 +142,6 @@ pub fn get_source_ip_from_headers(headers: &HeaderMap) -> Option<String> {
} }
} }
} }
}
}
addr addr
} }
+4 -5
View File
@@ -174,17 +174,16 @@ pub async fn get_host_ip(host: Host<&str>) -> std::io::Result<HashSet<IpAddr>> {
match host { match host {
Host::Domain(domain) => { Host::Domain(domain) => {
// Check cache first // Check cache first
if CUSTOM_DNS_RESOLVER.read().unwrap().is_none() { if CUSTOM_DNS_RESOLVER.read().unwrap().is_none()
if let Ok(mut cache) = DNS_CACHE.lock() { && let Ok(mut cache) = DNS_CACHE.lock()
if let Some(entry) = cache.get(domain) { && let Some(entry) = cache.get(domain)
{
if !entry.is_expired(DNS_CACHE_TTL) { if !entry.is_expired(DNS_CACHE_TTL) {
return Ok(entry.ips.clone()); return Ok(entry.ips.clone());
} }
// Remove expired entry // Remove expired entry
cache.remove(domain); cache.remove(domain);
} }
}
}
info!("Cache miss for domain {domain}, querying system resolver."); info!("Cache miss for domain {domain}, querying system resolver.");
+7 -8
View File
@@ -196,14 +196,13 @@ impl ParsedURL {
impl std::fmt::Display for ParsedURL { impl std::fmt::Display for ParsedURL {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut url = self.0.clone(); let mut url = self.0.clone();
if let Some(host) = url.host_str().map(|h| h.to_string()) { if let Some(host) = url.host_str().map(|h| h.to_string())
if let Some(port) = url.port() { && let Some(port) = url.port()
if (url.scheme() == "http" && port == 80) || (url.scheme() == "https" && port == 443) { && ((url.scheme() == "http" && port == 80) || (url.scheme() == "https" && port == 443))
{
url.set_host(Some(&host)).unwrap(); url.set_host(Some(&host)).unwrap();
url.set_port(None).unwrap(); url.set_port(None).unwrap();
} }
}
}
let mut s = url.to_string(); let mut s = url.to_string();
// If the URL ends with a slash and the path is just "/", remove the trailing slash. // If the URL ends with a slash and the path is just "/", remove the trailing slash.
@@ -251,14 +250,14 @@ impl<'de> serde::Deserialize<'de> for ParsedURL {
/// Returns NetError if parsing fails or host is invalid. /// Returns NetError if parsing fails or host is invalid.
/// ///
pub fn parse_url(s: &str) -> Result<ParsedURL, NetError> { pub fn parse_url(s: &str) -> Result<ParsedURL, NetError> {
if let Some(scheme_end) = s.find("://") { if let Some(scheme_end) = s.find("://")
if s[scheme_end + 3..].starts_with('/') { && s[scheme_end + 3..].starts_with('/')
{
let scheme = &s[..scheme_end]; let scheme = &s[..scheme_end];
if !scheme.is_empty() { if !scheme.is_empty() {
return Err(NetError::SchemeWithEmptyHost); return Err(NetError::SchemeWithEmptyHost);
} }
} }
}
let mut uu = Url::parse(s).map_err(|e| NetError::ParseError(e.to_string()))?; let mut uu = Url::parse(s).map_err(|e| NetError::ParseError(e.to_string()))?;
if uu.host_str().is_none_or(|h| h.is_empty()) { if uu.host_str().is_none_or(|h| h.is_empty()) {
+4 -4
View File
@@ -367,11 +367,12 @@ async fn _setup_console_tls_config(tls_path: Option<&String>) -> Result<Option<R
debug!("Found TLS directory for console, checking for certificates"); debug!("Found TLS directory for console, checking for certificates");
// Make sure to use a modern encryption suite // Make sure to use a modern encryption suite
let _ = rustls::crypto::ring::default_provider().install_default(); let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
// 1. Attempt to load all certificates in the directory (multi-certificate support, for SNI) // 1. Attempt to load all certificates in the directory (multi-certificate support, for SNI)
if let Ok(cert_key_pairs) = rustfs_utils::load_all_certs_from_directory(tls_path) { if let Ok(cert_key_pairs) = rustfs_utils::load_all_certs_from_directory(tls_path)
if !cert_key_pairs.is_empty() { && !cert_key_pairs.is_empty()
{
debug!( debug!(
"Found {} certificates for console, creating SNI-aware multi-cert resolver", "Found {} certificates for console, creating SNI-aware multi-cert resolver",
cert_key_pairs.len() cert_key_pairs.len()
@@ -396,7 +397,6 @@ async fn _setup_console_tls_config(tls_path: Option<&String>) -> Result<Option<R
info!(target: "rustfs::console::tls", "Console TLS enabled with multi-certificate SNI support"); info!(target: "rustfs::console::tls", "Console TLS enabled with multi-certificate SNI support");
return Ok(Some(RustlsConfig::from_config(Arc::new(server_config)))); return Ok(Some(RustlsConfig::from_config(Arc::new(server_config))));
} }
}
// 2. Revert to the traditional single-certificate mode // 2. Revert to the traditional single-certificate mode
let key_path = format!("{tls_path}/{RUSTFS_TLS_KEY}"); let key_path = format!("{tls_path}/{RUSTFS_TLS_KEY}");
+30 -30
View File
@@ -636,54 +636,54 @@ fn extract_metrics_init_params(uri: &Uri) -> MetricsParams {
for param in params { for param in params {
let mut parts = param.split('='); let mut parts = param.split('=');
if let Some(key) = parts.next() { if let Some(key) = parts.next() {
if key == "disks" { if key == "disks"
if let Some(value) = parts.next() { && let Some(value) = parts.next()
{
mp.disks = value.to_string(); mp.disks = value.to_string();
} }
} if key == "hosts"
if key == "hosts" { && let Some(value) = parts.next()
if let Some(value) = parts.next() { {
mp.hosts = value.to_string(); mp.hosts = value.to_string();
} }
} if key == "interval"
if key == "interval" { && let Some(value) = parts.next()
if let Some(value) = parts.next() { {
mp.tick = value.to_string(); mp.tick = value.to_string();
} }
} if key == "n"
if key == "n" { && let Some(value) = parts.next()
if let Some(value) = parts.next() { {
mp.n = value.parse::<u64>().unwrap_or(u64::MAX); mp.n = value.parse::<u64>().unwrap_or(u64::MAX);
} }
} if key == "types"
if key == "types" { && let Some(value) = parts.next()
if let Some(value) = parts.next() { {
mp.types = value.parse::<u32>().unwrap_or_default(); mp.types = value.parse::<u32>().unwrap_or_default();
} }
} if key == "by-disk"
if key == "by-disk" { && let Some(value) = parts.next()
if let Some(value) = parts.next() { {
mp.by_disk = value.to_string(); mp.by_disk = value.to_string();
} }
} if key == "by-host"
if key == "by-host" { && let Some(value) = parts.next()
if let Some(value) = parts.next() { {
mp.by_host = value.to_string(); mp.by_host = value.to_string();
} }
} if key == "by-jobID"
if key == "by-jobID" { && let Some(value) = parts.next()
if let Some(value) = parts.next() { {
mp.by_job_id = value.to_string(); mp.by_job_id = value.to_string();
} }
} if key == "by-depID"
if key == "by-depID" { && let Some(value) = parts.next()
if let Some(value) = parts.next() { {
mp.by_dep_id = value.to_string(); mp.by_dep_id = value.to_string();
} }
} }
} }
} }
}
mp mp
} }
@@ -830,11 +830,11 @@ fn extract_heal_init_params(body: &Bytes, uri: &Uri, params: Params<'_, '_>) ->
for param in params { for param in params {
let mut parts = param.split('='); let mut parts = param.split('=');
if let Some(key) = parts.next() { if let Some(key) = parts.next() {
if key == "clientToken" { if key == "clientToken"
if let Some(value) = parts.next() { && let Some(value) = parts.next()
{
hip.client_token = value.to_string(); hip.client_token = value.to_string();
} }
}
if key == "forceStart" && parts.next().is_some() { if key == "forceStart" && parts.next().is_some() {
hip.force_start = true; hip.force_start = true;
} }
+4 -3
View File
@@ -277,11 +277,12 @@ impl Operation for UpdateGroupMembers {
} else { } else {
warn!("add group members"); warn!("add group members");
if let Err(err) = iam_store.get_group_description(&args.group).await { if let Err(err) = iam_store.get_group_description(&args.group).await
if is_err_no_such_group(&err) && has_space_be(&args.group) { && is_err_no_such_group(&err)
&& has_space_be(&args.group)
{
return Err(s3_error!(InvalidArgument, "not such group")); return Err(s3_error!(InvalidArgument, "not such group"));
} }
}
iam_store.add_users_to_group(&args.group, args.members).await.map_err(|e| { iam_store.add_users_to_group(&args.group, args.members).await.map_err(|e| {
warn!("add group members failed, e: {:?}", e); warn!("add group members failed, e: {:?}", e);
+6 -6
View File
@@ -96,11 +96,11 @@ impl Operation for AddUser {
return Err(s3_error!(InvalidArgument, "access key is empty")); return Err(s3_error!(InvalidArgument, "access key is empty"));
} }
if let Some(sys_cred) = get_global_action_cred() { if let Some(sys_cred) = get_global_action_cred()
if constant_time_eq(&sys_cred.access_key, ak) { && constant_time_eq(&sys_cred.access_key, ak)
{
return Err(s3_error!(InvalidArgument, "can't create user with system access key")); return Err(s3_error!(InvalidArgument, "can't create user with system access key"));
} }
}
let Ok(iam_store) = rustfs_iam::get() else { let Ok(iam_store) = rustfs_iam::get() else {
return Err(s3_error!(InvalidRequest, "iam not init")); return Err(s3_error!(InvalidRequest, "iam not init"));
@@ -777,11 +777,11 @@ impl Operation for ImportIam {
let groups: HashMap<String, GroupInfo> = serde_json::from_slice(&file_content) let groups: HashMap<String, GroupInfo> = serde_json::from_slice(&file_content)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?; .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?;
for (group_name, group_info) in groups { for (group_name, group_info) in groups {
if let Err(e) = iam_store.get_group_description(&group_name).await { if let Err(e) = iam_store.get_group_description(&group_name).await
if matches!(e, rustfs_iam::error::Error::NoSuchGroup(_)) || has_space_be(&group_name) { && (matches!(e, rustfs_iam::error::Error::NoSuchGroup(_)) || has_space_be(&group_name))
{
return Err(s3_error!(InvalidArgument, "group not found or has space be")); return Err(s3_error!(InvalidArgument, "group not found or has space be"));
} }
}
if let Err(e) = iam_store.add_users_to_group(&group_name, group_info.members.clone()).await { if let Err(e) = iam_store.add_users_to_group(&group_name, group_info.members.clone()).await {
failed.groups.push(IAMErrEntity { failed.groups.push(IAMErrEntity {
+39 -39
View File
@@ -175,11 +175,11 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("check claims failed1 {e}")))?; .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("check claims failed1 {e}")))?;
if !ok { if !ok {
if let Some(u) = u { if let Some(u) = u
if u.credentials.status == "off" { && u.credentials.status == "off"
{
return Err(s3_error!(InvalidRequest, "ErrAccessKeyDisabled")); return Err(s3_error!(InvalidRequest, "ErrAccessKeyDisabled"));
} }
}
return Err(s3_error!(InvalidRequest, "ErrAccessKeyDisabled")); return Err(s3_error!(InvalidRequest, "ErrAccessKeyDisabled"));
} }
@@ -200,11 +200,11 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
constant_time_eq(&sys_cred.access_key, &cred.access_key) || constant_time_eq(&cred.parent_user, &sys_cred.access_key); constant_time_eq(&sys_cred.access_key, &cred.access_key) || constant_time_eq(&cred.parent_user, &sys_cred.access_key);
// permitRootAccess // permitRootAccess
if let Some(claims) = &cred.claims { if let Some(claims) = &cred.claims
if claims.contains_key(SESSION_POLICY_NAME) { && claims.contains_key(SESSION_POLICY_NAME)
{
owner = false owner = false
} }
}
Ok((cred, owner)) Ok((cred, owner))
} }
@@ -358,11 +358,11 @@ pub fn get_condition_values(
args.insert("authType".to_owned(), vec![auth_type]); args.insert("authType".to_owned(), vec![auth_type]);
} }
if let Some(lc) = region { if let Some(lc) = region
if !lc.is_empty() { && !lc.is_empty()
{
args.insert("LocationConstraint".to_owned(), vec![lc.to_string()]); args.insert("LocationConstraint".to_owned(), vec![lc.to_string()]);
} }
}
let mut clone_header = header.clone(); let mut clone_header = header.clone();
if let Some(v) = clone_header.get("x-amz-signature-age") { if let Some(v) = clone_header.get("x-amz-signature-age") {
@@ -411,8 +411,9 @@ pub fn get_condition_values(
} }
} }
if let Some(grps_val) = claims.get("groups") { if let Some(grps_val) = claims.get("groups")
if let Some(grps_is) = grps_val.as_array() { && let Some(grps_is) = grps_val.as_array()
{
let grps = grps_is let grps = grps_is
.iter() .iter()
.filter_map(|g| g.as_str().map(|s| s.to_string())) .filter_map(|g| g.as_str().map(|s| s.to_string()))
@@ -422,13 +423,12 @@ pub fn get_condition_values(
} }
} }
} }
}
if let Some(groups) = &cred.groups { if let Some(groups) = &cred.groups
if !args.contains_key("groups") { && !args.contains_key("groups")
{
args.insert("groups".to_string(), groups.clone()); args.insert("groups".to_string(), groups.clone());
} }
}
args args
} }
@@ -502,11 +502,11 @@ fn determine_auth_type_and_version(header: &HeaderMap) -> (String, String) {
/// # Returns /// # Returns
/// * `bool` - True if request has JWT, false otherwise /// * `bool` - True if request has JWT, false otherwise
fn is_request_jwt(header: &HeaderMap) -> bool { fn is_request_jwt(header: &HeaderMap) -> bool {
if let Some(auth) = header.get("authorization") { if let Some(auth) = header.get("authorization")
if let Ok(auth_str) = auth.to_str() { && let Ok(auth_str) = auth.to_str()
{
return auth_str.starts_with(JWT_ALGORITHM); return auth_str.starts_with(JWT_ALGORITHM);
} }
}
false false
} }
@@ -518,11 +518,11 @@ fn is_request_jwt(header: &HeaderMap) -> bool {
/// # Returns /// # Returns
/// * `bool` - True if request has AWS Signature Version '4', false otherwise /// * `bool` - True if request has AWS Signature Version '4', false otherwise
fn is_request_signature_v4(header: &HeaderMap) -> bool { fn is_request_signature_v4(header: &HeaderMap) -> bool {
if let Some(auth) = header.get("authorization") { if let Some(auth) = header.get("authorization")
if let Ok(auth_str) = auth.to_str() { && let Ok(auth_str) = auth.to_str()
{
return auth_str.starts_with(SIGN_V4_ALGORITHM); return auth_str.starts_with(SIGN_V4_ALGORITHM);
} }
}
false false
} }
@@ -534,11 +534,11 @@ fn is_request_signature_v4(header: &HeaderMap) -> bool {
/// # Returns /// # Returns
/// * `bool` - True if request has AWS Signature Version '2', false otherwise /// * `bool` - True if request has AWS Signature Version '2', false otherwise
fn is_request_signature_v2(header: &HeaderMap) -> bool { fn is_request_signature_v2(header: &HeaderMap) -> bool {
if let Some(auth) = header.get("authorization") { if let Some(auth) = header.get("authorization")
if let Ok(auth_str) = auth.to_str() { && let Ok(auth_str) = auth.to_str()
{
return !auth_str.starts_with(SIGN_V4_ALGORITHM) && auth_str.starts_with(SIGN_V2_ALGORITHM); return !auth_str.starts_with(SIGN_V4_ALGORITHM) && auth_str.starts_with(SIGN_V2_ALGORITHM);
} }
}
false false
} }
@@ -578,41 +578,41 @@ fn is_request_presigned_signature_v2(header: &HeaderMap) -> bool {
/// # Returns /// # Returns
/// * `bool` - True if request has AWS Post policy Signature Version '4', false otherwise /// * `bool` - True if request has AWS Post policy Signature Version '4', false otherwise
fn is_request_post_policy_signature_v4(header: &HeaderMap) -> bool { fn is_request_post_policy_signature_v4(header: &HeaderMap) -> bool {
if let Some(content_type) = header.get("content-type") { if let Some(content_type) = header.get("content-type")
if let Ok(ct) = content_type.to_str() { && let Ok(ct) = content_type.to_str()
{
return ct.contains("multipart/form-data"); return ct.contains("multipart/form-data");
} }
}
false false
} }
/// Verify if the request has AWS Streaming Signature Version '4' /// Verify if the request has AWS Streaming Signature Version '4'
fn is_request_sign_streaming_v4(header: &HeaderMap) -> bool { fn is_request_sign_streaming_v4(header: &HeaderMap) -> bool {
if let Some(content_sha256) = header.get("x-amz-content-sha256") { if let Some(content_sha256) = header.get("x-amz-content-sha256")
if let Ok(sha256_str) = content_sha256.to_str() { && let Ok(sha256_str) = content_sha256.to_str()
{
return sha256_str == STREAMING_CONTENT_SHA256; return sha256_str == STREAMING_CONTENT_SHA256;
} }
}
false false
} }
// Verify if the request has AWS Streaming Signature Version '4' with trailer // Verify if the request has AWS Streaming Signature Version '4' with trailer
fn is_request_sign_streaming_trailer_v4(header: &HeaderMap) -> bool { fn is_request_sign_streaming_trailer_v4(header: &HeaderMap) -> bool {
if let Some(content_sha256) = header.get("x-amz-content-sha256") { if let Some(content_sha256) = header.get("x-amz-content-sha256")
if let Ok(sha256_str) = content_sha256.to_str() { && let Ok(sha256_str) = content_sha256.to_str()
{
return sha256_str == STREAMING_CONTENT_SHA256_TRAILER; return sha256_str == STREAMING_CONTENT_SHA256_TRAILER;
} }
}
false false
} }
// Verify if the request has AWS Streaming Signature Version '4' with unsigned content and trailer // Verify if the request has AWS Streaming Signature Version '4' with unsigned content and trailer
fn is_request_unsigned_trailer_v4(header: &HeaderMap) -> bool { fn is_request_unsigned_trailer_v4(header: &HeaderMap) -> bool {
if let Some(content_sha256) = header.get("x-amz-content-sha256") { if let Some(content_sha256) = header.get("x-amz-content-sha256")
if let Ok(sha256_str) = content_sha256.to_str() { && let Ok(sha256_str) = content_sha256.to_str()
{
return sha256_str == UNSIGNED_PAYLOAD_TRAILER; return sha256_str == UNSIGNED_PAYLOAD_TRAILER;
} }
}
false false
} }
@@ -634,12 +634,12 @@ pub fn get_query_param<'a>(query: &'a str, param_name: &str) -> Option<&'a str>
for pair in query.split('&') { for pair in query.split('&') {
let mut parts = pair.splitn(2, '='); let mut parts = pair.splitn(2, '=');
if let (Some(key), Some(value)) = (parts.next(), parts.next()) { if let (Some(key), Some(value)) = (parts.next(), parts.next())
if key.to_lowercase() == param_name { && key.to_lowercase() == param_name
{
return Some(value); return Some(value);
} }
} }
}
None None
} }
+4 -6
View File
@@ -193,10 +193,10 @@ impl From<ApiError> for S3Error {
impl From<StorageError> for ApiError { impl From<StorageError> for ApiError {
fn from(err: StorageError) -> Self { fn from(err: StorageError) -> Self {
// Special handling for Io errors that may contain ChecksumMismatch // Special handling for Io errors that may contain ChecksumMismatch
if let StorageError::Io(ref io_err) = err { if let StorageError::Io(ref io_err) = err
if let Some(inner) = io_err.get_ref() { && let Some(inner) = io_err.get_ref()
if inner.downcast_ref::<rustfs_rio::ChecksumMismatch>().is_some() && (inner.downcast_ref::<rustfs_rio::ChecksumMismatch>().is_some()
|| inner.downcast_ref::<rustfs_rio::BadDigest>().is_some() || inner.downcast_ref::<rustfs_rio::BadDigest>().is_some())
{ {
return ApiError { return ApiError {
code: S3ErrorCode::BadDigest, code: S3ErrorCode::BadDigest,
@@ -204,8 +204,6 @@ impl From<StorageError> for ApiError {
source: Some(Box::new(err)), source: Some(Box::new(err)),
}; };
} }
}
}
let code = match &err { let code = match &err {
StorageError::NotImplemented => S3ErrorCode::NotImplemented, StorageError::NotImplemented => S3ErrorCode::NotImplemented,
+3 -3
View File
@@ -735,8 +735,9 @@ impl StorageBackend<super::server::FtpsUser> for FtpsDriver {
match s3_client.list_objects_v2(list_input).await { match s3_client.list_objects_v2(list_input).await {
Ok(output) => { Ok(output) => {
if let Some(objects) = output.contents { if let Some(objects) = output.contents
if !objects.is_empty() { && !objects.is_empty()
{
debug!("FTPS RMD - bucket '{}' is not empty, cannot delete", bucket); debug!("FTPS RMD - bucket '{}' is not empty, cannot delete", bucket);
return Err(Error::new( return Err(Error::new(
ErrorKind::PermanentFileNotAvailable, ErrorKind::PermanentFileNotAvailable,
@@ -744,7 +745,6 @@ impl StorageBackend<super::server::FtpsUser> for FtpsDriver {
)); ));
} }
} }
}
Err(e) => { Err(e) => {
debug!("FTPS RMD - failed to list objects: {}", e); debug!("FTPS RMD - failed to list objects: {}", e);
} }
+6 -6
View File
@@ -98,17 +98,17 @@ impl FtpsConfig {
)); ));
} }
if let Some(path) = &self.cert_file { if let Some(path) = &self.cert_file
if !tokio::fs::try_exists(path).await.unwrap_or(false) { && !tokio::fs::try_exists(path).await.unwrap_or(false)
{
return Err(FtpsInitError::InvalidConfig(format!("Certificate file not found: {}", path))); return Err(FtpsInitError::InvalidConfig(format!("Certificate file not found: {}", path)));
} }
}
if let Some(path) = &self.key_file { if let Some(path) = &self.key_file
if !tokio::fs::try_exists(path).await.unwrap_or(false) { && !tokio::fs::try_exists(path).await.unwrap_or(false)
{
return Err(FtpsInitError::InvalidConfig(format!("Key file not found: {}", path))); return Err(FtpsInitError::InvalidConfig(format!("Key file not found: {}", path)));
} }
}
// Validate passive ports format // Validate passive ports format
if self.passive_ports.is_some() { if self.passive_ports.is_some() {
+3 -3
View File
@@ -753,8 +753,9 @@ impl Handler for SftpHandler {
match s3_client.list_objects_v2(list_input).await { match s3_client.list_objects_v2(list_input).await {
Ok(output) => { Ok(output) => {
if let Some(objects) = output.contents { if let Some(objects) = output.contents
if !objects.is_empty() { && !objects.is_empty()
{
debug!("SFTP REMOVE - bucket '{}' is not empty, cannot delete", bucket); debug!("SFTP REMOVE - bucket '{}' is not empty, cannot delete", bucket);
return Ok(Status { return Ok(Status {
id, id,
@@ -764,7 +765,6 @@ impl Handler for SftpHandler {
}); });
} }
} }
}
Err(e) => { Err(e) => {
debug!("SFTP REMOVE - failed to list objects: {}", e); debug!("SFTP REMOVE - failed to list objects: {}", e);
} }
+3 -3
View File
@@ -696,11 +696,11 @@ fn compare_keys(stored_key: &str, client_key_base64: &str) -> bool {
return true; return true;
} }
if let Ok(stored_bytes) = BASE64.decode(stored_key_data) { if let Ok(stored_bytes) = BASE64.decode(stored_key_data)
if let Ok(client_bytes) = BASE64.decode(client_key_base64) { && let Ok(client_bytes) = BASE64.decode(client_key_base64)
{
return stored_bytes == client_bytes; return stored_bytes == client_bytes;
} }
}
false false
} }
+6 -6
View File
@@ -260,14 +260,14 @@ async fn walk_dir(path: PathBuf, cert_name: &str, cert_data: &mut Vec<u8>) {
// Only check direct subdirectories, no deeper recursion // Only check direct subdirectories, no deeper recursion
if let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await { if let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await {
while let Ok(Some(sub_entry)) = sub_rd.next_entry().await { while let Ok(Some(sub_entry)) = sub_rd.next_entry().await {
if let Ok(sub_ft) = sub_entry.file_type().await { if let Ok(sub_ft) = sub_entry.file_type().await
if sub_ft.is_file() { && sub_ft.is_file()
{
load_if_matches(&sub_entry, cert_name, cert_data).await; load_if_matches(&sub_entry, cert_name, cert_data).await;
} }
// Ignore subdirectories and symlinks in subdirs to limit to one level // Ignore subdirectories and symlinks in subdirs to limit to one level
} }
} }
}
} else if ft.is_symlink() { } else if ft.is_symlink() {
// Follow symlink and treat target as file or directory, but limit to one level // Follow symlink and treat target as file or directory, but limit to one level
if let Ok(meta) = tokio::fs::metadata(&entry.path()).await { if let Ok(meta) = tokio::fs::metadata(&entry.path()).await {
@@ -277,8 +277,9 @@ async fn walk_dir(path: PathBuf, cert_name: &str, cert_data: &mut Vec<u8>) {
// Treat as directory but only check its direct contents // Treat as directory but only check its direct contents
if let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await { if let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await {
while let Ok(Some(sub_entry)) = sub_rd.next_entry().await { while let Ok(Some(sub_entry)) = sub_rd.next_entry().await {
if let Ok(sub_ft) = sub_entry.file_type().await { if let Ok(sub_ft) = sub_entry.file_type().await
if sub_ft.is_file() { && sub_ft.is_file()
{
load_if_matches(&sub_entry, cert_name, cert_data).await; load_if_matches(&sub_entry, cert_name, cert_data).await;
} }
// Ignore deeper levels // Ignore deeper levels
@@ -289,7 +290,6 @@ async fn walk_dir(path: PathBuf, cert_name: &str, cert_data: &mut Vec<u8>) {
} }
} }
} }
}
} else { } else {
debug!("Certificate directory not found: {}", path.display()); debug!("Certificate directory not found: {}", path.display());
} }
+14 -19
View File
@@ -282,41 +282,36 @@ impl Predicate for CompressionPredicate {
// CompressionLayer before calling this predicate, so we don't need to check them here. // CompressionLayer before calling this predicate, so we don't need to check them here.
// Check Content-Length header for minimum size threshold // Check Content-Length header for minimum size threshold
if let Some(content_length) = response.headers().get(http::header::CONTENT_LENGTH) { if let Some(content_length) = response.headers().get(http::header::CONTENT_LENGTH)
if let Ok(length_str) = content_length.to_str() { && let Ok(length_str) = content_length.to_str()
if let Ok(length) = length_str.parse::<u64>() { && let Ok(length) = length_str.parse::<u64>()
if length < self.config.min_size { && length < self.config.min_size
{
debug!( debug!(
"Skipping compression for small response: size={} bytes, min_size={}", "Skipping compression for small response: size={} bytes, min_size={}",
length, self.config.min_size length, self.config.min_size
); );
return false; return false;
} }
}
}
}
// Check if the response matches configured extension via Content-Disposition // Check if the response matches configured extension via Content-Disposition
if let Some(content_disposition) = response.headers().get(http::header::CONTENT_DISPOSITION) { if let Some(content_disposition) = response.headers().get(http::header::CONTENT_DISPOSITION)
if let Ok(cd) = content_disposition.to_str() { && let Ok(cd) = content_disposition.to_str()
if let Some(filename) = CompressionConfig::extract_filename_from_content_disposition(cd) { && let Some(filename) = CompressionConfig::extract_filename_from_content_disposition(cd)
if self.config.matches_extension(&filename) { && self.config.matches_extension(&filename)
{
debug!("Compressing response: filename '{}' matches configured extension", filename); debug!("Compressing response: filename '{}' matches configured extension", filename);
return true; return true;
} }
}
}
}
// Check if the response matches configured MIME type // Check if the response matches configured MIME type
if let Some(content_type) = response.headers().get(http::header::CONTENT_TYPE) { if let Some(content_type) = response.headers().get(http::header::CONTENT_TYPE)
if let Ok(ct) = content_type.to_str() { && let Ok(ct) = content_type.to_str()
if self.config.matches_mime_type(ct) { && self.config.matches_mime_type(ct)
{
debug!("Compressing response: Content-Type '{}' matches configured MIME pattern", ct); debug!("Compressing response: Content-Type '{}' matches configured MIME pattern", ct);
return true; return true;
} }
}
}
// Default: don't compress (whitelist approach) // Default: don't compress (whitelist approach)
debug!("Skipping compression: response does not match any configured extension or MIME type"); debug!("Skipping compression: response does not match any configured extension or MIME type");
+9 -8
View File
@@ -139,14 +139,14 @@ pub async fn start_http_server(
}; };
// If address is IPv6 try to enable dual-stack; on failure, switch to IPv4 socket. // If address is IPv6 try to enable dual-stack; on failure, switch to IPv4 socket.
if server_addr.is_ipv6() { if server_addr.is_ipv6()
if let Err(e) = socket.set_only_v6(false) { && let Err(e) = socket.set_only_v6(false)
{
warn!("Failed to set IPV6_V6ONLY=false, attempting IPv4 fallback: {}", e); warn!("Failed to set IPV6_V6ONLY=false, attempting IPv4 fallback: {}", e);
let ipv4_addr = SocketAddr::new(std::net::Ipv4Addr::UNSPECIFIED.into(), server_addr.port()); let ipv4_addr = SocketAddr::new(std::net::Ipv4Addr::UNSPECIFIED.into(), server_addr.port());
server_addr = ipv4_addr; server_addr = ipv4_addr;
socket = socket2::Socket::new(socket2::Domain::IPV4, socket2::Type::STREAM, Some(socket2::Protocol::TCP))?; socket = socket2::Socket::new(socket2::Domain::IPV4, socket2::Type::STREAM, Some(socket2::Protocol::TCP))?;
} }
}
// Common setup for both IPv4 and successful dual-stack IPv6 // Common setup for both IPv4 and successful dual-stack IPv6
let backlog = get_listen_backlog(); let backlog = get_listen_backlog();
@@ -434,12 +434,13 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result<Option<TlsAcceptor>> {
debug!("Found TLS directory, checking for certificates"); debug!("Found TLS directory, checking for certificates");
// Make sure to use a modern encryption suite // Make sure to use a modern encryption suite
let _ = rustls::crypto::ring::default_provider().install_default(); let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
let mtls_verifier = rustfs_utils::build_webpki_client_verifier(tls_path)?; let mtls_verifier = rustfs_utils::build_webpki_client_verifier(tls_path)?;
// 1. Attempt to load all certificates in the directory (multi-certificate support, for SNI) // 1. Attempt to load all certificates in the directory (multi-certificate support, for SNI)
if let Ok(cert_key_pairs) = rustfs_utils::load_all_certs_from_directory(tls_path) { if let Ok(cert_key_pairs) = rustfs_utils::load_all_certs_from_directory(tls_path)
if !cert_key_pairs.is_empty() { && !cert_key_pairs.is_empty()
{
debug!("Found {} certificates, creating SNI-aware multi-cert resolver", cert_key_pairs.len()); debug!("Found {} certificates, creating SNI-aware multi-cert resolver", cert_key_pairs.len());
// Create an SNI-enabled certificate resolver // Create an SNI-enabled certificate resolver
@@ -466,7 +467,6 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result<Option<TlsAcceptor>> {
return Ok(Some(TlsAcceptor::from(Arc::new(server_config)))); return Ok(Some(TlsAcceptor::from(Arc::new(server_config))));
} }
}
// 2. Revert to the traditional single-certificate mode // 2. Revert to the traditional single-certificate mode
let key_path = format!("{tls_path}/{RUSTFS_TLS_KEY}"); let key_path = format!("{tls_path}/{RUSTFS_TLS_KEY}");
@@ -520,7 +520,8 @@ struct ConnectionContext {
/// 2. Build a complete service stack for this connection, including S3, RPC services, and all middleware. /// 2. Build a complete service stack for this connection, including S3, RPC services, and all middleware.
/// 3. Use Hyper to handle HTTP requests on this connection. /// 3. Use Hyper to handle HTTP requests on this connection.
/// 4. Incorporate connections into the management of elegant closures. /// 4. Incorporate connections into the management of elegant closures.
#[instrument(skip_all, fields(peer_addr = %socket.peer_addr().map(|a| a.to_string()).unwrap_or_else(|_| "unknown".to_string())))] #[instrument(skip_all, fields(peer_addr = %socket.peer_addr().map(|a| a.to_string()).unwrap_or_else(|_| "unknown".to_string())
))]
fn process_connection( fn process_connection(
socket: TcpStream, socket: TcpStream,
tls_acceptor: Option<Arc<TlsAcceptor>>, tls_acceptor: Option<Arc<TlsAcceptor>>,
+91 -109
View File
@@ -519,14 +519,14 @@ fn validate_list_object_unordered_with_delimiter(delimiter: Option<&Delimiter>,
return Ok(()); return Ok(());
}; };
if let Ok(params) = from_bytes::<ListObjectUnorderedQuery>(query.as_bytes()) { if let Ok(params) = from_bytes::<ListObjectUnorderedQuery>(query.as_bytes())
if params.allow_unordered.as_deref() == Some("true") { && params.allow_unordered.as_deref() == Some("true")
{
return Err(S3Error::with_message( return Err(S3Error::with_message(
S3ErrorCode::InvalidArgument, S3ErrorCode::InvalidArgument,
"The allow-unordered parameter cannot be used when delimiter is specified.".to_string(), "The allow-unordered parameter cannot be used when delimiter is specified.".to_string(),
)); ));
} }
}
Ok(()) Ok(())
} }
@@ -735,8 +735,8 @@ impl FS {
let mut checksum_sha256 = input.checksum_sha256; let mut checksum_sha256 = input.checksum_sha256;
let mut checksum_crc64nvme = input.checksum_crc64nvme; let mut checksum_crc64nvme = input.checksum_crc64nvme;
if let Some(alg) = &input.checksum_algorithm { if let Some(alg) = &input.checksum_algorithm
if let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| { && let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| {
let key = match alg.as_str() { let key = match alg.as_str() {
ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(), ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(),
ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(), ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(),
@@ -750,7 +750,8 @@ impl FS {
.get(key.unwrap_or_default()) .get(key.unwrap_or_default())
.and_then(|value| value.to_str().ok().map(|s| s.to_string())) .and_then(|value| value.to_str().ok().map(|s| s.to_string()))
}) })
}) { })
{
match alg.as_str() { match alg.as_str() {
ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str, ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str,
ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str, ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str,
@@ -760,7 +761,6 @@ impl FS {
_ => (), _ => (),
} }
} }
}
warn!( warn!(
"put object extract checksum_crc32={checksum_crc32:?}, checksum_crc32c={checksum_crc32c:?}, checksum_sha1={checksum_sha1:?}, checksum_sha256={checksum_sha256:?}, checksum_crc64nvme={checksum_crc64nvme:?}", "put object extract checksum_crc32={checksum_crc32:?}, checksum_crc32c={checksum_crc32c:?}, checksum_sha1={checksum_sha1:?}, checksum_sha256={checksum_sha256:?}, checksum_crc64nvme={checksum_crc64nvme:?}",
@@ -977,8 +977,9 @@ impl S3 for FS {
let mut reader = HashReader::new(reader, length, actual_size, None, None, false).map_err(ApiError::from)?; let mut reader = HashReader::new(reader, length, actual_size, None, None, false).map_err(ApiError::from)?;
if let Some(ref sse_alg) = effective_sse { if let Some(ref sse_alg) = effective_sse
if is_managed_sse(sse_alg) { && is_managed_sse(sse_alg)
{
let material = let material =
create_managed_encryption_material(&bucket, &key, sse_alg, effective_kms_key_id.clone(), actual_size).await?; create_managed_encryption_material(&bucket, &key, sse_alg, effective_kms_key_id.clone(), actual_size).await?;
@@ -997,12 +998,11 @@ impl S3 for FS {
let encrypt_reader = EncryptReader::new(reader, key_bytes, nonce); let encrypt_reader = EncryptReader::new(reader, key_bytes, nonce);
reader = HashReader::new(Box::new(encrypt_reader), -1, actual_size, None, None, false).map_err(ApiError::from)?; reader = HashReader::new(Box::new(encrypt_reader), -1, actual_size, None, None, false).map_err(ApiError::from)?;
} }
}
// Apply SSE-C encryption if customer-provided key is specified // Apply SSE-C encryption if customer-provided key is specified
if let (Some(sse_alg), Some(sse_key), Some(sse_md5)) = (&sse_customer_algorithm, &sse_customer_key, &sse_customer_key_md5) if let (Some(sse_alg), Some(sse_key), Some(sse_md5)) = (&sse_customer_algorithm, &sse_customer_key, &sse_customer_key_md5)
&& sse_alg.as_str() == "AES256"
{ {
if sse_alg.as_str() == "AES256" {
let key_bytes = BASE64_STANDARD.decode(sse_key.as_str()).map_err(|e| { let key_bytes = BASE64_STANDARD.decode(sse_key.as_str()).map_err(|e| {
error!("Failed to decode SSE-C key: {}", e); error!("Failed to decode SSE-C key: {}", e);
ApiError::from(StorageError::other("Invalid SSE-C key")) ApiError::from(StorageError::other("Invalid SSE-C key"))
@@ -1035,7 +1035,6 @@ impl S3 for FS {
let encrypt_reader = EncryptReader::new(reader, key_array, nonce); let encrypt_reader = EncryptReader::new(reader, key_array, nonce);
reader = HashReader::new(Box::new(encrypt_reader), -1, actual_size, None, None, false).map_err(ApiError::from)?; reader = HashReader::new(Box::new(encrypt_reader), -1, actual_size, None, None, false).map_err(ApiError::from)?;
} }
}
src_info.put_object_reader = Some(PutObjReader::new(reader)); src_info.put_object_reader = Some(PutObjReader::new(reader));
@@ -1246,16 +1245,15 @@ impl S3 for FS {
let restore_object = Uuid::new_v4().to_string(); let restore_object = Uuid::new_v4().to_string();
//if let Some(rreq) = rreq { //if let Some(rreq) = rreq {
if let Some(output_location) = &rreq.output_location { if let Some(output_location) = &rreq.output_location
if let Some(s3) = &output_location.s3 { && let Some(s3) = &output_location.s3
if !s3.bucket_name.is_empty() { && !s3.bucket_name.is_empty()
{
header.insert( header.insert(
X_AMZ_RESTORE_OUTPUT_PATH, X_AMZ_RESTORE_OUTPUT_PATH,
format!("{}{}{}", s3.bucket_name, s3.prefix, restore_object).parse().unwrap(), format!("{}{}{}", s3.bucket_name, s3.prefix, restore_object).parse().unwrap(),
); );
} }
}
}
//} //}
/*send_event(EventArgs { /*send_event(EventArgs {
event_name: event::ObjectRestorePost, event_name: event::ObjectRestorePost,
@@ -1730,8 +1728,8 @@ impl S3 for FS {
}; };
for dobjs in delete_results.iter() { for dobjs in delete_results.iter() {
if let Some(dobj) = &dobjs.delete_object { if let Some(dobj) = &dobjs.delete_object
if replicate_deletes && replicate_deletes
&& (dobj.delete_marker_replication_status() == ReplicationStatusType::Pending && (dobj.delete_marker_replication_status() == ReplicationStatusType::Pending
|| dobj.version_purge_status() == VersionPurgeStatusType::Pending) || dobj.version_purge_status() == VersionPurgeStatusType::Pending)
{ {
@@ -1749,7 +1747,6 @@ impl S3 for FS {
schedule_replication_delete(deleted_object).await; schedule_replication_delete(deleted_object).await;
} }
} }
}
let req_headers = req.headers.clone(); let req_headers = req.headers.clone();
tokio::spawn(async move { tokio::spawn(async move {
@@ -1854,8 +1851,11 @@ impl S3 for FS {
let cache_key = ConcurrencyManager::make_cache_key(&bucket, &key, version_id.as_deref()); let cache_key = ConcurrencyManager::make_cache_key(&bucket, &key, version_id.as_deref());
// Only attempt cache lookup if caching is enabled and for objects without range/part requests // Only attempt cache lookup if caching is enabled and for objects without range/part requests
if manager.is_cache_enabled() && part_number.is_none() && range.is_none() { if manager.is_cache_enabled()
if let Some(cached) = manager.get_cached_object(&cache_key).await { && part_number.is_none()
&& range.is_none()
&& let Some(cached) = manager.get_cached_object(&cache_key).await
{
let cache_serve_duration = request_start.elapsed(); let cache_serve_duration = request_start.elapsed();
debug!("Serving object from response cache: {} (latency: {:?})", cache_key, cache_serve_duration); debug!("Serving object from response cache: {} (latency: {:?})", cache_key, cache_serve_duration);
@@ -1944,7 +1944,6 @@ impl S3 for FS {
let _ = helper.complete(&result); let _ = helper.complete(&result);
return result; return result;
} }
}
// TODO: getObjectInArchiveFileHandler object = xxx.zip/xxx/xxx.xxx // TODO: getObjectInArchiveFileHandler object = xxx.zip/xxx/xxx.xxx
@@ -1954,11 +1953,11 @@ impl S3 for FS {
let part_number = part_number.map(|v| v as usize); let part_number = part_number.map(|v| v as usize);
if let Some(part_num) = part_number { if let Some(part_num) = part_number
if part_num == 0 { && part_num == 0
{
return Err(s3_error!(InvalidArgument, "Invalid part number: part number must be greater than 0")); return Err(s3_error!(InvalidArgument, "Invalid part number: part number must be greater than 0"));
} }
}
let rs = range.map(|v| match v { let rs = range.map(|v| match v {
Range::Int { first, last } => HTTPRangeSpec { Range::Int { first, last } => HTTPRangeSpec {
@@ -2065,11 +2064,11 @@ impl S3 for FS {
let mut rs = rs; let mut rs = rs;
if let Some(part_number) = part_number { if let Some(part_number) = part_number
if rs.is_none() { && rs.is_none()
{
rs = HTTPRangeSpec::from_object_info(&info, part_number); rs = HTTPRangeSpec::from_object_info(&info, part_number);
} }
}
let mut content_length = info.get_actual_size().map_err(ApiError::from)?; let mut content_length = info.get_actual_size().map_err(ApiError::from)?;
@@ -2183,8 +2182,8 @@ impl S3 for FS {
} }
} }
if stored_sse_algorithm.is_none() { if stored_sse_algorithm.is_none()
if let Some((key_bytes, nonce, original_size)) = && let Some((key_bytes, nonce, original_size)) =
decrypt_managed_encryption_key(&bucket, &key, &info.user_defined).await? decrypt_managed_encryption_key(&bucket, &key, &info.user_defined).await?
{ {
if info.parts.len() > 1 { if info.parts.len() > 1 {
@@ -2201,7 +2200,6 @@ impl S3 for FS {
} }
managed_encryption_applied = true; managed_encryption_applied = true;
} }
}
// For SSE-C encrypted objects, use the original size instead of encrypted size // For SSE-C encrypted objects, use the original size instead of encrypted size
let response_content_length = if stored_sse_algorithm.is_some() { let response_content_length = if stored_sse_algorithm.is_some() {
@@ -2518,11 +2516,11 @@ impl S3 for FS {
let part_number = part_number.map(|v| v as usize); let part_number = part_number.map(|v| v as usize);
if let Some(part_num) = part_number { if let Some(part_num) = part_number
if part_num == 0 { && part_num == 0
{
return Err(s3_error!(InvalidArgument, "part_number invalid")); return Err(s3_error!(InvalidArgument, "part_number invalid"));
} }
}
let rs = range.map(|v| match v { let rs = range.map(|v| match v {
Range::Int { first, last } => HTTPRangeSpec { Range::Int { first, last } => HTTPRangeSpec {
@@ -2558,17 +2556,15 @@ impl S3 for FS {
return Err(S3Error::new(S3ErrorCode::MethodNotAllowed)); return Err(S3Error::new(S3ErrorCode::MethodNotAllowed));
} }
if let Some(match_etag) = if_none_match { if let Some(match_etag) = if_none_match
if let Some(strong_etag) = match_etag.into_etag() { && let Some(strong_etag) = match_etag.into_etag()
if info && info
.etag .etag
.as_ref() .as_ref()
.is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag) .is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag)
{ {
return Err(S3Error::new(S3ErrorCode::NotModified)); return Err(S3Error::new(S3ErrorCode::NotModified));
} }
}
}
if let Some(modified_since) = if_modified_since { if let Some(modified_since) = if_modified_since {
// obj_time < givenTime + 1s // obj_time < givenTime + 1s
@@ -2581,23 +2577,22 @@ impl S3 for FS {
} }
if let Some(match_etag) = if_match { if let Some(match_etag) = if_match {
if let Some(strong_etag) = match_etag.into_etag() { if let Some(strong_etag) = match_etag.into_etag()
if info && info
.etag .etag
.as_ref() .as_ref()
.is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag) .is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag)
{ {
return Err(S3Error::new(S3ErrorCode::PreconditionFailed)); return Err(S3Error::new(S3ErrorCode::PreconditionFailed));
} }
} } else if let Some(unmodified_since) = if_unmodified_since
} else if let Some(unmodified_since) = if_unmodified_since { && info.mod_time.is_some_and(|mod_time| {
if info.mod_time.is_some_and(|mod_time| {
let give_time: OffsetDateTime = unmodified_since.into(); let give_time: OffsetDateTime = unmodified_since.into();
mod_time > give_time.add(time::Duration::seconds(1)) mod_time > give_time.add(time::Duration::seconds(1))
}) { })
{
return Err(S3Error::new(S3ErrorCode::PreconditionFailed)); return Err(S3Error::new(S3ErrorCode::PreconditionFailed));
} }
}
let event_info = info.clone(); let event_info = info.clone();
let content_type = { let content_type = {
@@ -3080,11 +3075,11 @@ impl S3 for FS {
let input = req.input; let input = req.input;
// Save SSE-C parameters before moving input // Save SSE-C parameters before moving input
if let Some(ref storage_class) = input.storage_class { if let Some(ref storage_class) = input.storage_class
if !is_valid_storage_class(storage_class.as_str()) { && !is_valid_storage_class(storage_class.as_str())
{
return Err(s3_error!(InvalidStorageClass)); return Err(s3_error!(InvalidStorageClass));
} }
}
let PutObjectInput { let PutObjectInput {
body, body,
bucket, bucket,
@@ -3116,20 +3111,18 @@ impl S3 for FS {
match store.get_object_info(&bucket, &key, &ObjectOptions::default()).await { match store.get_object_info(&bucket, &key, &ObjectOptions::default()).await {
Ok(info) => { Ok(info) => {
if !info.delete_marker { if !info.delete_marker {
if let Some(ifmatch) = if_match { if let Some(ifmatch) = if_match
if let Some(strong_etag) = ifmatch.into_etag() { && let Some(strong_etag) = ifmatch.into_etag()
if info && info
.etag .etag
.as_ref() .as_ref()
.is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag) .is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag)
{ {
return Err(s3_error!(PreconditionFailed)); return Err(s3_error!(PreconditionFailed));
} }
} if let Some(ifnonematch) = if_none_match
} && let Some(strong_etag) = ifnonematch.into_etag()
if let Some(ifnonematch) = if_none_match { && info
if let Some(strong_etag) = ifnonematch.into_etag() {
if info
.etag .etag
.as_ref() .as_ref()
.is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag) .is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag)
@@ -3138,8 +3131,6 @@ impl S3 for FS {
} }
} }
} }
}
}
Err(err) => { Err(err) => {
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) { if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
return Err(ApiError::from(err).into()); return Err(ApiError::from(err).into());
@@ -3344,12 +3335,12 @@ impl S3 for FS {
} }
// Apply managed SSE (SSE-S3 or SSE-KMS) when requested // Apply managed SSE (SSE-S3 or SSE-KMS) when requested
if sse_customer_algorithm.is_none() { if sse_customer_algorithm.is_none()
if let Some(sse_alg) = &effective_sse { && let Some(sse_alg) = &effective_sse
if is_managed_sse(sse_alg) { && is_managed_sse(sse_alg)
{
let material = let material =
create_managed_encryption_material(&bucket, &key, sse_alg, effective_kms_key_id.clone(), actual_size) create_managed_encryption_material(&bucket, &key, sse_alg, effective_kms_key_id.clone(), actual_size).await?;
.await?;
let ManagedEncryptionMaterial { let ManagedEncryptionMaterial {
data_key, data_key,
@@ -3364,10 +3355,7 @@ impl S3 for FS {
effective_kms_key_id = Some(kms_key_used.clone()); effective_kms_key_id = Some(kms_key_used.clone());
let encrypt_reader = EncryptReader::new(reader, key_bytes, nonce); let encrypt_reader = EncryptReader::new(reader, key_bytes, nonce);
reader = reader = HashReader::new(Box::new(encrypt_reader), -1, actual_size, None, None, false).map_err(ApiError::from)?;
HashReader::new(Box::new(encrypt_reader), -1, actual_size, None, None, false).map_err(ApiError::from)?;
}
}
} }
let mut reader = PutObjReader::new(reader); let mut reader = PutObjReader::new(reader);
@@ -3428,8 +3416,8 @@ impl S3 for FS {
let mut checksum_sha256 = input.checksum_sha256; let mut checksum_sha256 = input.checksum_sha256;
let mut checksum_crc64nvme = input.checksum_crc64nvme; let mut checksum_crc64nvme = input.checksum_crc64nvme;
if let Some(alg) = &input.checksum_algorithm { if let Some(alg) = &input.checksum_algorithm
if let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| { && let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| {
let key = match alg.as_str() { let key = match alg.as_str() {
ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(), ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(),
ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(), ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(),
@@ -3443,7 +3431,8 @@ impl S3 for FS {
.get(key.unwrap_or_default()) .get(key.unwrap_or_default())
.and_then(|value| value.to_str().ok().map(|s| s.to_string())) .and_then(|value| value.to_str().ok().map(|s| s.to_string()))
}) })
}) { })
{
match alg.as_str() { match alg.as_str() {
ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str, ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str,
ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str, ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str,
@@ -3453,7 +3442,6 @@ impl S3 for FS {
_ => (), _ => (),
} }
} }
}
let output = PutObjectOutput { let output = PutObjectOutput {
e_tag, e_tag,
@@ -3495,11 +3483,11 @@ impl S3 for FS {
} = req.input.clone(); } = req.input.clone();
// Validate storage class if provided // Validate storage class if provided
if let Some(ref storage_class) = storage_class { if let Some(ref storage_class) = storage_class
if !is_valid_storage_class(storage_class.as_str()) { && !is_valid_storage_class(storage_class.as_str())
{
return Err(s3_error!(InvalidStorageClass)); return Err(s3_error!(InvalidStorageClass));
} }
}
// mc cp step 3 // mc cp step 3
@@ -3654,11 +3642,11 @@ impl S3 for FS {
let mut body_stream = body.ok_or_else(|| s3_error!(IncompleteBody))?; let mut body_stream = body.ok_or_else(|| s3_error!(IncompleteBody))?;
if size.is_none() { if size.is_none() {
if let Some(val) = req.headers.get(AMZ_DECODED_CONTENT_LENGTH) { if let Some(val) = req.headers.get(AMZ_DECODED_CONTENT_LENGTH)
if let Some(x) = atoi::atoi::<i64>(val.as_bytes()) { && let Some(x) = atoi::atoi::<i64>(val.as_bytes())
{
size = Some(x); size = Some(x);
} }
}
if size.is_none() { if size.is_none() {
let mut total = 0i64; let mut total = 0i64;
@@ -3828,8 +3816,8 @@ impl S3 for FS {
let mut checksum_sha256 = input.checksum_sha256; let mut checksum_sha256 = input.checksum_sha256;
let mut checksum_crc64nvme = input.checksum_crc64nvme; let mut checksum_crc64nvme = input.checksum_crc64nvme;
if let Some(alg) = &input.checksum_algorithm { if let Some(alg) = &input.checksum_algorithm
if let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| { && let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| {
let key = match alg.as_str() { let key = match alg.as_str() {
ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(), ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(),
ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(), ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(),
@@ -3843,7 +3831,8 @@ impl S3 for FS {
.get(key.unwrap_or_default()) .get(key.unwrap_or_default())
.and_then(|value| value.to_str().ok().map(|s| s.to_string())) .and_then(|value| value.to_str().ok().map(|s| s.to_string()))
}) })
}) { })
{
match alg.as_str() { match alg.as_str() {
ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str, ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str,
ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str, ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str,
@@ -3853,7 +3842,6 @@ impl S3 for FS {
_ => (), _ => (),
} }
} }
}
let output = UploadPartOutput { let output = UploadPartOutput {
checksum_crc32, checksum_crc32,
@@ -3949,16 +3937,14 @@ impl S3 for FS {
} }
} }
if let Some(if_none_match) = copy_source_if_none_match { if let Some(if_none_match) = copy_source_if_none_match
if let Some(ref etag) = src_info.etag { && let Some(ref etag) = src_info.etag
if let Some(strong_etag) = if_none_match.into_etag() { && let Some(strong_etag) = if_none_match.into_etag()
if ETag::Strong(etag.clone()) == strong_etag { && ETag::Strong(etag.clone()) == strong_etag
{
return Err(s3_error!(PreconditionFailed)); return Err(s3_error!(PreconditionFailed));
} }
}
// Weak ETag in If-None-Match is ignored (doesn't match) // Weak ETag in If-None-Match is ignored (doesn't match)
}
}
// TODO: Implement proper time comparison for if_modified_since and if_unmodified_since // TODO: Implement proper time comparison for if_modified_since and if_unmodified_since
// For now, we'll skip these conditions // For now, we'll skip these conditions
@@ -4157,11 +4143,11 @@ impl S3 for FS {
let max_uploads = max_uploads.map(|x| x as usize).unwrap_or(MAX_PARTS_COUNT); let max_uploads = max_uploads.map(|x| x as usize).unwrap_or(MAX_PARTS_COUNT);
if let Some(key_marker) = &key_marker { if let Some(key_marker) = &key_marker
if !key_marker.starts_with(prefix.as_str()) { && !key_marker.starts_with(prefix.as_str())
{
return Err(s3_error!(NotImplemented, "Invalid key marker")); return Err(s3_error!(NotImplemented, "Invalid key marker"));
} }
}
let result = store let result = store
.list_multipart_uploads(&bucket, &prefix, delimiter, key_marker, upload_id_marker, max_uploads) .list_multipart_uploads(&bucket, &prefix, delimiter, key_marker, upload_id_marker, max_uploads)
@@ -4227,20 +4213,18 @@ impl S3 for FS {
match store.get_object_info(&bucket, &key, &ObjectOptions::default()).await { match store.get_object_info(&bucket, &key, &ObjectOptions::default()).await {
Ok(info) => { Ok(info) => {
if !info.delete_marker { if !info.delete_marker {
if let Some(ifmatch) = if_match { if let Some(ifmatch) = if_match
if let Some(strong_etag) = ifmatch.into_etag() { && let Some(strong_etag) = ifmatch.into_etag()
if info && info
.etag .etag
.as_ref() .as_ref()
.is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag) .is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag)
{ {
return Err(s3_error!(PreconditionFailed)); return Err(s3_error!(PreconditionFailed));
} }
} if let Some(ifnonematch) = if_none_match
} && let Some(strong_etag) = ifnonematch.into_etag()
if let Some(ifnonematch) = if_none_match { && info
if let Some(strong_etag) = ifnonematch.into_etag() {
if info
.etag .etag
.as_ref() .as_ref()
.is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag) .is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag)
@@ -4249,8 +4233,6 @@ impl S3 for FS {
} }
} }
} }
}
}
Err(err) => { Err(err) => {
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) { if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
return Err(ApiError::from(err).into()); return Err(ApiError::from(err).into());
@@ -4852,12 +4834,12 @@ impl S3 for FS {
let Some(input_cfg) = lifecycle_configuration else { return Err(s3_error!(InvalidArgument)) }; let Some(input_cfg) = lifecycle_configuration else { return Err(s3_error!(InvalidArgument)) };
let rcfg = metadata_sys::get_object_lock_config(&bucket).await; let rcfg = metadata_sys::get_object_lock_config(&bucket).await;
if let Ok(rcfg) = rcfg { if let Ok(rcfg) = rcfg
if let Err(err) = input_cfg.validate(&rcfg.0).await { && let Err(err) = input_cfg.validate(&rcfg.0).await
{
//return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockValidateFailed".into()), err.to_string())); //return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockValidateFailed".into()), err.to_string()));
return Err(S3Error::with_message(S3ErrorCode::Custom("ValidateFailed".into()), err.to_string())); return Err(S3Error::with_message(S3ErrorCode::Custom("ValidateFailed".into()), err.to_string()));
} }
}
if let Err(err) = validate_transition_tier(&input_cfg).await { if let Err(err) = validate_transition_tier(&input_cfg).await {
//warn!("lifecycle_configuration add failed, err: {:?}", err); //warn!("lifecycle_configuration add failed, err: {:?}", err);
@@ -5735,8 +5717,9 @@ impl S3 for FS {
/// Auxiliary functions: extract prefixes and suffixes /// Auxiliary functions: extract prefixes and suffixes
fn extract_prefix_suffix(filter: Option<&NotificationConfigurationFilter>) -> (String, String) { fn extract_prefix_suffix(filter: Option<&NotificationConfigurationFilter>) -> (String, String) {
if let Some(filter) = filter { if let Some(filter) = filter
if let Some(filter_rules) = &filter.key { && let Some(filter_rules) = &filter.key
{
let mut prefix = String::new(); let mut prefix = String::new();
let mut suffix = String::new(); let mut suffix = String::new();
if let Some(rules) = &filter_rules.filter_rules { if let Some(rules) = &filter_rules.filter_rules {
@@ -5752,7 +5735,6 @@ fn extract_prefix_suffix(filter: Option<&NotificationConfigurationFilter>) -> (S
} }
return (prefix, suffix); return (prefix, suffix);
} }
}
(String::new(), String::new()) (String::new(), String::new())
} }
+6 -6
View File
@@ -86,11 +86,11 @@ impl OperationHelper {
.req_path(req.uri.path().to_string()) .req_path(req.uri.path().to_string())
.req_query(extract_req_params(req)); .req_query(extract_req_params(req));
if let Some(req_id) = req.headers.get("x-amz-request-id") { if let Some(req_id) = req.headers.get("x-amz-request-id")
if let Ok(id_str) = req_id.to_str() { && let Ok(id_str) = req_id.to_str()
{
audit_builder = audit_builder.request_id(id_str); audit_builder = audit_builder.request_id(id_str);
} }
}
// initialize event builder // initialize event builder
// object is a placeholder that must be set later using the `object()` method. // object is a placeholder that must be set later using the `object()` method.
@@ -194,8 +194,9 @@ impl Drop for OperationHelper {
} }
// Distribute event notification (only on success) // Distribute event notification (only on success)
if self.api_builder.0.status.as_deref() == Some("success") { if self.api_builder.0.status.as_deref() == Some("success")
if let Some(builder) = self.event_builder.take() { && let Some(builder) = self.event_builder.take()
{
let event_args = builder.build(); let event_args = builder.build();
// Avoid generating notifications for copy requests // Avoid generating notifications for copy requests
if !event_args.is_replication_request() { if !event_args.is_replication_request() {
@@ -206,4 +207,3 @@ impl Drop for OperationHelper {
} }
} }
} }
}
+13 -17
View File
@@ -64,14 +64,13 @@ pub async fn del_opts(
let vid = vid.map(|v| v.as_str().trim().to_owned()); let vid = vid.map(|v| v.as_str().trim().to_owned());
if let Some(ref id) = vid { if let Some(ref id) = vid
if *id != Uuid::nil().to_string() && *id != Uuid::nil().to_string()
&& let Err(err) = Uuid::parse_str(id.as_str()) && let Err(err) = Uuid::parse_str(id.as_str())
{ {
error!("del_opts: invalid version id: {} error: {}", id, err); error!("del_opts: invalid version id: {} error: {}", id, err);
return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone())); return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone()));
} }
}
let mut opts = put_opts_from_headers(headers, metadata.clone()).map_err(|err| { let mut opts = put_opts_from_headers(headers, metadata.clone()).map_err(|err| {
error!("del_opts: invalid argument: {} error: {}", object, err); error!("del_opts: invalid argument: {} error: {}", object, err);
@@ -111,13 +110,12 @@ pub async fn get_opts(
let vid = vid.map(|v| v.as_str().trim().to_owned()); let vid = vid.map(|v| v.as_str().trim().to_owned());
if let Some(ref id) = vid { if let Some(ref id) = vid
if *id != Uuid::nil().to_string() && *id != Uuid::nil().to_string()
&& let Err(_err) = Uuid::parse_str(id.as_str()) && let Err(_err) = Uuid::parse_str(id.as_str())
{ {
return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone())); return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone()));
} }
}
let mut opts = get_default_opts(headers, HashMap::new(), false) let mut opts = get_default_opts(headers, HashMap::new(), false)
.map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?; .map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?;
@@ -187,13 +185,12 @@ pub async fn put_opts(
let vid = vid.map(|v| v.as_str().trim().to_owned()); let vid = vid.map(|v| v.as_str().trim().to_owned());
if let Some(ref id) = vid { if let Some(ref id) = vid
if *id != Uuid::nil().to_string() && *id != Uuid::nil().to_string()
&& let Err(_err) = Uuid::parse_str(id.as_str()) && let Err(_err) = Uuid::parse_str(id.as_str())
{ {
return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone())); return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone()));
} }
}
let mut opts = put_opts_from_headers(headers, metadata) let mut opts = put_opts_from_headers(headers, metadata)
.map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?; .map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?;
@@ -512,13 +509,12 @@ fn skip_content_sha256_cksum(headers: &HeaderMap<HeaderValue>) -> bool {
// such broken clients and content-length > 0. // such broken clients and content-length > 0.
// For now, we'll assume strict compatibility is disabled // For now, we'll assume strict compatibility is disabled
// In a real implementation, you would check a global config // In a real implementation, you would check a global config
if let Some(content_length) = headers.get("content-length") { if let Some(content_length) = headers.get("content-length")
if let Ok(length_str) = content_length.to_str() { && let Ok(length_str) = content_length.to_str()
if let Ok(length) = length_str.parse::<i64>() { && let Ok(length) = length_str.parse::<i64>()
{
return length > 0; // && !global_server_ctxt.strict_s3_compat return length > 0; // && !global_server_ctxt.strict_s3_compat
} }
}
}
false false
} }
_ => false, _ => false,
@@ -546,11 +542,11 @@ fn get_content_sha256_cksum(headers: &HeaderMap<HeaderValue>, service_type: Serv
}; };
// We found 'X-Amz-Content-Sha256' return the captured value. // We found 'X-Amz-Content-Sha256' return the captured value.
if let Some(header_value) = content_sha256 { if let Some(header_value) = content_sha256
if let Ok(value) = header_value.to_str() { && let Ok(value) = header_value.to_str()
{
return value.to_string(); return value.to_string();
} }
}
// We couldn't find 'X-Amz-Content-Sha256'. // We couldn't find 'X-Amz-Content-Sha256'.
default_sha256_cksum default_sha256_cksum
+6 -6
View File
@@ -75,11 +75,11 @@ fn increment_version(version: &str) -> Result<String, Box<dyn std::error::Error>
let (major, minor, patch, pre_release) = parse_version(version)?; let (major, minor, patch, pre_release) = parse_version(version)?;
// If there's a pre-release identifier, increment the pre-release version number // If there's a pre-release identifier, increment the pre-release version number
if let Some(pre) = pre_release { if let Some(pre) = pre_release
if let Some(new_pre) = increment_pre_release(&pre) { && let Some(new_pre) = increment_pre_release(&pre)
{
return Ok(format!("{major}.{minor}.{patch}-{new_pre}")); return Ok(format!("{major}.{minor}.{patch}-{new_pre}"));
} }
}
// Otherwise increment patch version number // Otherwise increment patch version number
Ok(format!("{major}.{minor}.{}", patch + 1)) Ok(format!("{major}.{minor}.{}", patch + 1))
@@ -107,11 +107,11 @@ pub fn parse_version(version: &str) -> VersionParseResult {
fn increment_pre_release(pre_release: &str) -> Option<String> { fn increment_pre_release(pre_release: &str) -> Option<String> {
// Handle pre-release versions like "alpha.19" // Handle pre-release versions like "alpha.19"
let parts: Vec<&str> = pre_release.split('.').collect(); let parts: Vec<&str> = pre_release.split('.').collect();
if parts.len() == 2 { if parts.len() == 2
if let Ok(num) = parts[1].parse::<u32>() { && let Ok(num) = parts[1].parse::<u32>()
{
return Some(format!("{}.{}", parts[0], num + 1)); return Some(format!("{}.{}", parts[0], num + 1));
} }
}
// Handle pre-release versions like "alpha19" // Handle pre-release versions like "alpha19"
if let Some(pos) = pre_release.rfind(|c: char| c.is_alphabetic()) { if let Some(pos) = pre_release.rfind(|c: char| c.is_alphabetic()) {