mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-31 17:28:12 +00:00
chore: upgrade dependencies and migrate to aws-lc-rs (#1333)
This commit is contained in:
Generated
+129
-245
File diff suppressed because it is too large
Load Diff
+7
-7
@@ -50,7 +50,7 @@ resolver = "2"
|
|||||||
edition = "2024"
|
edition = "2024"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
repository = "https://github.com/rustfs/rustfs"
|
repository = "https://github.com/rustfs/rustfs"
|
||||||
rust-version = "1.85"
|
rust-version = "1.88"
|
||||||
version = "0.0.5"
|
version = "0.0.5"
|
||||||
homepage = "https://rustfs.com"
|
homepage = "https://rustfs.com"
|
||||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||||
@@ -100,21 +100,21 @@ async-compression = { version = "0.4.19" }
|
|||||||
async-recursion = "1.1.1"
|
async-recursion = "1.1.1"
|
||||||
async-trait = "0.1.89"
|
async-trait = "0.1.89"
|
||||||
axum = "0.8.8"
|
axum = "0.8.8"
|
||||||
axum-server = { version = "0.8.0", features = ["tls-rustls-no-provider"], default-features = false }
|
axum-server = { version = "0.8.0", features = ["tls-rustls"], default-features = false }
|
||||||
futures = "0.3.31"
|
futures = "0.3.31"
|
||||||
futures-core = "0.3.31"
|
futures-core = "0.3.31"
|
||||||
futures-util = "0.3.31"
|
futures-util = "0.3.31"
|
||||||
pollster = "0.4.0"
|
pollster = "0.4.0"
|
||||||
hyper = { version = "1.8.1", features = ["http2", "http1", "server"] }
|
hyper = { version = "1.8.1", features = ["http2", "http1", "server"] }
|
||||||
hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "ring", "webpki-roots"] }
|
hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs", "webpki-roots"] }
|
||||||
hyper-util = { version = "0.1.19", features = ["tokio", "server-auto", "server-graceful"] }
|
hyper-util = { version = "0.1.19", features = ["tokio", "server-auto", "server-graceful"] }
|
||||||
http = "1.4.0"
|
http = "1.4.0"
|
||||||
http-body = "1.0.1"
|
http-body = "1.0.1"
|
||||||
http-body-util = "0.1.3"
|
http-body-util = "0.1.3"
|
||||||
reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2", "system-proxy", "stream", "json", "blocking"] }
|
reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls-no-provider", "charset", "http2", "system-proxy", "stream", "json", "blocking"] }
|
||||||
socket2 = "0.6.1"
|
socket2 = "0.6.1"
|
||||||
tokio = { version = "1.48.0", features = ["fs", "rt-multi-thread"] }
|
tokio = { version = "1.48.0", features = ["fs", "rt-multi-thread"] }
|
||||||
tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "ring"] }
|
tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "aws-lc-rs"] }
|
||||||
tokio-stream = { version = "0.1.17" }
|
tokio-stream = { version = "0.1.17" }
|
||||||
tokio-test = "0.4.4"
|
tokio-test = "0.4.4"
|
||||||
tokio-util = { version = "0.7.17", features = ["io", "compat"] }
|
tokio-util = { version = "0.7.17", features = ["io", "compat"] }
|
||||||
@@ -150,7 +150,7 @@ hmac = { version = "0.13.0-rc.3" }
|
|||||||
jsonwebtoken = { version = "10.2.0", features = ["rust_crypto"] }
|
jsonwebtoken = { version = "10.2.0", features = ["rust_crypto"] }
|
||||||
pbkdf2 = "0.13.0-rc.5"
|
pbkdf2 = "0.13.0-rc.5"
|
||||||
rsa = { version = "0.10.0-rc.10" }
|
rsa = { version = "0.10.0-rc.10" }
|
||||||
rustls = { version = "0.23.35", features = ["ring", "logging", "std", "tls12"], default-features = false }
|
rustls = { version = "0.23.35" }
|
||||||
rustls-pemfile = "2.2.0"
|
rustls-pemfile = "2.2.0"
|
||||||
rustls-pki-types = "1.13.2"
|
rustls-pki-types = "1.13.2"
|
||||||
sha1 = "0.11.0-rc.3"
|
sha1 = "0.11.0-rc.3"
|
||||||
@@ -171,7 +171,7 @@ atoi = "2.0.0"
|
|||||||
atomic_enum = "0.3.0"
|
atomic_enum = "0.3.0"
|
||||||
aws-config = { version = "1.8.12" }
|
aws-config = { version = "1.8.12" }
|
||||||
aws-credential-types = { version = "1.2.11" }
|
aws-credential-types = { version = "1.2.11" }
|
||||||
aws-sdk-s3 = { version = "1.119.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] }
|
aws-sdk-s3 = { version = "1.119.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||||
aws-smithy-types = { version = "1.3.5" }
|
aws-smithy-types = { version = "1.3.5" }
|
||||||
base64 = "0.22.1"
|
base64 = "0.22.1"
|
||||||
base64-simd = "0.8.0"
|
base64-simd = "0.8.0"
|
||||||
|
|||||||
@@ -348,7 +348,7 @@ impl ErasureSetHealer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// save checkpoint periodically
|
// save checkpoint periodically
|
||||||
if global_obj_idx % 100 == 0 {
|
if global_obj_idx.is_multiple_of(100) {
|
||||||
checkpoint_manager
|
checkpoint_manager
|
||||||
.update_position(bucket_index, *current_object_index)
|
.update_position(bucket_index, *current_object_index)
|
||||||
.await?;
|
.await?;
|
||||||
|
|||||||
@@ -492,14 +492,13 @@ impl HealManager {
|
|||||||
for (_, disk_opt) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() {
|
for (_, disk_opt) in GLOBAL_LOCAL_DISK_MAP.read().await.iter() {
|
||||||
if let Some(disk) = disk_opt {
|
if let Some(disk) = disk_opt {
|
||||||
// detect unformatted disk via get_disk_id()
|
// detect unformatted disk via get_disk_id()
|
||||||
if let Err(err) = disk.get_disk_id().await {
|
if let Err(err) = disk.get_disk_id().await
|
||||||
if err == DiskError::UnformattedDisk {
|
&& err == DiskError::UnformattedDisk {
|
||||||
endpoints.push(disk.endpoint());
|
endpoints.push(disk.endpoint());
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if endpoints.is_empty() {
|
if endpoints.is_empty() {
|
||||||
info!("start_auto_disk_scanner: No endpoints need healing");
|
info!("start_auto_disk_scanner: No endpoints need healing");
|
||||||
|
|||||||
@@ -541,13 +541,13 @@ impl ResumeUtils {
|
|||||||
for entry in entries {
|
for entry in entries {
|
||||||
if entry.ends_with(&format!("_{RESUME_STATE_FILE}")) {
|
if entry.ends_with(&format!("_{RESUME_STATE_FILE}")) {
|
||||||
// Extract task ID from filename: {task_id}_ahm_resume_state.json
|
// Extract task ID from filename: {task_id}_ahm_resume_state.json
|
||||||
if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}")) {
|
if let Some(task_id) = entry.strip_suffix(&format!("_{RESUME_STATE_FILE}"))
|
||||||
if !task_id.is_empty() {
|
&& !task_id.is_empty()
|
||||||
|
{
|
||||||
task_ids.push(task_id.to_string());
|
task_ids.push(task_id.to_string());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
debug!("Found {} resumable tasks: {:?}", task_ids.len(), task_ids);
|
debug!("Found {} resumable tasks: {:?}", task_ids.len(), task_ids);
|
||||||
Ok(task_ids)
|
Ok(task_ids)
|
||||||
|
|||||||
@@ -83,11 +83,11 @@ pub struct CheckpointManager {
|
|||||||
|
|
||||||
impl CheckpointManager {
|
impl CheckpointManager {
|
||||||
pub fn new(node_id: &str, data_dir: &Path) -> Self {
|
pub fn new(node_id: &str, data_dir: &Path) -> Self {
|
||||||
if !data_dir.exists() {
|
if !data_dir.exists()
|
||||||
if let Err(e) = std::fs::create_dir_all(data_dir) {
|
&& let Err(e) = std::fs::create_dir_all(data_dir)
|
||||||
|
{
|
||||||
error!("create data dir failed {:?}: {}", data_dir, e);
|
error!("create data dir failed {:?}: {}", data_dir, e);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let checkpoint_file = data_dir.join(format!("scanner_checkpoint_{node_id}.json"));
|
let checkpoint_file = data_dir.join(format!("scanner_checkpoint_{node_id}.json"));
|
||||||
let backup_file = data_dir.join(format!("scanner_checkpoint_{node_id}.backup"));
|
let backup_file = data_dir.join(format!("scanner_checkpoint_{node_id}.backup"));
|
||||||
|
|||||||
@@ -401,11 +401,11 @@ impl Scanner {
|
|||||||
let mut latest_update: Option<SystemTime> = None;
|
let mut latest_update: Option<SystemTime> = None;
|
||||||
|
|
||||||
for snapshot in &outcome.snapshots {
|
for snapshot in &outcome.snapshots {
|
||||||
if let Some(update) = snapshot.last_update {
|
if let Some(update) = snapshot.last_update
|
||||||
if latest_update.is_none_or(|current| update > current) {
|
&& latest_update.is_none_or(|current| update > current)
|
||||||
|
{
|
||||||
latest_update = Some(update);
|
latest_update = Some(update);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
aggregated.objects_total_count = aggregated.objects_total_count.saturating_add(snapshot.objects_total_count);
|
aggregated.objects_total_count = aggregated.objects_total_count.saturating_add(snapshot.objects_total_count);
|
||||||
aggregated.versions_total_count = aggregated.versions_total_count.saturating_add(snapshot.versions_total_count);
|
aggregated.versions_total_count = aggregated.versions_total_count.saturating_add(snapshot.versions_total_count);
|
||||||
@@ -527,28 +527,20 @@ impl Scanner {
|
|||||||
let (disks, _) = set_disks.get_online_disks_with_healing(false).await;
|
let (disks, _) = set_disks.get_online_disks_with_healing(false).await;
|
||||||
if let Some(disk) = disks.first() {
|
if let Some(disk) = disks.first() {
|
||||||
let bucket_path = disk.path().join(bucket_name);
|
let bucket_path = disk.path().join(bucket_name);
|
||||||
if bucket_path.exists() {
|
if bucket_path.exists()
|
||||||
if let Ok(entries) = std::fs::read_dir(&bucket_path) {
|
&& let Ok(entries) = std::fs::read_dir(&bucket_path)
|
||||||
|
{
|
||||||
for entry in entries.flatten() {
|
for entry in entries.flatten() {
|
||||||
if let Ok(file_type) = entry.file_type() {
|
if let Ok(file_type) = entry.file_type()
|
||||||
if file_type.is_dir() {
|
&& file_type.is_dir()
|
||||||
if let Some(object_name) = entry.file_name().to_str() {
|
&& let Some(object_name) = entry.file_name().to_str()
|
||||||
if !object_name.starts_with('.') {
|
&& !object_name.starts_with('.')
|
||||||
|
{
|
||||||
debug!("Deep scanning object: {}/{}", bucket_name, object_name);
|
debug!("Deep scanning object: {}/{}", bucket_name, object_name);
|
||||||
if let Err(e) = self.verify_object_integrity(bucket_name, object_name).await {
|
if let Err(e) = self.verify_object_integrity(bucket_name, object_name).await {
|
||||||
warn!(
|
warn!("Object integrity verification failed for {}/{}: {}", bucket_name, object_name, e);
|
||||||
"Object integrity verification failed for {}/{}: {}",
|
|
||||||
bucket_name, object_name, e
|
|
||||||
);
|
|
||||||
} else {
|
} else {
|
||||||
debug!(
|
debug!("Object integrity verification passed for {}/{}", bucket_name, object_name);
|
||||||
"Object integrity verification passed for {}/{}",
|
|
||||||
bucket_name, object_name
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -859,11 +851,11 @@ impl Scanner {
|
|||||||
|
|
||||||
// Phase 2: Minimal EC verification for critical objects only
|
// Phase 2: Minimal EC verification for critical objects only
|
||||||
// Note: The main scanning is now handled by NodeScanner in the background
|
// Note: The main scanning is now handled by NodeScanner in the background
|
||||||
if let Some(ecstore) = rustfs_ecstore::new_object_layer_fn() {
|
if let Some(ecstore) = rustfs_ecstore::new_object_layer_fn()
|
||||||
if let Err(e) = self.minimal_ec_verification(&ecstore).await {
|
&& let Err(e) = self.minimal_ec_verification(&ecstore).await
|
||||||
|
{
|
||||||
error!("Minimal EC verification failed: {}", e);
|
error!("Minimal EC verification failed: {}", e);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Update scan duration
|
// Update scan duration
|
||||||
let scan_duration = SystemTime::now().duration_since(start_time).unwrap_or(Duration::ZERO);
|
let scan_duration = SystemTime::now().duration_since(start_time).unwrap_or(Duration::ZERO);
|
||||||
@@ -950,14 +942,13 @@ impl Scanner {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// If there is still no data, try backend before persisting zeros
|
// If there is still no data, try backend before persisting zeros
|
||||||
if data_usage.buckets_usage.is_empty() {
|
if data_usage.buckets_usage.is_empty()
|
||||||
if let Ok(existing) = rustfs_ecstore::data_usage::load_data_usage_from_backend(ecstore.clone()).await {
|
&& let Ok(existing) = rustfs_ecstore::data_usage::load_data_usage_from_backend(ecstore.clone()).await
|
||||||
if !existing.buckets_usage.is_empty() {
|
&& !existing.buckets_usage.is_empty()
|
||||||
|
{
|
||||||
info!("Using existing backend data usage during fallback backoff");
|
info!("Using existing backend data usage during fallback backoff");
|
||||||
data_usage = existing;
|
data_usage = existing;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Avoid overwriting valid backend stats with zeros when fallback is throttled
|
// Avoid overwriting valid backend stats with zeros when fallback is throttled
|
||||||
if data_usage.buckets_usage.is_empty() {
|
if data_usage.buckets_usage.is_empty() {
|
||||||
@@ -1721,8 +1712,7 @@ impl Scanner {
|
|||||||
// check disk status, if offline, submit erasure set heal task
|
// check disk status, if offline, submit erasure set heal task
|
||||||
if !metrics.is_online {
|
if !metrics.is_online {
|
||||||
let enable_healing = self.config.read().await.enable_healing;
|
let enable_healing = self.config.read().await.enable_healing;
|
||||||
if enable_healing {
|
if enable_healing && let Some(heal_manager) = &self.heal_manager {
|
||||||
if let Some(heal_manager) = &self.heal_manager {
|
|
||||||
// Get bucket list for erasure set healing
|
// Get bucket list for erasure set healing
|
||||||
let buckets = match rustfs_ecstore::new_object_layer_fn() {
|
let buckets = match rustfs_ecstore::new_object_layer_fn() {
|
||||||
Some(ecstore) => match ecstore.list_bucket(&ecstore::store_api::BucketOptions::default()).await {
|
Some(ecstore) => match ecstore.list_bucket(&ecstore::store_api::BucketOptions::default()).await {
|
||||||
@@ -1754,7 +1744,6 @@ impl Scanner {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Additional disk info for debugging
|
// Additional disk info for debugging
|
||||||
debug!(
|
debug!(
|
||||||
@@ -1778,8 +1767,7 @@ impl Scanner {
|
|||||||
|
|
||||||
// disk access failed, submit erasure set heal task
|
// disk access failed, submit erasure set heal task
|
||||||
let enable_healing = self.config.read().await.enable_healing;
|
let enable_healing = self.config.read().await.enable_healing;
|
||||||
if enable_healing {
|
if enable_healing && let Some(heal_manager) = &self.heal_manager {
|
||||||
if let Some(heal_manager) = &self.heal_manager {
|
|
||||||
// Get bucket list for erasure set healing
|
// Get bucket list for erasure set healing
|
||||||
let buckets = match rustfs_ecstore::new_object_layer_fn() {
|
let buckets = match rustfs_ecstore::new_object_layer_fn() {
|
||||||
Some(ecstore) => match ecstore.list_bucket(&ecstore::store_api::BucketOptions::default()).await {
|
Some(ecstore) => match ecstore.list_bucket(&ecstore::store_api::BucketOptions::default()).await {
|
||||||
@@ -1810,7 +1798,6 @@ impl Scanner {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return Err(Error::Storage(e.into()));
|
return Err(Error::Storage(e.into()));
|
||||||
}
|
}
|
||||||
@@ -1820,12 +1807,12 @@ impl Scanner {
|
|||||||
let mut disk_objects = HashMap::new();
|
let mut disk_objects = HashMap::new();
|
||||||
for volume in volumes {
|
for volume in volumes {
|
||||||
// check cancel token
|
// check cancel token
|
||||||
if let Some(cancel_token) = get_ahm_services_cancel_token() {
|
if let Some(cancel_token) = get_ahm_services_cancel_token()
|
||||||
if cancel_token.is_cancelled() {
|
&& cancel_token.is_cancelled()
|
||||||
|
{
|
||||||
info!("Cancellation requested, stopping disk scan");
|
info!("Cancellation requested, stopping disk scan");
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
match self.scan_volume(disk, &volume.name).await {
|
match self.scan_volume(disk, &volume.name).await {
|
||||||
Ok(object_metadata) => {
|
Ok(object_metadata) => {
|
||||||
@@ -1955,29 +1942,22 @@ impl Scanner {
|
|||||||
|
|
||||||
// object metadata damaged, submit metadata heal task
|
// object metadata damaged, submit metadata heal task
|
||||||
let enable_healing = self.config.read().await.enable_healing;
|
let enable_healing = self.config.read().await.enable_healing;
|
||||||
if enable_healing {
|
if enable_healing && let Some(heal_manager) = &self.heal_manager {
|
||||||
if let Some(heal_manager) = &self.heal_manager {
|
|
||||||
let req = HealRequest::metadata(bucket.to_string(), entry.name.clone());
|
let req = HealRequest::metadata(bucket.to_string(), entry.name.clone());
|
||||||
match heal_manager.submit_heal_request(req).await {
|
match heal_manager.submit_heal_request(req).await {
|
||||||
Ok(task_id) => {
|
Ok(task_id) => {
|
||||||
warn!(
|
warn!("object metadata damaged, submit heal task: {} {} / {}", task_id, bucket, entry.name);
|
||||||
"object metadata damaged, submit heal task: {} {} / {}",
|
|
||||||
task_id, bucket, entry.name
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!(
|
error!("object metadata damaged, submit heal task failed: {} / {} {}", bucket, entry.name, e);
|
||||||
"object metadata damaged, submit heal task failed: {} / {} {}",
|
|
||||||
bucket, entry.name, e
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Apply lifecycle actions
|
// Apply lifecycle actions
|
||||||
if let Some(lifecycle_config) = &lifecycle_config {
|
if let Some(lifecycle_config) = &lifecycle_config
|
||||||
if disk.is_local() {
|
&& disk.is_local()
|
||||||
|
{
|
||||||
let vcfg = BucketVersioningSys::get(bucket).await.ok();
|
let vcfg = BucketVersioningSys::get(bucket).await.ok();
|
||||||
|
|
||||||
let mut scanner_item = ScannerItem {
|
let mut scanner_item = ScannerItem {
|
||||||
@@ -2053,7 +2033,6 @@ impl Scanner {
|
|||||||
return Err(Error::other(ERR_IGNORE_FILE_CONTRIB).into());
|
return Err(Error::other(ERR_IGNORE_FILE_CONTRIB).into());
|
||||||
}*/
|
}*/
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Store object metadata for later analysis
|
// Store object metadata for later analysis
|
||||||
object_metadata.insert(entry.name.clone(), file_meta.clone());
|
object_metadata.insert(entry.name.clone(), file_meta.clone());
|
||||||
@@ -2064,15 +2043,11 @@ impl Scanner {
|
|||||||
|
|
||||||
// object metadata parse failed, submit metadata heal task
|
// object metadata parse failed, submit metadata heal task
|
||||||
let enable_healing = self.config.read().await.enable_healing;
|
let enable_healing = self.config.read().await.enable_healing;
|
||||||
if enable_healing {
|
if enable_healing && let Some(heal_manager) = &self.heal_manager {
|
||||||
if let Some(heal_manager) = &self.heal_manager {
|
|
||||||
let req = HealRequest::metadata(bucket.to_string(), entry.name.clone());
|
let req = HealRequest::metadata(bucket.to_string(), entry.name.clone());
|
||||||
match heal_manager.submit_heal_request(req).await {
|
match heal_manager.submit_heal_request(req).await {
|
||||||
Ok(task_id) => {
|
Ok(task_id) => {
|
||||||
warn!(
|
warn!("object metadata parse failed, submit heal task: {} {} / {}", task_id, bucket, entry.name);
|
||||||
"object metadata parse failed, submit heal task: {} {} / {}",
|
|
||||||
task_id, bucket, entry.name
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!(
|
error!(
|
||||||
@@ -2085,7 +2060,6 @@ impl Scanner {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Update metrics
|
// Update metrics
|
||||||
self.metrics.increment_objects_scanned(objects_scanned);
|
self.metrics.increment_objects_scanned(objects_scanned);
|
||||||
@@ -2190,19 +2164,16 @@ impl Scanner {
|
|||||||
// the delete marker, but we keep it conservative here.
|
// the delete marker, but we keep it conservative here.
|
||||||
let mut has_latest_delete_marker = false;
|
let mut has_latest_delete_marker = false;
|
||||||
for &disk_idx in locations {
|
for &disk_idx in locations {
|
||||||
if let Some(bucket_map) = all_disk_objects.get(disk_idx) {
|
if let Some(bucket_map) = all_disk_objects.get(disk_idx)
|
||||||
if let Some(file_map) = bucket_map.get(bucket) {
|
&& let Some(file_map) = bucket_map.get(bucket)
|
||||||
if let Some(fm) = file_map.get(object_name) {
|
&& let Some(fm) = file_map.get(object_name)
|
||||||
if let Some(first_ver) = fm.versions.first() {
|
&& let Some(first_ver) = fm.versions.first()
|
||||||
if first_ver.header.version_type == VersionType::Delete {
|
&& first_ver.header.version_type == VersionType::Delete
|
||||||
|
{
|
||||||
has_latest_delete_marker = true;
|
has_latest_delete_marker = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if has_latest_delete_marker {
|
if has_latest_delete_marker {
|
||||||
debug!(
|
debug!(
|
||||||
"Object {}/{} is a delete marker on some disk(s), skipping heal for missing parts",
|
"Object {}/{} is a delete marker on some disk(s), skipping heal for missing parts",
|
||||||
@@ -2248,8 +2219,7 @@ impl Scanner {
|
|||||||
|
|
||||||
// submit heal task
|
// submit heal task
|
||||||
let enable_healing = self.config.read().await.enable_healing;
|
let enable_healing = self.config.read().await.enable_healing;
|
||||||
if enable_healing {
|
if enable_healing && let Some(heal_manager) = &self.heal_manager {
|
||||||
if let Some(heal_manager) = &self.heal_manager {
|
|
||||||
use crate::heal::{HealPriority, HealRequest};
|
use crate::heal::{HealPriority, HealRequest};
|
||||||
let req = HealRequest::new(
|
let req = HealRequest::new(
|
||||||
crate::heal::HealType::Object {
|
crate::heal::HealType::Object {
|
||||||
@@ -2273,18 +2243,17 @@ impl Scanner {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Step 3: Deep scan EC verification
|
// Step 3: Deep scan EC verification
|
||||||
let config = self.config.read().await;
|
let config = self.config.read().await;
|
||||||
if config.scan_mode == ScanMode::Deep {
|
if config.scan_mode == ScanMode::Deep
|
||||||
if let Err(e) = self.verify_object_integrity(bucket, object_name).await {
|
&& let Err(e) = self.verify_object_integrity(bucket, object_name).await
|
||||||
|
{
|
||||||
objects_with_ec_issues += 1;
|
objects_with_ec_issues += 1;
|
||||||
warn!("Object integrity verification failed for object {}/{}: {}", bucket, object_name, e);
|
warn!("Object integrity verification failed for object {}/{}: {}", bucket, object_name, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
info!(
|
info!(
|
||||||
"Analysis complete: {} objects need healing, {} objects have EC issues",
|
"Analysis complete: {} objects need healing, {} objects have EC issues",
|
||||||
@@ -2293,11 +2262,11 @@ impl Scanner {
|
|||||||
|
|
||||||
// Step 4: Collect data usage statistics if enabled
|
// Step 4: Collect data usage statistics if enabled
|
||||||
let config = self.config.read().await;
|
let config = self.config.read().await;
|
||||||
if config.enable_data_usage_stats {
|
if config.enable_data_usage_stats
|
||||||
if let Err(e) = self.collect_data_usage_statistics(all_disk_objects).await {
|
&& let Err(e) = self.collect_data_usage_statistics(all_disk_objects).await
|
||||||
|
{
|
||||||
error!("Failed to collect data usage statistics: {}", e);
|
error!("Failed to collect data usage statistics: {}", e);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
drop(config);
|
drop(config);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -2526,23 +2495,21 @@ impl Scanner {
|
|||||||
info!("Starting legacy scan loop for backward compatibility");
|
info!("Starting legacy scan loop for backward compatibility");
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
if let Some(token) = get_ahm_services_cancel_token() {
|
if let Some(token) = get_ahm_services_cancel_token()
|
||||||
if token.is_cancelled() {
|
&& token.is_cancelled()
|
||||||
|
{
|
||||||
info!("Cancellation requested, exiting legacy scan loop");
|
info!("Cancellation requested, exiting legacy scan loop");
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let (enable_data_usage_stats, scan_interval) = {
|
let (enable_data_usage_stats, scan_interval) = {
|
||||||
let config = self.config.read().await;
|
let config = self.config.read().await;
|
||||||
(config.enable_data_usage_stats, config.scan_interval)
|
(config.enable_data_usage_stats, config.scan_interval)
|
||||||
};
|
};
|
||||||
|
|
||||||
if enable_data_usage_stats {
|
if enable_data_usage_stats && let Err(e) = self.collect_and_persist_data_usage().await {
|
||||||
if let Err(e) = self.collect_and_persist_data_usage().await {
|
|
||||||
warn!("Background data usage collection failed: {}", e);
|
warn!("Background data usage collection failed: {}", e);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Update local stats in aggregator after latest scan
|
// Update local stats in aggregator after latest scan
|
||||||
let local_stats = self.node_scanner.get_stats_summary().await;
|
let local_stats = self.node_scanner.get_stats_summary().await;
|
||||||
@@ -2656,11 +2623,11 @@ mod tests {
|
|||||||
// create temp dir as 4 disks
|
// create temp dir as 4 disks
|
||||||
let test_base_dir = test_dir.unwrap_or("/tmp/rustfs_ahm_test");
|
let test_base_dir = test_dir.unwrap_or("/tmp/rustfs_ahm_test");
|
||||||
let temp_dir = std::path::PathBuf::from(test_base_dir);
|
let temp_dir = std::path::PathBuf::from(test_base_dir);
|
||||||
if temp_dir.exists() {
|
if temp_dir.exists()
|
||||||
if let Err(e) = fs::remove_dir_all(&temp_dir) {
|
&& let Err(e) = fs::remove_dir_all(&temp_dir)
|
||||||
|
{
|
||||||
panic!("Failed to remove test directory: {e}");
|
panic!("Failed to remove test directory: {e}");
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if let Err(e) = fs::create_dir_all(&temp_dir) {
|
if let Err(e) = fs::create_dir_all(&temp_dir) {
|
||||||
panic!("Failed to create test directory: {e}");
|
panic!("Failed to create test directory: {e}");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -305,13 +305,13 @@ fn compute_object_usage(bucket: &str, object: &str, file_meta: &FileMeta) -> Res
|
|||||||
has_live_object = true;
|
has_live_object = true;
|
||||||
versions_count = versions_count.saturating_add(1);
|
versions_count = versions_count.saturating_add(1);
|
||||||
|
|
||||||
if latest_file_info.is_none() {
|
if latest_file_info.is_none()
|
||||||
if let Ok(info) = file_meta.into_fileinfo(bucket, object, "", false, false) {
|
&& let Ok(info) = file_meta.into_fileinfo(bucket, object, "", false, false)
|
||||||
|
{
|
||||||
latest_file_info = Some(info);
|
latest_file_info = Some(info);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
VersionType::Delete => {
|
VersionType::Delete => {
|
||||||
delete_markers_count = delete_markers_count.saturating_add(1);
|
delete_markers_count = delete_markers_count.saturating_add(1);
|
||||||
versions_count = versions_count.saturating_add(1);
|
versions_count = versions_count.saturating_add(1);
|
||||||
|
|||||||
@@ -112,11 +112,11 @@ impl LocalStatsManager {
|
|||||||
/// create new local stats manager
|
/// create new local stats manager
|
||||||
pub fn new(node_id: &str, data_dir: &Path) -> Self {
|
pub fn new(node_id: &str, data_dir: &Path) -> Self {
|
||||||
// ensure data directory exists
|
// ensure data directory exists
|
||||||
if !data_dir.exists() {
|
if !data_dir.exists()
|
||||||
if let Err(e) = std::fs::create_dir_all(data_dir) {
|
&& let Err(e) = std::fs::create_dir_all(data_dir)
|
||||||
|
{
|
||||||
error!("create stats data directory failed {:?}: {}", data_dir, e);
|
error!("create stats data directory failed {:?}: {}", data_dir, e);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let stats_file = data_dir.join(format!("scanner_stats_{node_id}.json"));
|
let stats_file = data_dir.join(format!("scanner_stats_{node_id}.json"));
|
||||||
let backup_file = data_dir.join(format!("scanner_stats_{node_id}.backup"));
|
let backup_file = data_dir.join(format!("scanner_stats_{node_id}.backup"));
|
||||||
|
|||||||
@@ -436,11 +436,11 @@ impl NodeScanner {
|
|||||||
/// create a new node scanner
|
/// create a new node scanner
|
||||||
pub fn new(node_id: String, config: NodeScannerConfig) -> Self {
|
pub fn new(node_id: String, config: NodeScannerConfig) -> Self {
|
||||||
// Ensure data directory exists
|
// Ensure data directory exists
|
||||||
if !config.data_dir.exists() {
|
if !config.data_dir.exists()
|
||||||
if let Err(e) = std::fs::create_dir_all(&config.data_dir) {
|
&& let Err(e) = std::fs::create_dir_all(&config.data_dir)
|
||||||
|
{
|
||||||
error!("create data directory failed {:?}: {}", config.data_dir, e);
|
error!("create data directory failed {:?}: {}", config.data_dir, e);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let stats_manager = Arc::new(LocalStatsManager::new(&node_id, &config.data_dir));
|
let stats_manager = Arc::new(LocalStatsManager::new(&node_id, &config.data_dir));
|
||||||
|
|
||||||
|
|||||||
@@ -327,8 +327,9 @@ impl DecentralizedStatsAggregator {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Check cache validity if timestamp is not initial value (UNIX_EPOCH)
|
// Check cache validity if timestamp is not initial value (UNIX_EPOCH)
|
||||||
if cache_timestamp != SystemTime::UNIX_EPOCH {
|
if cache_timestamp != SystemTime::UNIX_EPOCH
|
||||||
if let Ok(elapsed) = now.duration_since(cache_timestamp) {
|
&& let Ok(elapsed) = now.duration_since(cache_timestamp)
|
||||||
|
{
|
||||||
if elapsed < cache_ttl {
|
if elapsed < cache_ttl {
|
||||||
if let Some(cached) = self.cached_stats.read().await.as_ref() {
|
if let Some(cached) = self.cached_stats.read().await.as_ref() {
|
||||||
debug!("Returning cached aggregated stats, remaining TTL: {:?}", cache_ttl - elapsed);
|
debug!("Returning cached aggregated stats, remaining TTL: {:?}", cache_ttl - elapsed);
|
||||||
@@ -338,7 +339,6 @@ impl DecentralizedStatsAggregator {
|
|||||||
debug!("Cache expired: elapsed={:?} >= ttl={:?}", elapsed, cache_ttl);
|
debug!("Cache expired: elapsed={:?} >= ttl={:?}", elapsed, cache_ttl);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// cache expired, re-aggregate
|
// cache expired, re-aggregate
|
||||||
info!("cache expired, start re-aggregating stats data");
|
info!("cache expired, start re-aggregating stats data");
|
||||||
|
|||||||
@@ -421,8 +421,9 @@ mod serial_tests {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(lmdb_env) = GLOBAL_LMDB_ENV.get() {
|
if let Some(lmdb_env) = GLOBAL_LMDB_ENV.get()
|
||||||
if let Some(lmdb) = GLOBAL_LMDB_DB.get() {
|
&& let Some(lmdb) = GLOBAL_LMDB_DB.get()
|
||||||
|
{
|
||||||
let mut wtxn = lmdb_env.write_txn().unwrap();
|
let mut wtxn = lmdb_env.write_txn().unwrap();
|
||||||
|
|
||||||
/*if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await {
|
/*if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await {
|
||||||
@@ -501,7 +502,6 @@ mod serial_tests {
|
|||||||
//drop(iter);
|
//drop(iter);
|
||||||
wtxn.commit().unwrap();
|
wtxn.commit().unwrap();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
println!("Lifecycle cache test completed");
|
println!("Lifecycle cache test completed");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -415,8 +415,8 @@ mod serial_tests {
|
|||||||
.await;
|
.await;
|
||||||
println!("Pending expiry tasks: {pending}");
|
println!("Pending expiry tasks: {pending}");
|
||||||
|
|
||||||
if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await {
|
if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await
|
||||||
if let Ok(object_info) = ecstore
|
&& let Ok(object_info) = ecstore
|
||||||
.get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
|
.get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -438,7 +438,6 @@ mod serial_tests {
|
|||||||
|
|
||||||
expired = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(2)).await;
|
expired = wait_for_object_absence(&ecstore, bucket_name.as_str(), object_name, Duration::from_secs(2)).await;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if !expired {
|
if !expired {
|
||||||
println!("❌ Object was not deleted by lifecycle processing");
|
println!("❌ Object was not deleted by lifecycle processing");
|
||||||
@@ -550,8 +549,8 @@ mod serial_tests {
|
|||||||
.await;
|
.await;
|
||||||
println!("Pending expiry tasks: {pending}");
|
println!("Pending expiry tasks: {pending}");
|
||||||
|
|
||||||
if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await {
|
if let Ok((lc_config, _)) = rustfs_ecstore::bucket::metadata_sys::get_lifecycle_config(bucket_name.as_str()).await
|
||||||
if let Ok(obj_info) = ecstore
|
&& let Ok(obj_info) = ecstore
|
||||||
.get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
|
.get_object_info(bucket_name.as_str(), object_name, &rustfs_ecstore::store_api::ObjectOptions::default())
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -577,7 +576,6 @@ mod serial_tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if !deleted {
|
if !deleted {
|
||||||
println!("❌ Object was not deleted by lifecycle processing");
|
println!("❌ Object was not deleted by lifecycle processing");
|
||||||
|
|||||||
@@ -204,12 +204,12 @@ impl TargetFactory for MQTTTargetFactory {
|
|||||||
if !std::path::Path::new(&queue_dir).is_absolute() {
|
if !std::path::Path::new(&queue_dir).is_absolute() {
|
||||||
return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string()));
|
return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string()));
|
||||||
}
|
}
|
||||||
if let Some(qos_str) = config.lookup(MQTT_QOS) {
|
if let Some(qos_str) = config.lookup(MQTT_QOS)
|
||||||
if qos_str == "0" {
|
&& qos_str == "0"
|
||||||
|
{
|
||||||
warn!("Using queue_dir with QoS 0 may result in event loss");
|
warn!("Using queue_dir with QoS 0 may result in event loss");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -138,14 +138,13 @@ impl AuditRegistry {
|
|||||||
format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}")
|
format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}")
|
||||||
.to_uppercase();
|
.to_uppercase();
|
||||||
for (key, value) in &all_env {
|
for (key, value) in &all_env {
|
||||||
if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false) {
|
if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false)
|
||||||
if let Some(id) = key.strip_prefix(&enable_prefix) {
|
&& let Some(id) = key.strip_prefix(&enable_prefix)
|
||||||
if !id.is_empty() {
|
&& !id.is_empty()
|
||||||
|
{
|
||||||
instance_ids_from_env.insert(id.to_lowercase());
|
instance_ids_from_env.insert(id.to_lowercase());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3.2. Parse all relevant environment variable configurations
|
// 3.2. Parse all relevant environment variable configurations
|
||||||
// 3.2.1. Build environment variable prefixes such as 'RUSTFS_AUDIT_WEBHOOK_'
|
// 3.2.1. Build environment variable prefixes such as 'RUSTFS_AUDIT_WEBHOOK_'
|
||||||
@@ -292,11 +291,11 @@ impl AuditRegistry {
|
|||||||
for section in sections {
|
for section in sections {
|
||||||
let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
|
let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
|
||||||
// Add default item
|
// Add default item
|
||||||
if let Some(default_kvs) = section_defaults.get(§ion) {
|
if let Some(default_kvs) = section_defaults.get(§ion)
|
||||||
if !default_kvs.is_empty() {
|
&& !default_kvs.is_empty()
|
||||||
|
{
|
||||||
section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
|
section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Add successful instance item
|
// Add successful instance item
|
||||||
if let Some(instances) = successes_by_section.get(§ion) {
|
if let Some(instances) = successes_by_section.get(§ion) {
|
||||||
|
|||||||
@@ -573,11 +573,11 @@ impl AuditSystem {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Remove existing target if present
|
// Remove existing target if present
|
||||||
if let Some(old_target) = registry.remove_target(&target_id) {
|
if let Some(old_target) = registry.remove_target(&target_id)
|
||||||
if let Err(e) = old_target.close().await {
|
&& let Err(e) = old_target.close().await
|
||||||
|
{
|
||||||
error!(target_id = %target_id, error = %e, "Failed to close old target during upsert");
|
error!(target_id = %target_id, error = %e, "Failed to close old target during upsert");
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
registry.add_target(target_id.clone(), target);
|
registry.add_target(target_id.clone(), target);
|
||||||
info!(target_id = %target_id, "Target upserted");
|
info!(target_id = %target_id, "Target upserted");
|
||||||
|
|||||||
@@ -605,14 +605,13 @@ impl DataUsageCache {
|
|||||||
|
|
||||||
pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> {
|
pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> {
|
||||||
let want = hash.key();
|
let want = hash.key();
|
||||||
if let Some(last_index) = want.rfind('/') {
|
if let Some(last_index) = want.rfind('/')
|
||||||
if let Some(v) = self.find(&want[0..last_index]) {
|
&& let Some(v) = self.find(&want[0..last_index])
|
||||||
if v.children.contains(&want) {
|
&& v.children.contains(&want)
|
||||||
|
{
|
||||||
let found = hash_path(&want[0..last_index]);
|
let found = hash_path(&want[0..last_index]);
|
||||||
return Some(found);
|
return Some(found);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (k, v) in self.cache.iter() {
|
for (k, v) in self.cache.iter() {
|
||||||
if v.children.contains(&want) {
|
if v.children.contains(&want) {
|
||||||
@@ -1150,13 +1149,13 @@ impl DataUsageInfo {
|
|||||||
self.buckets_count = self.buckets_usage.len() as u64;
|
self.buckets_count = self.buckets_usage.len() as u64;
|
||||||
|
|
||||||
// Update last update time
|
// Update last update time
|
||||||
if let Some(other_update) = other.last_update {
|
if let Some(other_update) = other.last_update
|
||||||
if self.last_update.is_none() || other_update > self.last_update.unwrap() {
|
&& (self.last_update.is_none() || other_update > self.last_update.unwrap())
|
||||||
|
{
|
||||||
self.last_update = Some(other_update);
|
self.last_update = Some(other_update);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
impl BucketUsageInfo {
|
impl BucketUsageInfo {
|
||||||
/// Create a new BucketUsageInfo
|
/// Create a new BucketUsageInfo
|
||||||
|
|||||||
@@ -403,12 +403,12 @@ fn lc_get_prefix(rule: &LifecycleRule) -> String {
|
|||||||
} else if let Some(filter) = &rule.filter {
|
} else if let Some(filter) = &rule.filter {
|
||||||
if let Some(p) = &filter.prefix {
|
if let Some(p) = &filter.prefix {
|
||||||
return p.to_string();
|
return p.to_string();
|
||||||
} else if let Some(and) = &filter.and {
|
} else if let Some(and) = &filter.and
|
||||||
if let Some(p) = &and.prefix {
|
&& let Some(p) = &and.prefix
|
||||||
|
{
|
||||||
return p.to_string();
|
return p.to_string();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
"".into()
|
"".into()
|
||||||
}
|
}
|
||||||
@@ -475,10 +475,11 @@ pub fn rep_has_active_rules(config: &ReplicationConfiguration, prefix: &str, rec
|
|||||||
{
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if !prefix.is_empty() {
|
if !prefix.is_empty()
|
||||||
if let Some(filter) = &rule.filter {
|
&& let Some(filter) = &rule.filter
|
||||||
if let Some(r_prefix) = &filter.prefix {
|
&& let Some(r_prefix) = &filter.prefix
|
||||||
if !r_prefix.is_empty() {
|
&& !r_prefix.is_empty()
|
||||||
|
{
|
||||||
// incoming prefix must be in rule prefix
|
// incoming prefix must be in rule prefix
|
||||||
if !recursive && !prefix.starts_with(r_prefix) {
|
if !recursive && !prefix.starts_with(r_prefix) {
|
||||||
continue;
|
continue;
|
||||||
@@ -489,9 +490,6 @@ pub fn rep_has_active_rules(config: &ReplicationConfiguration, prefix: &str, rec
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
false
|
false
|
||||||
|
|||||||
@@ -466,23 +466,23 @@ impl Metrics {
|
|||||||
// Lifetime operations
|
// Lifetime operations
|
||||||
for i in 0..Metric::Last as usize {
|
for i in 0..Metric::Last as usize {
|
||||||
let count = self.operations[i].load(Ordering::Relaxed);
|
let count = self.operations[i].load(Ordering::Relaxed);
|
||||||
if count > 0 {
|
if count > 0
|
||||||
if let Some(metric) = Metric::from_index(i) {
|
&& let Some(metric) = Metric::from_index(i)
|
||||||
|
{
|
||||||
metrics.life_time_ops.insert(metric.as_str().to_string(), count);
|
metrics.life_time_ops.insert(metric.as_str().to_string(), count);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Last minute statistics for realtime metrics
|
// Last minute statistics for realtime metrics
|
||||||
for i in 0..Metric::LastRealtime as usize {
|
for i in 0..Metric::LastRealtime as usize {
|
||||||
let last_min = self.latency[i].total().await;
|
let last_min = self.latency[i].total().await;
|
||||||
if last_min.n > 0 {
|
if last_min.n > 0
|
||||||
if let Some(_metric) = Metric::from_index(i) {
|
&& let Some(_metric) = Metric::from_index(i)
|
||||||
|
{
|
||||||
// Convert to madmin TimedAction format if needed
|
// Convert to madmin TimedAction format if needed
|
||||||
// This would require implementing the conversion
|
// This would require implementing the conversion
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
metrics
|
metrics
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -178,12 +178,12 @@ impl RustFSTestEnvironment {
|
|||||||
info!("Cleaning up any existing RustFS processes");
|
info!("Cleaning up any existing RustFS processes");
|
||||||
let output = Command::new("pkill").args(["-f", "rustfs"]).output();
|
let output = Command::new("pkill").args(["-f", "rustfs"]).output();
|
||||||
|
|
||||||
if let Ok(output) = output {
|
if let Ok(output) = output
|
||||||
if output.status.success() {
|
&& output.status.success()
|
||||||
|
{
|
||||||
info!("Killed existing RustFS processes");
|
info!("Killed existing RustFS processes");
|
||||||
sleep(Duration::from_millis(1000)).await;
|
sleep(Duration::from_millis(1000)).await;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -406,13 +406,13 @@ impl VaultTestEnvironment {
|
|||||||
let port_check = TcpStream::connect(VAULT_ADDRESS).await.is_ok();
|
let port_check = TcpStream::connect(VAULT_ADDRESS).await.is_ok();
|
||||||
if port_check {
|
if port_check {
|
||||||
// Additional check by making a health request
|
// Additional check by making a health request
|
||||||
if let Ok(response) = reqwest::get(&format!("{VAULT_URL}/v1/sys/health")).await {
|
if let Ok(response) = reqwest::get(&format!("{VAULT_URL}/v1/sys/health")).await
|
||||||
if response.status().is_success() {
|
&& response.status().is_success()
|
||||||
|
{
|
||||||
info!("Vault server is ready after {} seconds", i);
|
info!("Vault server is ready after {} seconds", i);
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if i == 29 {
|
if i == 29 {
|
||||||
return Err("Vault server failed to become ready".into());
|
return Err("Vault server failed to become ready".into());
|
||||||
|
|||||||
@@ -498,8 +498,9 @@ impl BucketTargetSys {
|
|||||||
bucket: bucket.to_string(),
|
bucket: bucket.to_string(),
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
if arn.arn_type == BucketTargetType::ReplicationService {
|
if arn.arn_type == BucketTargetType::ReplicationService
|
||||||
if let Ok((config, _)) = get_replication_config(bucket).await {
|
&& let Ok((config, _)) = get_replication_config(bucket).await
|
||||||
|
{
|
||||||
for rule in config.filter_target_arns(&ObjectOpts {
|
for rule in config.filter_target_arns(&ObjectOpts {
|
||||||
op_type: ReplicationType::All,
|
op_type: ReplicationType::All,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
@@ -514,7 +515,6 @@ impl BucketTargetSys {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut targets_map = self.targets_map.write().await;
|
let mut targets_map = self.targets_map.write().await;
|
||||||
@@ -691,8 +691,9 @@ impl BucketTargetSys {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Add new targets
|
// Add new targets
|
||||||
if let Some(new_targets) = targets {
|
if let Some(new_targets) = targets
|
||||||
if !new_targets.is_empty() {
|
&& !new_targets.is_empty()
|
||||||
|
{
|
||||||
for target in &new_targets.targets {
|
for target in &new_targets.targets {
|
||||||
if let Ok(client) = self.get_remote_target_client_internal(target).await {
|
if let Ok(client) = self.get_remote_target_client_internal(target).await {
|
||||||
arn_remotes_map.insert(
|
arn_remotes_map.insert(
|
||||||
@@ -708,7 +709,6 @@ impl BucketTargetSys {
|
|||||||
targets_map.insert(bucket.to_string(), new_targets.targets.clone());
|
targets_map.insert(bucket.to_string(), new_targets.targets.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn set(&self, bucket: &str, meta: &BucketMetadata) {
|
pub async fn set(&self, bucket: &str, meta: &BucketMetadata) {
|
||||||
let Some(config) = &meta.bucket_target_config else {
|
let Some(config) = &meta.bucket_target_config else {
|
||||||
|
|||||||
@@ -31,11 +31,11 @@ impl BucketObjectLockSys {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get(bucket: &str) -> Option<DefaultRetention> {
|
pub async fn get(bucket: &str) -> Option<DefaultRetention> {
|
||||||
if let Ok(object_lock_config) = get_object_lock_config(bucket).await {
|
if let Ok(object_lock_config) = get_object_lock_config(bucket).await
|
||||||
if let Some(object_lock_rule) = object_lock_config.0.rule {
|
&& let Some(object_lock_rule) = object_lock_config.0.rule
|
||||||
|
{
|
||||||
return object_lock_rule.default_retention;
|
return object_lock_rule.default_retention;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -55,13 +55,13 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
|||||||
if !has_arn {
|
if !has_arn {
|
||||||
has_arn = true;
|
has_arn = true;
|
||||||
}
|
}
|
||||||
if let Some(status) = &rule.existing_object_replication {
|
if let Some(status) = &rule.existing_object_replication
|
||||||
if status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED) {
|
&& status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::ENABLED)
|
||||||
|
{
|
||||||
return (true, true);
|
return (true, true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
(has_arn, false)
|
(has_arn, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -86,13 +86,12 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(status) = &rule.existing_object_replication {
|
if let Some(status) = &rule.existing_object_replication
|
||||||
if obj.existing_object
|
&& obj.existing_object
|
||||||
&& status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
|
&& status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
|
||||||
{
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if !obj.name.starts_with(rule.prefix()) {
|
if !obj.name.starts_with(rule.prefix()) {
|
||||||
continue;
|
continue;
|
||||||
@@ -145,13 +144,12 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(status) = &rule.existing_object_replication {
|
if let Some(status) = &rule.existing_object_replication
|
||||||
if obj.existing_object
|
&& obj.existing_object
|
||||||
&& status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
|
&& status.status == ExistingObjectReplicationStatus::from_static(ExistingObjectReplicationStatus::DISABLED)
|
||||||
{
|
{
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if obj.op_type == ReplicationType::Delete {
|
if obj.op_type == ReplicationType::Delete {
|
||||||
if obj.version_id.is_some() {
|
if obj.version_id.is_some() {
|
||||||
@@ -186,8 +184,9 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(filter) = &rule.filter {
|
if let Some(filter) = &rule.filter
|
||||||
if let Some(filter_prefix) = &filter.prefix {
|
&& let Some(filter_prefix) = &filter.prefix
|
||||||
|
{
|
||||||
if !prefix.is_empty() && !filter_prefix.is_empty() {
|
if !prefix.is_empty() && !filter_prefix.is_empty() {
|
||||||
// The provided prefix must fall within the rule prefix
|
// The provided prefix must fall within the rule prefix
|
||||||
if !recursive && !prefix.starts_with(filter_prefix) {
|
if !recursive && !prefix.starts_with(filter_prefix) {
|
||||||
@@ -200,7 +199,6 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
false
|
false
|
||||||
|
|||||||
@@ -512,8 +512,9 @@ impl<S: StorageAPI> ReplicationPool<S> {
|
|||||||
if !lrg_workers.is_empty() {
|
if !lrg_workers.is_empty() {
|
||||||
let index = (hash as usize) % lrg_workers.len();
|
let index = (hash as usize) % lrg_workers.len();
|
||||||
|
|
||||||
if let Some(worker) = lrg_workers.get(index) {
|
if let Some(worker) = lrg_workers.get(index)
|
||||||
if worker.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_err() {
|
&& worker.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_err()
|
||||||
|
{
|
||||||
// Queue to MRF if worker is busy
|
// Queue to MRF if worker is busy
|
||||||
let _ = self.mrf_save_tx.try_send(ri.to_mrf_entry());
|
let _ = self.mrf_save_tx.try_send(ri.to_mrf_entry());
|
||||||
|
|
||||||
@@ -528,7 +529,6 @@ impl<S: StorageAPI> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -539,8 +539,9 @@ impl<S: StorageAPI> ReplicationPool<S> {
|
|||||||
_ => self.get_worker_ch(&ri.bucket, &ri.name, ri.size).await,
|
_ => self.get_worker_ch(&ri.bucket, &ri.name, ri.size).await,
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(channel) = ch {
|
if let Some(channel) = ch
|
||||||
if channel.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_err() {
|
&& channel.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_err()
|
||||||
|
{
|
||||||
// Queue to MRF if all workers are busy
|
// Queue to MRF if all workers are busy
|
||||||
let _ = self.mrf_save_tx.try_send(ri.to_mrf_entry());
|
let _ = self.mrf_save_tx.try_send(ri.to_mrf_entry());
|
||||||
|
|
||||||
@@ -554,9 +555,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
|
|||||||
info!("Warning: Unable to keep up with incoming traffic");
|
info!("Warning: Unable to keep up with incoming traffic");
|
||||||
}
|
}
|
||||||
ReplicationPriority::Slow => {
|
ReplicationPriority::Slow => {
|
||||||
info!(
|
info!("Warning: Unable to keep up with incoming traffic - recommend increasing replication priority to auto");
|
||||||
"Warning: Unable to keep up with incoming traffic - recommend increasing replication priority to auto"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
ReplicationPriority::Auto => {
|
ReplicationPriority::Auto => {
|
||||||
let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT);
|
let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT);
|
||||||
@@ -584,7 +583,6 @@ impl<S: StorageAPI> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
/// Queues a replica delete task
|
/// Queues a replica delete task
|
||||||
pub async fn queue_replica_delete_task(&self, doi: DeletedObjectReplicationInfo) {
|
pub async fn queue_replica_delete_task(&self, doi: DeletedObjectReplicationInfo) {
|
||||||
@@ -593,8 +591,9 @@ impl<S: StorageAPI> ReplicationPool<S> {
|
|||||||
_ => self.get_worker_ch(&doi.bucket, &doi.delete_object.object_name, 0).await,
|
_ => self.get_worker_ch(&doi.bucket, &doi.delete_object.object_name, 0).await,
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(channel) = ch {
|
if let Some(channel) = ch
|
||||||
if channel.try_send(ReplicationOperation::Delete(Box::new(doi.clone()))).is_err() {
|
&& channel.try_send(ReplicationOperation::Delete(Box::new(doi.clone()))).is_err()
|
||||||
|
{
|
||||||
let _ = self.mrf_save_tx.try_send(doi.to_mrf_entry());
|
let _ = self.mrf_save_tx.try_send(doi.to_mrf_entry());
|
||||||
|
|
||||||
let priority = self.priority.read().await.clone();
|
let priority = self.priority.read().await.clone();
|
||||||
@@ -605,9 +604,7 @@ impl<S: StorageAPI> ReplicationPool<S> {
|
|||||||
info!("Warning: Unable to keep up with incoming deletes");
|
info!("Warning: Unable to keep up with incoming deletes");
|
||||||
}
|
}
|
||||||
ReplicationPriority::Slow => {
|
ReplicationPriority::Slow => {
|
||||||
info!(
|
info!("Warning: Unable to keep up with incoming deletes - recommend increasing replication priority to auto");
|
||||||
"Warning: Unable to keep up with incoming deletes - recommend increasing replication priority to auto"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
ReplicationPriority::Auto => {
|
ReplicationPriority::Auto => {
|
||||||
let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT);
|
let max_w = std::cmp::min(max_workers, WORKER_MAX_LIMIT);
|
||||||
@@ -622,7 +619,6 @@ impl<S: StorageAPI> ReplicationPool<S> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
/// Queues an MRF save operation
|
/// Queues an MRF save operation
|
||||||
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
|
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
|
||||||
|
|||||||
@@ -242,11 +242,10 @@ impl ReplicationResyncer {
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
if let Some(last_update) = status.last_update {
|
if let Some(last_update) = status.last_update
|
||||||
if last_update > *last_update_times.get(bucket).unwrap_or(&OffsetDateTime::UNIX_EPOCH) {
|
&& last_update > *last_update_times.get(bucket).unwrap_or(&OffsetDateTime::UNIX_EPOCH) {
|
||||||
update = true;
|
update = true;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if update {
|
if update {
|
||||||
if let Err(err) = save_resync_status(bucket, status, api.clone()).await {
|
if let Err(err) = save_resync_status(bucket, status, api.clone()).await {
|
||||||
@@ -345,14 +344,13 @@ impl ReplicationResyncer {
|
|||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|
||||||
if !heal {
|
if !heal
|
||||||
if let Err(e) = self
|
&& let Err(e) = self
|
||||||
.mark_status(ResyncStatusType::ResyncStarted, opts.clone(), storage.clone())
|
.mark_status(ResyncStatusType::ResyncStarted, opts.clone(), storage.clone())
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
error!("Failed to mark resync status: {}", e);
|
error!("Failed to mark resync status: {}", e);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let (tx, mut rx) = tokio::sync::mpsc::channel(100);
|
let (tx, mut rx) = tokio::sync::mpsc::channel(100);
|
||||||
|
|
||||||
@@ -1463,21 +1461,18 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
|||||||
Some(version_id.to_string())
|
Some(version_id.to_string())
|
||||||
};
|
};
|
||||||
|
|
||||||
if dobj.delete_object.delete_marker_version_id.is_some() {
|
if dobj.delete_object.delete_marker_version_id.is_some()
|
||||||
if let Err(e) = tgt_client
|
&& let Err(e) = tgt_client
|
||||||
.head_object(&tgt_client.bucket, &dobj.delete_object.object_name, version_id.clone())
|
.head_object(&tgt_client.bucket, &dobj.delete_object.object_name, version_id.clone())
|
||||||
.await
|
.await
|
||||||
|
&& let SdkError::ServiceError(service_err) = &e
|
||||||
|
&& !service_err.err().is_not_found()
|
||||||
{
|
{
|
||||||
if let SdkError::ServiceError(service_err) = &e {
|
|
||||||
if !service_err.err().is_not_found() {
|
|
||||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||||
rinfo.error = Some(e.to_string());
|
rinfo.error = Some(e.to_string());
|
||||||
|
|
||||||
return rinfo;
|
return rinfo;
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
}
|
|
||||||
|
|
||||||
match tgt_client
|
match tgt_client
|
||||||
.remove_object(
|
.remove_object(
|
||||||
|
|||||||
@@ -49,8 +49,9 @@ impl ExponentialMovingAverage {
|
|||||||
pub fn update_exponential_moving_average(&self, now: SystemTime) {
|
pub fn update_exponential_moving_average(&self, now: SystemTime) {
|
||||||
if let Ok(mut last_update_guard) = self.last_update.try_lock() {
|
if let Ok(mut last_update_guard) = self.last_update.try_lock() {
|
||||||
let last_update = *last_update_guard;
|
let last_update = *last_update_guard;
|
||||||
if let Ok(duration) = now.duration_since(last_update) {
|
if let Ok(duration) = now.duration_since(last_update)
|
||||||
if duration.as_secs() > 0 {
|
&& duration.as_secs() > 0
|
||||||
|
{
|
||||||
let decay = (-duration.as_secs_f64() / 60.0).exp(); // 1 minute decay
|
let decay = (-duration.as_secs_f64() / 60.0).exp(); // 1 minute decay
|
||||||
let current_value = f64::from_bits(self.value.load(AtomicOrdering::Relaxed));
|
let current_value = f64::from_bits(self.value.load(AtomicOrdering::Relaxed));
|
||||||
self.value.store((current_value * decay).to_bits(), AtomicOrdering::Relaxed);
|
self.value.store((current_value * decay).to_bits(), AtomicOrdering::Relaxed);
|
||||||
@@ -58,7 +59,6 @@ impl ExponentialMovingAverage {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
pub fn merge(&self, other: &ExponentialMovingAverage) -> Self {
|
pub fn merge(&self, other: &ExponentialMovingAverage) -> Self {
|
||||||
let now = SystemTime::now();
|
let now = SystemTime::now();
|
||||||
@@ -757,11 +757,11 @@ impl ReplicationStats {
|
|||||||
|
|
||||||
/// Check if bucket replication statistics have usage
|
/// Check if bucket replication statistics have usage
|
||||||
pub fn has_replication_usage(&self, bucket: &str) -> bool {
|
pub fn has_replication_usage(&self, bucket: &str) -> bool {
|
||||||
if let Ok(cache) = self.cache.try_read() {
|
if let Ok(cache) = self.cache.try_read()
|
||||||
if let Some(stats) = cache.get(bucket) {
|
&& let Some(stats) = cache.get(bucket)
|
||||||
|
{
|
||||||
return stats.has_replication_usage();
|
return stats.has_replication_usage();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -37,11 +37,12 @@ impl VersioningApi for VersioningConfiguration {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(exclude_folders) = self.exclude_folders {
|
if let Some(exclude_folders) = self.exclude_folders
|
||||||
if exclude_folders && prefix.ends_with('/') {
|
&& exclude_folders
|
||||||
|
&& prefix.ends_with('/')
|
||||||
|
{
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(ref excluded_prefixes) = self.excluded_prefixes {
|
if let Some(ref excluded_prefixes) = self.excluded_prefixes {
|
||||||
for p in excluded_prefixes.iter() {
|
for p in excluded_prefixes.iter() {
|
||||||
@@ -67,11 +68,12 @@ impl VersioningApi for VersioningConfiguration {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(exclude_folders) = self.exclude_folders {
|
if let Some(exclude_folders) = self.exclude_folders
|
||||||
if exclude_folders && prefix.ends_with('/') {
|
&& exclude_folders
|
||||||
|
&& prefix.ends_with('/')
|
||||||
|
{
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(ref excluded_prefixes) = self.excluded_prefixes {
|
if let Some(ref excluded_prefixes) = self.excluded_prefixes {
|
||||||
for p in excluded_prefixes.iter() {
|
for p in excluded_prefixes.iter() {
|
||||||
|
|||||||
@@ -308,13 +308,12 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
|
|||||||
|
|
||||||
// Break if all at EOF or error.
|
// Break if all at EOF or error.
|
||||||
if at_eof + has_err == readers.len() {
|
if at_eof + has_err == readers.len() {
|
||||||
if has_err > 0 {
|
if has_err > 0
|
||||||
if let Some(finished_fn) = opts.finished.as_ref() {
|
&& let Some(finished_fn) = opts.finished.as_ref()
|
||||||
if has_err > 0 {
|
&& has_err > 0
|
||||||
|
{
|
||||||
finished_fn(&errs).await;
|
finished_fn(&errs).await;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// error!("list_path_raw: at_eof + has_err == readers.len() break {:?}", &errs);
|
// error!("list_path_raw: at_eof + has_err == readers.len() break {:?}", &errs);
|
||||||
break;
|
break;
|
||||||
|
|||||||
@@ -161,7 +161,7 @@ impl TransitionClient {
|
|||||||
async fn private_new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
|
async fn private_new(endpoint: &str, opts: Options, tier_type: &str) -> Result<TransitionClient, std::io::Error> {
|
||||||
let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
|
let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
|
||||||
|
|
||||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||||
let scheme = endpoint_url.scheme();
|
let scheme = endpoint_url.scheme();
|
||||||
let client;
|
let client;
|
||||||
let tls = if let Some(store) = load_root_store_from_tls_path() {
|
let tls = if let Some(store) = load_root_store_from_tls_path() {
|
||||||
|
|||||||
@@ -211,12 +211,13 @@ async fn apply_dynamic_config_for_sub_sys<S: StorageAPI>(cfg: &mut Config, api:
|
|||||||
for (i, count) in set_drive_counts.iter().enumerate() {
|
for (i, count) in set_drive_counts.iter().enumerate() {
|
||||||
match storageclass::lookup_config(&kvs, *count) {
|
match storageclass::lookup_config(&kvs, *count) {
|
||||||
Ok(res) => {
|
Ok(res) => {
|
||||||
if i == 0 && GLOBAL_STORAGE_CLASS.get().is_none() {
|
if i == 0
|
||||||
if let Err(r) = GLOBAL_STORAGE_CLASS.set(res) {
|
&& GLOBAL_STORAGE_CLASS.get().is_none()
|
||||||
|
&& let Err(r) = GLOBAL_STORAGE_CLASS.set(res)
|
||||||
|
{
|
||||||
error!("GLOBAL_STORAGE_CLASS.set failed {:?}", r);
|
error!("GLOBAL_STORAGE_CLASS.set failed {:?}", r);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
error!("init storage class err:{:?}", &err);
|
error!("init storage class err:{:?}", &err);
|
||||||
break;
|
break;
|
||||||
|
|||||||
@@ -180,14 +180,14 @@ impl Config {
|
|||||||
let mut default = HashMap::new();
|
let mut default = HashMap::new();
|
||||||
default.insert(DEFAULT_DELIMITER.to_owned(), v.clone());
|
default.insert(DEFAULT_DELIMITER.to_owned(), v.clone());
|
||||||
self.0.insert(k.clone(), default);
|
self.0.insert(k.clone(), default);
|
||||||
} else if !self.0[k].contains_key(DEFAULT_DELIMITER) {
|
} else if !self.0[k].contains_key(DEFAULT_DELIMITER)
|
||||||
if let Some(m) = self.0.get_mut(k) {
|
&& let Some(m) = self.0.get_mut(k)
|
||||||
|
{
|
||||||
m.insert(DEFAULT_DELIMITER.to_owned(), v.clone());
|
m.insert(DEFAULT_DELIMITER.to_owned(), v.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
pub fn unmarshal(data: &[u8]) -> Result<Config> {
|
pub fn unmarshal(data: &[u8]) -> Result<Config> {
|
||||||
let m: HashMap<String, HashMap<String, KVS>> = serde_json::from_slice(data)?;
|
let m: HashMap<String, HashMap<String, KVS>> = serde_json::from_slice(data)?;
|
||||||
|
|||||||
@@ -65,19 +65,17 @@ lazy_static::lazy_static! {
|
|||||||
/// Store data usage info to backend storage
|
/// Store data usage info to backend storage
|
||||||
pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
|
pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
|
||||||
// Prevent older data from overwriting newer persisted stats
|
// Prevent older data from overwriting newer persisted stats
|
||||||
if let Ok(buf) = read_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH).await {
|
if let Ok(buf) = read_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH).await
|
||||||
if let Ok(existing) = serde_json::from_slice::<DataUsageInfo>(&buf) {
|
&& let Ok(existing) = serde_json::from_slice::<DataUsageInfo>(&buf)
|
||||||
if let (Some(new_ts), Some(existing_ts)) = (data_usage_info.last_update, existing.last_update) {
|
&& let (Some(new_ts), Some(existing_ts)) = (data_usage_info.last_update, existing.last_update)
|
||||||
if new_ts <= existing_ts {
|
&& new_ts <= existing_ts
|
||||||
|
{
|
||||||
info!(
|
info!(
|
||||||
"Skip persisting data usage: incoming last_update {:?} <= existing {:?}",
|
"Skip persisting data usage: incoming last_update {:?} <= existing {:?}",
|
||||||
new_ts, existing_ts
|
new_ts, existing_ts
|
||||||
);
|
);
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let data =
|
let data =
|
||||||
serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
|
serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
|
||||||
@@ -149,13 +147,13 @@ pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsa
|
|||||||
|
|
||||||
// Handle replication info
|
// Handle replication info
|
||||||
for (bucket, bui) in &data_usage_info.buckets_usage {
|
for (bucket, bui) in &data_usage_info.buckets_usage {
|
||||||
if bui.replicated_size_v1 > 0
|
if (bui.replicated_size_v1 > 0
|
||||||
|| bui.replication_failed_count_v1 > 0
|
|| bui.replication_failed_count_v1 > 0
|
||||||
|| bui.replication_failed_size_v1 > 0
|
|| bui.replication_failed_size_v1 > 0
|
||||||
|| bui.replication_pending_count_v1 > 0
|
|| bui.replication_pending_count_v1 > 0)
|
||||||
|
&& let Ok((cfg, _)) = get_replication_config(bucket).await
|
||||||
|
&& !cfg.role.is_empty()
|
||||||
{
|
{
|
||||||
if let Ok((cfg, _)) = get_replication_config(bucket).await {
|
|
||||||
if !cfg.role.is_empty() {
|
|
||||||
data_usage_info.replication_info.insert(
|
data_usage_info.replication_info.insert(
|
||||||
cfg.role.clone(),
|
cfg.role.clone(),
|
||||||
BucketTargetUsageInfo {
|
BucketTargetUsageInfo {
|
||||||
@@ -169,19 +167,17 @@ pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsa
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(data_usage_info)
|
Ok(data_usage_info)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Aggregate usage information from local disk snapshots.
|
/// Aggregate usage information from local disk snapshots.
|
||||||
fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapshot, latest_update: &mut Option<SystemTime>) {
|
fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapshot, latest_update: &mut Option<SystemTime>) {
|
||||||
if let Some(update) = snapshot.last_update {
|
if let Some(update) = snapshot.last_update
|
||||||
if latest_update.is_none_or(|current| update > current) {
|
&& latest_update.is_none_or(|current| update > current)
|
||||||
|
{
|
||||||
*latest_update = Some(update);
|
*latest_update = Some(update);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
snapshot.recompute_totals();
|
snapshot.recompute_totals();
|
||||||
|
|
||||||
@@ -255,12 +251,12 @@ pub async fn aggregate_local_snapshots(store: Arc<ECStore>) -> Result<(Vec<DiskU
|
|||||||
);
|
);
|
||||||
// Best-effort cleanup so next scan can rebuild a fresh snapshot instead of repeatedly failing
|
// Best-effort cleanup so next scan can rebuild a fresh snapshot instead of repeatedly failing
|
||||||
let snapshot_file = snapshot_path(root.as_path(), &disk_id);
|
let snapshot_file = snapshot_path(root.as_path(), &disk_id);
|
||||||
if let Err(remove_err) = fs::remove_file(&snapshot_file).await {
|
if let Err(remove_err) = fs::remove_file(&snapshot_file).await
|
||||||
if remove_err.kind() != std::io::ErrorKind::NotFound {
|
&& remove_err.kind() != std::io::ErrorKind::NotFound
|
||||||
|
{
|
||||||
warn!("Failed to remove corrupted snapshot {:?}: {}", snapshot_file, remove_err);
|
warn!("Failed to remove corrupted snapshot {:?}: {}", snapshot_file, remove_err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Ok(Some(mut snapshot)) = snapshot_result {
|
if let Ok(Some(mut snapshot)) = snapshot_result {
|
||||||
status.last_update = snapshot.last_update;
|
status.last_update = snapshot.last_update;
|
||||||
|
|||||||
@@ -288,17 +288,17 @@ impl LocalDisk {
|
|||||||
let path = path_join(&[trash.clone(), name.into()]);
|
let path = path_join(&[trash.clone(), name.into()]);
|
||||||
|
|
||||||
if file_type.is_dir() {
|
if file_type.is_dir() {
|
||||||
if let Err(e) = tokio::fs::remove_dir_all(path).await {
|
if let Err(e) = tokio::fs::remove_dir_all(path).await
|
||||||
if e.kind() != ErrorKind::NotFound {
|
&& e.kind() != ErrorKind::NotFound
|
||||||
|
{
|
||||||
return Err(e.into());
|
return Err(e.into());
|
||||||
}
|
}
|
||||||
}
|
} else if let Err(e) = tokio::fs::remove_file(path).await
|
||||||
} else if let Err(e) = tokio::fs::remove_file(path).await {
|
&& e.kind() != ErrorKind::NotFound
|
||||||
if e.kind() != ErrorKind::NotFound {
|
{
|
||||||
return Err(e.into());
|
return Err(e.into());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -684,14 +684,12 @@ impl LocalDisk {
|
|||||||
Err(err) => {
|
Err(err) => {
|
||||||
if err == Error::FileNotFound
|
if err == Error::FileNotFound
|
||||||
&& !skip_access_checks(volume_dir.as_ref().to_string_lossy().to_string().as_str())
|
&& !skip_access_checks(volume_dir.as_ref().to_string_lossy().to_string().as_str())
|
||||||
|
&& let Err(e) = access(volume_dir.as_ref()).await
|
||||||
|
&& e.kind() == ErrorKind::NotFound
|
||||||
{
|
{
|
||||||
if let Err(e) = access(volume_dir.as_ref()).await {
|
|
||||||
if e.kind() == ErrorKind::NotFound {
|
|
||||||
// warn!("read_metadata_with_dmtime os err {:?}", &aerr);
|
// warn!("read_metadata_with_dmtime os err {:?}", &aerr);
|
||||||
return Err(DiskError::VolumeNotFound);
|
return Err(DiskError::VolumeNotFound);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Err(err)
|
Err(err)
|
||||||
}
|
}
|
||||||
@@ -763,14 +761,14 @@ impl LocalDisk {
|
|||||||
let mut f = match super::fs::open_file(file_path.as_ref(), O_RDONLY).await {
|
let mut f = match super::fs::open_file(file_path.as_ref(), O_RDONLY).await {
|
||||||
Ok(f) => f,
|
Ok(f) => f,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
if e.kind() == ErrorKind::NotFound && !skip_access_checks(volume) {
|
if e.kind() == ErrorKind::NotFound
|
||||||
if let Err(er) = access(volume_dir.as_ref()).await {
|
&& !skip_access_checks(volume)
|
||||||
if er.kind() == ErrorKind::NotFound {
|
&& let Err(er) = access(volume_dir.as_ref()).await
|
||||||
|
&& er.kind() == ErrorKind::NotFound
|
||||||
|
{
|
||||||
warn!("read_all_data_with_dmtime os err {:?}", &er);
|
warn!("read_all_data_with_dmtime os err {:?}", &er);
|
||||||
return Err(DiskError::VolumeNotFound);
|
return Err(DiskError::VolumeNotFound);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return Err(to_file_error(e).into());
|
return Err(to_file_error(e).into());
|
||||||
}
|
}
|
||||||
@@ -828,10 +826,10 @@ impl LocalDisk {
|
|||||||
let _ = fm.data.remove(vec![vid, dir]);
|
let _ = fm.data.remove(vec![vid, dir]);
|
||||||
|
|
||||||
let dir_path = self.get_object_path(volume, format!("{path}/{dir}").as_str())?;
|
let dir_path = self.get_object_path(volume, format!("{path}/{dir}").as_str())?;
|
||||||
if let Err(err) = self.move_to_trash(&dir_path, true, false).await {
|
if let Err(err) = self.move_to_trash(&dir_path, true, false).await
|
||||||
if !(err == DiskError::FileNotFound || err == DiskError::VolumeNotFound) {
|
&& !(err == DiskError::FileNotFound || err == DiskError::VolumeNotFound)
|
||||||
|
{
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1051,12 +1049,12 @@ impl LocalDisk {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(forward) = &forward {
|
if let Some(forward) = &forward
|
||||||
if &entry < forward {
|
&& &entry < forward
|
||||||
|
{
|
||||||
*item = "".to_owned();
|
*item = "".to_owned();
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if entry.ends_with(SLASH_SEPARATOR) {
|
if entry.ends_with(SLASH_SEPARATOR) {
|
||||||
if entry.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) {
|
if entry.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) {
|
||||||
@@ -1133,11 +1131,11 @@ impl LocalDisk {
|
|||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
if opts.recursive {
|
if opts.recursive
|
||||||
if let Err(er) = Box::pin(self.scan_dir(pop, prefix.clone(), opts, out, objs_returned)).await {
|
&& let Err(er) = Box::pin(self.scan_dir(pop, prefix.clone(), opts, out, objs_returned)).await
|
||||||
|
{
|
||||||
error!("scan_dir err {:?}", er);
|
error!("scan_dir err {:?}", er);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
dir_stack.pop();
|
dir_stack.pop();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1200,12 +1198,12 @@ impl LocalDisk {
|
|||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
if opts.recursive {
|
if opts.recursive
|
||||||
if let Err(er) = Box::pin(self.scan_dir(dir, prefix.clone(), opts, out, objs_returned)).await {
|
&& let Err(er) = Box::pin(self.scan_dir(dir, prefix.clone(), opts, out, objs_returned)).await
|
||||||
|
{
|
||||||
warn!("scan_dir err {:?}", &er);
|
warn!("scan_dir err {:?}", &er);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1345,24 +1343,24 @@ impl DiskAPI for LocalDisk {
|
|||||||
|
|
||||||
if format_info.file_info.is_some() && id.is_some() {
|
if format_info.file_info.is_some() && id.is_some() {
|
||||||
// check last check time
|
// check last check time
|
||||||
if let Some(last_check) = format_info.last_check {
|
if let Some(last_check) = format_info.last_check
|
||||||
if last_check.unix_timestamp() + 1 < OffsetDateTime::now_utc().unix_timestamp() {
|
&& last_check.unix_timestamp() + 1 < OffsetDateTime::now_utc().unix_timestamp()
|
||||||
|
{
|
||||||
return Ok(id);
|
return Ok(id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let file_meta = self.check_format_json().await?;
|
let file_meta = self.check_format_json().await?;
|
||||||
|
|
||||||
if let Some(file_info) = &format_info.file_info {
|
if let Some(file_info) = &format_info.file_info
|
||||||
if super::fs::same_file(&file_meta, file_info) {
|
&& super::fs::same_file(&file_meta, file_info)
|
||||||
|
{
|
||||||
let mut format_info = self.format_info.write().await;
|
let mut format_info = self.format_info.write().await;
|
||||||
format_info.last_check = Some(OffsetDateTime::now_utc());
|
format_info.last_check = Some(OffsetDateTime::now_utc());
|
||||||
drop(format_info);
|
drop(format_info);
|
||||||
|
|
||||||
return Ok(id);
|
return Ok(id);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
debug!("get_disk_id: read format.json");
|
debug!("get_disk_id: read format.json");
|
||||||
|
|
||||||
@@ -1420,11 +1418,11 @@ impl DiskAPI for LocalDisk {
|
|||||||
#[tracing::instrument(skip(self))]
|
#[tracing::instrument(skip(self))]
|
||||||
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
|
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
|
||||||
let volume_dir = self.get_bucket_path(volume)?;
|
let volume_dir = self.get_bucket_path(volume)?;
|
||||||
if !skip_access_checks(volume) {
|
if !skip_access_checks(volume)
|
||||||
if let Err(e) = access(&volume_dir).await {
|
&& let Err(e) = access(&volume_dir).await
|
||||||
|
{
|
||||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let file_path = volume_dir.join(Path::new(&path));
|
let file_path = volume_dir.join(Path::new(&path));
|
||||||
check_path_length(file_path.to_string_lossy().to_string().as_str())?;
|
check_path_length(file_path.to_string_lossy().to_string().as_str())?;
|
||||||
@@ -1438,11 +1436,11 @@ impl DiskAPI for LocalDisk {
|
|||||||
#[tracing::instrument(skip(self))]
|
#[tracing::instrument(skip(self))]
|
||||||
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
|
async fn verify_file(&self, volume: &str, path: &str, fi: &FileInfo) -> Result<CheckPartsResp> {
|
||||||
let volume_dir = self.get_bucket_path(volume)?;
|
let volume_dir = self.get_bucket_path(volume)?;
|
||||||
if !skip_access_checks(volume) {
|
if !skip_access_checks(volume)
|
||||||
if let Err(e) = access(&volume_dir).await {
|
&& let Err(e) = access(&volume_dir).await
|
||||||
|
{
|
||||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let mut resp = CheckPartsResp {
|
let mut resp = CheckPartsResp {
|
||||||
results: vec![0; fi.parts.len()],
|
results: vec![0; fi.parts.len()],
|
||||||
@@ -1466,8 +1464,9 @@ impl DiskAPI for LocalDisk {
|
|||||||
.await
|
.await
|
||||||
.err();
|
.err();
|
||||||
resp.results[i] = conv_part_err_to_int(&err);
|
resp.results[i] = conv_part_err_to_int(&err);
|
||||||
if resp.results[i] == CHECK_PART_UNKNOWN {
|
if resp.results[i] == CHECK_PART_UNKNOWN
|
||||||
if let Some(err) = err {
|
&& let Some(err) = err
|
||||||
|
{
|
||||||
error!("verify_file: failed to bitrot verify file: {:?}, error: {:?}", &part_path, &err);
|
error!("verify_file: failed to bitrot verify file: {:?}, error: {:?}", &part_path, &err);
|
||||||
if err == DiskError::FileAccessDenied {
|
if err == DiskError::FileAccessDenied {
|
||||||
continue;
|
continue;
|
||||||
@@ -1475,7 +1474,6 @@ impl DiskAPI for LocalDisk {
|
|||||||
info!("part unknown, disk: {}, path: {:?}", self.to_string(), part_path);
|
info!("part unknown, disk: {}, path: {:?}", self.to_string(), part_path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok(resp)
|
Ok(resp)
|
||||||
}
|
}
|
||||||
@@ -1572,14 +1570,13 @@ impl DiskAPI for LocalDisk {
|
|||||||
let e: DiskError = to_file_error(err).into();
|
let e: DiskError = to_file_error(err).into();
|
||||||
|
|
||||||
if e == DiskError::FileNotFound {
|
if e == DiskError::FileNotFound {
|
||||||
if !skip_access_checks(volume) {
|
if !skip_access_checks(volume)
|
||||||
if let Err(err) = access(&volume_dir).await {
|
&& let Err(err) = access(&volume_dir).await
|
||||||
if err.kind() == ErrorKind::NotFound {
|
&& err.kind() == ErrorKind::NotFound
|
||||||
|
{
|
||||||
resp.results[i] = CHECK_PART_VOLUME_NOT_FOUND;
|
resp.results[i] = CHECK_PART_VOLUME_NOT_FOUND;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
resp.results[i] = CHECK_PART_FILE_NOT_FOUND;
|
resp.results[i] = CHECK_PART_FILE_NOT_FOUND;
|
||||||
} else {
|
} else {
|
||||||
error!("check_parts: failed to stat file: {:?}, error: {:?}", &file_path, &e);
|
error!("check_parts: failed to stat file: {:?}, error: {:?}", &file_path, &e);
|
||||||
@@ -1634,12 +1631,12 @@ impl DiskAPI for LocalDisk {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(meta) = meta_op {
|
if let Some(meta) = meta_op
|
||||||
if !meta.is_dir() {
|
&& !meta.is_dir()
|
||||||
|
{
|
||||||
warn!("rename_part src is not dir {:?}", &src_file_path);
|
warn!("rename_part src is not dir {:?}", &src_file_path);
|
||||||
return Err(DiskError::FileAccessDenied);
|
return Err(DiskError::FileAccessDenied);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
remove_std(&dst_file_path).map_err(to_file_error)?;
|
remove_std(&dst_file_path).map_err(to_file_error)?;
|
||||||
}
|
}
|
||||||
@@ -1695,11 +1692,11 @@ impl DiskAPI for LocalDisk {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(meta) = meta_op {
|
if let Some(meta) = meta_op
|
||||||
if !meta.is_dir() {
|
&& !meta.is_dir()
|
||||||
|
{
|
||||||
return Err(DiskError::FileAccessDenied);
|
return Err(DiskError::FileAccessDenied);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
remove(&dst_file_path).await.map_err(to_file_error)?;
|
remove(&dst_file_path).await.map_err(to_file_error)?;
|
||||||
}
|
}
|
||||||
@@ -1814,12 +1811,12 @@ impl DiskAPI for LocalDisk {
|
|||||||
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
|
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
|
||||||
if !origvolume.is_empty() {
|
if !origvolume.is_empty() {
|
||||||
let origvolume_dir = self.get_bucket_path(origvolume)?;
|
let origvolume_dir = self.get_bucket_path(origvolume)?;
|
||||||
if !skip_access_checks(origvolume) {
|
if !skip_access_checks(origvolume)
|
||||||
if let Err(e) = access(origvolume_dir).await {
|
&& let Err(e) = access(origvolume_dir).await
|
||||||
|
{
|
||||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let volume_dir = self.get_bucket_path(volume)?;
|
let volume_dir = self.get_bucket_path(volume)?;
|
||||||
let dir_path_abs = volume_dir.join(Path::new(&dir_path.trim_start_matches(SLASH_SEPARATOR)));
|
let dir_path_abs = volume_dir.join(Path::new(&dir_path.trim_start_matches(SLASH_SEPARATOR)));
|
||||||
@@ -1827,11 +1824,12 @@ impl DiskAPI for LocalDisk {
|
|||||||
let entries = match os::read_dir(&dir_path_abs, count).await {
|
let entries = match os::read_dir(&dir_path_abs, count).await {
|
||||||
Ok(res) => res,
|
Ok(res) => res,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
if e.kind() == std::io::ErrorKind::NotFound && !skip_access_checks(volume) {
|
if e.kind() == std::io::ErrorKind::NotFound
|
||||||
if let Err(e) = access(&volume_dir).await {
|
&& !skip_access_checks(volume)
|
||||||
|
&& let Err(e) = access(&volume_dir).await
|
||||||
|
{
|
||||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return Err(to_file_error(e).into());
|
return Err(to_file_error(e).into());
|
||||||
}
|
}
|
||||||
@@ -1845,11 +1843,11 @@ impl DiskAPI for LocalDisk {
|
|||||||
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
|
async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
|
||||||
let volume_dir = self.get_bucket_path(&opts.bucket)?;
|
let volume_dir = self.get_bucket_path(&opts.bucket)?;
|
||||||
|
|
||||||
if !skip_access_checks(&opts.bucket) {
|
if !skip_access_checks(&opts.bucket)
|
||||||
if let Err(e) = access(&volume_dir).await {
|
&& let Err(e) = access(&volume_dir).await
|
||||||
|
{
|
||||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let mut wr = wr;
|
let mut wr = wr;
|
||||||
|
|
||||||
@@ -1909,20 +1907,20 @@ impl DiskAPI for LocalDisk {
|
|||||||
dst_path: &str,
|
dst_path: &str,
|
||||||
) -> Result<RenameDataResp> {
|
) -> Result<RenameDataResp> {
|
||||||
let src_volume_dir = self.get_bucket_path(src_volume)?;
|
let src_volume_dir = self.get_bucket_path(src_volume)?;
|
||||||
if !skip_access_checks(src_volume) {
|
if !skip_access_checks(src_volume)
|
||||||
if let Err(e) = super::fs::access_std(&src_volume_dir) {
|
&& let Err(e) = super::fs::access_std(&src_volume_dir)
|
||||||
|
{
|
||||||
info!("access checks failed, src_volume_dir: {:?}, err: {}", src_volume_dir, e.to_string());
|
info!("access checks failed, src_volume_dir: {:?}, err: {}", src_volume_dir, e.to_string());
|
||||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let dst_volume_dir = self.get_bucket_path(dst_volume)?;
|
let dst_volume_dir = self.get_bucket_path(dst_volume)?;
|
||||||
if !skip_access_checks(dst_volume) {
|
if !skip_access_checks(dst_volume)
|
||||||
if let Err(e) = super::fs::access_std(&dst_volume_dir) {
|
&& let Err(e) = super::fs::access_std(&dst_volume_dir)
|
||||||
|
{
|
||||||
info!("access checks failed, dst_volume_dir: {:?}, err: {}", dst_volume_dir, e.to_string());
|
info!("access checks failed, dst_volume_dir: {:?}, err: {}", dst_volume_dir, e.to_string());
|
||||||
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
return Err(to_access_error(e, DiskError::VolumeAccessDenied).into());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// xl.meta path
|
// xl.meta path
|
||||||
let src_file_path = src_volume_dir.join(Path::new(format!("{}/{}", &src_path, STORAGE_FORMAT_FILE).as_str()));
|
let src_file_path = src_volume_dir.join(Path::new(format!("{}/{}", &src_path, STORAGE_FORMAT_FILE).as_str()));
|
||||||
@@ -1973,20 +1971,19 @@ impl DiskAPI for LocalDisk {
|
|||||||
|
|
||||||
let mut xlmeta = FileMeta::new();
|
let mut xlmeta = FileMeta::new();
|
||||||
|
|
||||||
if let Some(dst_buf) = has_dst_buf.as_ref() {
|
if let Some(dst_buf) = has_dst_buf.as_ref()
|
||||||
if FileMeta::is_xl2_v1_format(dst_buf) {
|
&& FileMeta::is_xl2_v1_format(dst_buf)
|
||||||
if let Ok(nmeta) = FileMeta::load(dst_buf) {
|
&& let Ok(nmeta) = FileMeta::load(dst_buf)
|
||||||
|
{
|
||||||
xlmeta = nmeta
|
xlmeta = nmeta
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut skip_parent = dst_volume_dir.clone();
|
let mut skip_parent = dst_volume_dir.clone();
|
||||||
if has_dst_buf.as_ref().is_some() {
|
if has_dst_buf.as_ref().is_some()
|
||||||
if let Some(parent) = dst_file_path.parent() {
|
&& let Some(parent) = dst_file_path.parent()
|
||||||
|
{
|
||||||
skip_parent = parent.to_path_buf();
|
skip_parent = parent.to_path_buf();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: Healing
|
// TODO: Healing
|
||||||
|
|
||||||
@@ -2017,8 +2014,7 @@ impl DiskAPI for LocalDisk {
|
|||||||
.await?;
|
.await?;
|
||||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() {
|
if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() {
|
||||||
let no_inline = fi.data.is_none() && fi.size > 0;
|
let no_inline = fi.data.is_none() && fi.size > 0;
|
||||||
if no_inline {
|
if no_inline && let Err(err) = rename_all(&src_data_path, &dst_data_path, &skip_parent).await {
|
||||||
if let Err(err) = rename_all(&src_data_path, &dst_data_path, &skip_parent).await {
|
|
||||||
let _ = self.delete_file(&dst_volume_dir, dst_data_path, false, false).await;
|
let _ = self.delete_file(&dst_volume_dir, dst_data_path, false, false).await;
|
||||||
info!(
|
info!(
|
||||||
"rename all failed src_data_path: {:?}, dst_data_path: {:?}, err: {:?}",
|
"rename all failed src_data_path: {:?}, dst_data_path: {:?}, err: {:?}",
|
||||||
@@ -2027,12 +2023,11 @@ impl DiskAPI for LocalDisk {
|
|||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(old_data_dir) = has_old_data_dir {
|
if let Some(old_data_dir) = has_old_data_dir {
|
||||||
// preserve current xl.meta inside the oldDataDir.
|
// preserve current xl.meta inside the oldDataDir.
|
||||||
if let Some(dst_buf) = has_dst_buf {
|
if let Some(dst_buf) = has_dst_buf
|
||||||
if let Err(err) = self
|
&& let Err(err) = self
|
||||||
.write_all_private(
|
.write_all_private(
|
||||||
dst_volume,
|
dst_volume,
|
||||||
format!("{}/{}/{}", &dst_path, &old_data_dir.to_string(), STORAGE_FORMAT_FILE).as_str(),
|
format!("{}/{}/{}", &dst_path, &old_data_dir.to_string(), STORAGE_FORMAT_FILE).as_str(),
|
||||||
@@ -2046,7 +2041,6 @@ impl DiskAPI for LocalDisk {
|
|||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Err(err) = rename_all(&src_file_path, &dst_file_path, &skip_parent).await {
|
if let Err(err) = rename_all(&src_file_path, &dst_file_path, &skip_parent).await {
|
||||||
if let Some((_, dst_data_path)) = has_data_dir_path.as_ref() {
|
if let Some((_, dst_data_path)) = has_data_dir_path.as_ref() {
|
||||||
@@ -2075,12 +2069,12 @@ impl DiskAPI for LocalDisk {
|
|||||||
#[tracing::instrument(skip(self))]
|
#[tracing::instrument(skip(self))]
|
||||||
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
|
async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> {
|
||||||
for vol in volumes {
|
for vol in volumes {
|
||||||
if let Err(e) = self.make_volume(vol).await {
|
if let Err(e) = self.make_volume(vol).await
|
||||||
if e != DiskError::VolumeExists {
|
&& e != DiskError::VolumeExists
|
||||||
|
{
|
||||||
error!("local disk make volumes failed: {e}");
|
error!("local disk make volumes failed: {e}");
|
||||||
return Err(e);
|
return Err(e);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
// TODO: health check
|
// TODO: health check
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -2313,12 +2307,13 @@ impl DiskAPI for LocalDisk {
|
|||||||
let old_path = file_path.join(Path::new(uuid.to_string().as_str()));
|
let old_path = file_path.join(Path::new(uuid.to_string().as_str()));
|
||||||
check_path_length(old_path.to_string_lossy().as_ref())?;
|
check_path_length(old_path.to_string_lossy().as_ref())?;
|
||||||
|
|
||||||
if let Err(err) = self.move_to_trash(&old_path, true, false).await {
|
if let Err(err) = self.move_to_trash(&old_path, true, false).await
|
||||||
if err != DiskError::FileNotFound && err != DiskError::VolumeNotFound {
|
&& err != DiskError::FileNotFound
|
||||||
|
&& err != DiskError::VolumeNotFound
|
||||||
|
{
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if !meta.versions.is_empty() {
|
if !meta.versions.is_empty() {
|
||||||
let buf = meta.marshal_msg()?;
|
let buf = meta.marshal_msg()?;
|
||||||
@@ -2328,14 +2323,14 @@ impl DiskAPI for LocalDisk {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// opts.undo_write && opts.old_data_dir.is_some_and(f)
|
// opts.undo_write && opts.old_data_dir.is_some_and(f)
|
||||||
if let Some(old_data_dir) = opts.old_data_dir {
|
if let Some(old_data_dir) = opts.old_data_dir
|
||||||
if opts.undo_write {
|
&& opts.undo_write
|
||||||
|
{
|
||||||
let src_path =
|
let src_path =
|
||||||
file_path.join(Path::new(format!("{old_data_dir}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE_BACKUP}").as_str()));
|
file_path.join(Path::new(format!("{old_data_dir}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE_BACKUP}").as_str()));
|
||||||
let dst_path = file_path.join(Path::new(format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str()));
|
let dst_path = file_path.join(Path::new(format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str()));
|
||||||
return rename_all(src_path, dst_path, file_path).await;
|
return rename_all(src_path, dst_path, file_path).await;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
self.delete_file(&volume_dir, &xl_path, true, false).await
|
self.delete_file(&volume_dir, &xl_path, true, false).await
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -147,12 +147,12 @@ async fn reliable_rename(
|
|||||||
dst_file_path: impl AsRef<Path>,
|
dst_file_path: impl AsRef<Path>,
|
||||||
base_dir: impl AsRef<Path>,
|
base_dir: impl AsRef<Path>,
|
||||||
) -> io::Result<()> {
|
) -> io::Result<()> {
|
||||||
if let Some(parent) = dst_file_path.as_ref().parent() {
|
if let Some(parent) = dst_file_path.as_ref().parent()
|
||||||
if !file_exists(parent) {
|
&& !file_exists(parent)
|
||||||
|
{
|
||||||
// info!("reliable_rename reliable_mkdir_all parent: {:?}", parent);
|
// info!("reliable_rename reliable_mkdir_all parent: {:?}", parent);
|
||||||
reliable_mkdir_all(parent, base_dir.as_ref()).await?;
|
reliable_mkdir_all(parent, base_dir.as_ref()).await?;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let mut i = 0;
|
let mut i = 0;
|
||||||
loop {
|
loop {
|
||||||
@@ -190,13 +190,12 @@ pub async fn reliable_mkdir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Pat
|
|||||||
if e.kind() == io::ErrorKind::NotFound && i == 0 {
|
if e.kind() == io::ErrorKind::NotFound && i == 0 {
|
||||||
i += 1;
|
i += 1;
|
||||||
|
|
||||||
if let Some(base_parent) = base_dir.parent() {
|
if let Some(base_parent) = base_dir.parent()
|
||||||
if let Some(c) = base_parent.components().next() {
|
&& let Some(c) = base_parent.components().next()
|
||||||
if c != Component::RootDir {
|
&& c != Component::RootDir
|
||||||
|
{
|
||||||
base_dir = base_parent
|
base_dir = base_parent
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -318,7 +318,7 @@ fn get_divisible_size(total_sizes: &[usize]) -> usize {
|
|||||||
fn possible_set_counts(set_size: usize) -> Vec<usize> {
|
fn possible_set_counts(set_size: usize) -> Vec<usize> {
|
||||||
let mut ss = Vec::new();
|
let mut ss = Vec::new();
|
||||||
for s in SET_SIZES {
|
for s in SET_SIZES {
|
||||||
if set_size % s == 0 {
|
if set_size.is_multiple_of(s) {
|
||||||
ss.push(s);
|
ss.push(s);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -340,7 +340,7 @@ fn common_set_drive_count(divisible_size: usize, set_counts: &[usize]) -> usize
|
|||||||
let mut prev_d = divisible_size / set_counts[0];
|
let mut prev_d = divisible_size / set_counts[0];
|
||||||
let mut set_size = 0;
|
let mut set_size = 0;
|
||||||
for &cnt in set_counts {
|
for &cnt in set_counts {
|
||||||
if divisible_size % cnt == 0 {
|
if divisible_size.is_multiple_of(cnt) {
|
||||||
let d = divisible_size / cnt;
|
let d = divisible_size / cnt;
|
||||||
if d <= prev_d {
|
if d <= prev_d {
|
||||||
prev_d = d;
|
prev_d = d;
|
||||||
|
|||||||
@@ -266,13 +266,12 @@ impl Erasure {
|
|||||||
|
|
||||||
let (mut shards, errs) = reader.read().await;
|
let (mut shards, errs) = reader.read().await;
|
||||||
|
|
||||||
if ret_err.is_none() {
|
if ret_err.is_none()
|
||||||
if let (_, Some(err)) = reduce_errs(&errs, &[]) {
|
&& let (_, Some(err)) = reduce_errs(&errs, &[])
|
||||||
if err == Error::FileNotFound || err == Error::FileCorrupt {
|
&& (err == Error::FileNotFound || err == Error::FileCorrupt)
|
||||||
|
{
|
||||||
ret_err = Some(err.into());
|
ret_err = Some(err.into());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !reader.can_decode(&shards) {
|
if !reader.can_decode(&shards) {
|
||||||
error!("erasure decode can_decode errs: {:?}", &errs);
|
error!("erasure decode can_decode errs: {:?}", &errs);
|
||||||
|
|||||||
@@ -150,11 +150,11 @@ impl Erasure {
|
|||||||
}
|
}
|
||||||
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
|
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
|
||||||
// Check if the inner error is a checksum mismatch - if so, propagate it
|
// Check if the inner error is a checksum mismatch - if so, propagate it
|
||||||
if let Some(inner) = e.get_ref() {
|
if let Some(inner) = e.get_ref()
|
||||||
if rustfs_rio::is_checksum_mismatch(inner) {
|
&& rustfs_rio::is_checksum_mismatch(inner)
|
||||||
|
{
|
||||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()));
|
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ impl super::Erasure {
|
|||||||
|
|
||||||
let start_block = 0;
|
let start_block = 0;
|
||||||
let mut end_block = total_length / self.block_size;
|
let mut end_block = total_length / self.block_size;
|
||||||
if total_length % self.block_size != 0 {
|
if !total_length.is_multiple_of(self.block_size) {
|
||||||
end_block += 1;
|
end_block += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+32
-32
@@ -244,11 +244,13 @@ impl PoolMeta {
|
|||||||
}
|
}
|
||||||
pub fn decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> {
|
pub fn decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> {
|
||||||
if let Some(pool) = self.pools.get_mut(idx) {
|
if let Some(pool) = self.pools.get_mut(idx) {
|
||||||
if let Some(ref info) = pool.decommission {
|
if let Some(ref info) = pool.decommission
|
||||||
if !info.complete && !info.failed && !info.canceled {
|
&& !info.complete
|
||||||
|
&& !info.failed
|
||||||
|
&& !info.canceled
|
||||||
|
{
|
||||||
return Err(StorageError::DecommissionAlreadyRunning);
|
return Err(StorageError::DecommissionAlreadyRunning);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let now = OffsetDateTime::now_utc();
|
let now = OffsetDateTime::now_utc();
|
||||||
pool.last_update = now;
|
pool.last_update = now;
|
||||||
@@ -273,14 +275,14 @@ impl PoolMeta {
|
|||||||
pub fn pending_buckets(&self, idx: usize) -> Vec<DecomBucketInfo> {
|
pub fn pending_buckets(&self, idx: usize) -> Vec<DecomBucketInfo> {
|
||||||
let mut list = Vec::new();
|
let mut list = Vec::new();
|
||||||
|
|
||||||
if let Some(pool) = self.pools.get(idx) {
|
if let Some(pool) = self.pools.get(idx)
|
||||||
if let Some(ref info) = pool.decommission {
|
&& let Some(ref info) = pool.decommission
|
||||||
|
{
|
||||||
for bk in info.queued_buckets.iter() {
|
for bk in info.queued_buckets.iter() {
|
||||||
let (name, prefix) = path2_bucket_object(bk);
|
let (name, prefix) = path2_bucket_object(bk);
|
||||||
list.push(DecomBucketInfo { name, prefix });
|
list.push(DecomBucketInfo { name, prefix });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
list
|
list
|
||||||
}
|
}
|
||||||
@@ -306,8 +308,9 @@ impl PoolMeta {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn count_item(&mut self, idx: usize, size: usize, failed: bool) {
|
pub fn count_item(&mut self, idx: usize, size: usize, failed: bool) {
|
||||||
if let Some(pool) = self.pools.get_mut(idx) {
|
if let Some(pool) = self.pools.get_mut(idx)
|
||||||
if let Some(info) = pool.decommission.as_mut() {
|
&& let Some(info) = pool.decommission.as_mut()
|
||||||
|
{
|
||||||
if failed {
|
if failed {
|
||||||
info.items_decommission_failed += 1;
|
info.items_decommission_failed += 1;
|
||||||
info.bytes_failed += size;
|
info.bytes_failed += size;
|
||||||
@@ -317,20 +320,19 @@ impl PoolMeta {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
pub fn track_current_bucket_object(&mut self, idx: usize, bucket: String, object: String) {
|
pub fn track_current_bucket_object(&mut self, idx: usize, bucket: String, object: String) {
|
||||||
if self.pools.get(idx).is_none_or(|v| v.decommission.is_none()) {
|
if self.pools.get(idx).is_none_or(|v| v.decommission.is_none()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(pool) = self.pools.get_mut(idx) {
|
if let Some(pool) = self.pools.get_mut(idx)
|
||||||
if let Some(info) = pool.decommission.as_mut() {
|
&& let Some(info) = pool.decommission.as_mut()
|
||||||
|
{
|
||||||
info.object = object;
|
info.object = object;
|
||||||
info.bucket = bucket;
|
info.bucket = bucket;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn update_after(&mut self, idx: usize, pools: Vec<Arc<Sets>>, duration: Duration) -> Result<bool> {
|
pub async fn update_after(&mut self, idx: usize, pools: Vec<Arc<Sets>>, duration: Duration) -> Result<bool> {
|
||||||
if self.pools.get(idx).is_none_or(|v| v.decommission.is_none()) {
|
if self.pools.get(idx).is_none_or(|v| v.decommission.is_none()) {
|
||||||
@@ -407,13 +409,13 @@ impl PoolMeta {
|
|||||||
|
|
||||||
if specified_pools.len() == remembered_pools.len() {
|
if specified_pools.len() == remembered_pools.len() {
|
||||||
for (k, pi) in remembered_pools.iter() {
|
for (k, pi) in remembered_pools.iter() {
|
||||||
if let Some(pos) = specified_pools.get(k) {
|
if let Some(pos) = specified_pools.get(k)
|
||||||
if *pos != pi.position {
|
&& *pos != pi.position
|
||||||
|
{
|
||||||
update = true; // Pool order changed, allow the update.
|
update = true; // Pool order changed, allow the update.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if !update {
|
if !update {
|
||||||
update = specified_pools.len() != remembered_pools.len();
|
update = specified_pools.len() != remembered_pools.len();
|
||||||
@@ -640,12 +642,14 @@ impl ECStore {
|
|||||||
pub async fn is_decommission_running(&self) -> bool {
|
pub async fn is_decommission_running(&self) -> bool {
|
||||||
let pool_meta = self.pool_meta.read().await;
|
let pool_meta = self.pool_meta.read().await;
|
||||||
for pool in pool_meta.pools.iter() {
|
for pool in pool_meta.pools.iter() {
|
||||||
if let Some(ref info) = pool.decommission {
|
if let Some(ref info) = pool.decommission
|
||||||
if !info.complete && !info.failed && !info.canceled {
|
&& !info.complete
|
||||||
|
&& !info.failed
|
||||||
|
&& !info.canceled
|
||||||
|
{
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
@@ -850,8 +854,8 @@ impl ECStore {
|
|||||||
decommissioned += 1;
|
decommissioned += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
if decommissioned == fivs.versions.len() {
|
if decommissioned == fivs.versions.len()
|
||||||
if let Err(err) = set
|
&& let Err(err) = set
|
||||||
.delete_object(
|
.delete_object(
|
||||||
bucket.as_str(),
|
bucket.as_str(),
|
||||||
&encode_dir_object(&entry.name),
|
&encode_dir_object(&entry.name),
|
||||||
@@ -866,7 +870,6 @@ impl ECStore {
|
|||||||
{
|
{
|
||||||
error!("decommission_pool: delete_object err {:?}", &err);
|
error!("decommission_pool: delete_object err {:?}", &err);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut pool_meta = self.pool_meta.write().await;
|
let mut pool_meta = self.pool_meta.write().await;
|
||||||
@@ -879,12 +882,10 @@ impl ECStore {
|
|||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
drop(pool_meta);
|
drop(pool_meta);
|
||||||
if ok {
|
if ok && let Some(notification_sys) = get_global_notification_sys() {
|
||||||
if let Some(notification_sys) = get_global_notification_sys() {
|
|
||||||
notification_sys.reload_pool_meta().await;
|
notification_sys.reload_pool_meta().await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
warn!("decommission_pool: decommission_entry done {} {}", &bucket, &entry.name);
|
warn!("decommission_pool: decommission_entry done {} {}", &bucket, &entry.name);
|
||||||
}
|
}
|
||||||
@@ -1080,12 +1081,12 @@ impl ECStore {
|
|||||||
|
|
||||||
{
|
{
|
||||||
let mut pool_meta = self.pool_meta.write().await;
|
let mut pool_meta = self.pool_meta.write().await;
|
||||||
if pool_meta.bucket_done(idx, bucket.to_string()) {
|
if pool_meta.bucket_done(idx, bucket.to_string())
|
||||||
if let Err(err) = pool_meta.save(self.pools.clone()).await {
|
&& let Err(err) = pool_meta.save(self.pools.clone()).await
|
||||||
|
{
|
||||||
error!("decom pool_meta.save err {:?}", err);
|
error!("decom pool_meta.save err {:?}", err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1100,11 +1101,11 @@ impl ECStore {
|
|||||||
|
|
||||||
{
|
{
|
||||||
let mut pool_meta = self.pool_meta.write().await;
|
let mut pool_meta = self.pool_meta.write().await;
|
||||||
if pool_meta.bucket_done(idx, bucket.to_string()) {
|
if pool_meta.bucket_done(idx, bucket.to_string())
|
||||||
if let Err(err) = pool_meta.save(self.pools.clone()).await {
|
&& let Err(err) = pool_meta.save(self.pools.clone()).await
|
||||||
|
{
|
||||||
error!("decom pool_meta.save err {:?}", err);
|
error!("decom pool_meta.save err {:?}", err);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
warn!("decommission: decommission_pool bucket_done {}", &bucket.name);
|
warn!("decommission: decommission_pool bucket_done {}", &bucket.name);
|
||||||
}
|
}
|
||||||
@@ -1138,13 +1139,12 @@ impl ECStore {
|
|||||||
if let Err(err) = self
|
if let Err(err) = self
|
||||||
.make_bucket(bk.to_string_lossy().to_string().as_str(), &MakeBucketOptions::default())
|
.make_bucket(bk.to_string_lossy().to_string().as_str(), &MakeBucketOptions::default())
|
||||||
.await
|
.await
|
||||||
|
&& !is_err_bucket_exists(&err)
|
||||||
{
|
{
|
||||||
if !is_err_bucket_exists(&err) {
|
|
||||||
error!("decommission: make bucket failed: {err}");
|
error!("decommission: make bucket failed: {err}");
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let mut pool_meta = self.pool_meta.write().await;
|
let mut pool_meta = self.pool_meta.write().await;
|
||||||
for idx in indices.iter() {
|
for idx in indices.iter() {
|
||||||
|
|||||||
@@ -380,11 +380,11 @@ impl ECStore {
|
|||||||
#[tracing::instrument(skip(self, fi))]
|
#[tracing::instrument(skip(self, fi))]
|
||||||
pub async fn update_pool_stats(&self, pool_index: usize, bucket: String, fi: &FileInfo) -> Result<()> {
|
pub async fn update_pool_stats(&self, pool_index: usize, bucket: String, fi: &FileInfo) -> Result<()> {
|
||||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||||
if let Some(meta) = rebalance_meta.as_mut() {
|
if let Some(meta) = rebalance_meta.as_mut()
|
||||||
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
|
&& let Some(pool_stat) = meta.pool_stats.get_mut(pool_index)
|
||||||
|
{
|
||||||
pool_stat.update(bucket, fi);
|
pool_stat.update(bucket, fi);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -394,8 +394,9 @@ impl ECStore {
|
|||||||
info!("next_rebal_bucket: pool_index: {}", pool_index);
|
info!("next_rebal_bucket: pool_index: {}", pool_index);
|
||||||
let rebalance_meta = self.rebalance_meta.read().await;
|
let rebalance_meta = self.rebalance_meta.read().await;
|
||||||
info!("next_rebal_bucket: rebalance_meta: {:?}", rebalance_meta);
|
info!("next_rebal_bucket: rebalance_meta: {:?}", rebalance_meta);
|
||||||
if let Some(meta) = rebalance_meta.as_ref() {
|
if let Some(meta) = rebalance_meta.as_ref()
|
||||||
if let Some(pool_stat) = meta.pool_stats.get(pool_index) {
|
&& let Some(pool_stat) = meta.pool_stats.get(pool_index)
|
||||||
|
{
|
||||||
if pool_stat.info.status == RebalStatus::Completed || !pool_stat.participating {
|
if pool_stat.info.status == RebalStatus::Completed || !pool_stat.participating {
|
||||||
info!("next_rebal_bucket: pool_index: {} completed or not participating", pool_index);
|
info!("next_rebal_bucket: pool_index: {} completed or not participating", pool_index);
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
@@ -408,7 +409,6 @@ impl ECStore {
|
|||||||
info!("next_rebal_bucket: pool_index: {} bucket: {}", pool_index, pool_stat.buckets[0]);
|
info!("next_rebal_bucket: pool_index: {} bucket: {}", pool_index, pool_stat.buckets[0]);
|
||||||
return Ok(Some(pool_stat.buckets[0].clone()));
|
return Ok(Some(pool_stat.buckets[0].clone()));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
info!("next_rebal_bucket: pool_index: {} None", pool_index);
|
info!("next_rebal_bucket: pool_index: {} None", pool_index);
|
||||||
Ok(None)
|
Ok(None)
|
||||||
@@ -417,8 +417,9 @@ impl ECStore {
|
|||||||
#[tracing::instrument(skip(self))]
|
#[tracing::instrument(skip(self))]
|
||||||
pub async fn bucket_rebalance_done(&self, pool_index: usize, bucket: String) -> Result<()> {
|
pub async fn bucket_rebalance_done(&self, pool_index: usize, bucket: String) -> Result<()> {
|
||||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||||
if let Some(meta) = rebalance_meta.as_mut() {
|
if let Some(meta) = rebalance_meta.as_mut()
|
||||||
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
|
&& let Some(pool_stat) = meta.pool_stats.get_mut(pool_index)
|
||||||
|
{
|
||||||
info!("bucket_rebalance_done: buckets {:?}", &pool_stat.buckets);
|
info!("bucket_rebalance_done: buckets {:?}", &pool_stat.buckets);
|
||||||
|
|
||||||
// Use retain to filter out buckets slated for removal
|
// Use retain to filter out buckets slated for removal
|
||||||
@@ -440,7 +441,6 @@ impl ECStore {
|
|||||||
info!("bucket_rebalance_done: bucket {} not found", bucket);
|
info!("bucket_rebalance_done: bucket {} not found", bucket);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
info!("bucket_rebalance_done: bucket {} not found", bucket);
|
info!("bucket_rebalance_done: bucket {} not found", bucket);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -492,11 +492,11 @@ impl ECStore {
|
|||||||
#[tracing::instrument(skip(self))]
|
#[tracing::instrument(skip(self))]
|
||||||
pub async fn stop_rebalance(self: &Arc<Self>) -> Result<()> {
|
pub async fn stop_rebalance(self: &Arc<Self>) -> Result<()> {
|
||||||
let rebalance_meta = self.rebalance_meta.read().await;
|
let rebalance_meta = self.rebalance_meta.read().await;
|
||||||
if let Some(meta) = rebalance_meta.as_ref() {
|
if let Some(meta) = rebalance_meta.as_ref()
|
||||||
if let Some(cancel_tx) = meta.cancel.as_ref() {
|
&& let Some(cancel_tx) = meta.cancel.as_ref()
|
||||||
|
{
|
||||||
cancel_tx.cancel();
|
cancel_tx.cancel();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -690,8 +690,9 @@ impl ECStore {
|
|||||||
async fn check_if_rebalance_done(&self, pool_index: usize) -> bool {
|
async fn check_if_rebalance_done(&self, pool_index: usize) -> bool {
|
||||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||||
|
|
||||||
if let Some(meta) = rebalance_meta.as_mut() {
|
if let Some(meta) = rebalance_meta.as_mut()
|
||||||
if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
|
&& let Some(pool_stat) = meta.pool_stats.get_mut(pool_index)
|
||||||
|
{
|
||||||
// Check if the pool's rebalance status is already completed
|
// Check if the pool's rebalance status is already completed
|
||||||
if pool_stat.info.status == RebalStatus::Completed {
|
if pool_stat.info.status == RebalStatus::Completed {
|
||||||
info!("check_if_rebalance_done: pool {} is already completed", pool_index);
|
info!("check_if_rebalance_done: pool {} is already completed", pool_index);
|
||||||
@@ -709,7 +710,6 @@ impl ECStore {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
@@ -1102,12 +1102,12 @@ impl ECStore {
|
|||||||
pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
|
pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
|
||||||
// TODO: lock
|
// TODO: lock
|
||||||
let mut meta = RebalanceMeta::new();
|
let mut meta = RebalanceMeta::new();
|
||||||
if let Err(err) = meta.load(self.pools[0].clone()).await {
|
if let Err(err) = meta.load(self.pools[0].clone()).await
|
||||||
if err != Error::ConfigNotFound {
|
&& err != Error::ConfigNotFound
|
||||||
|
{
|
||||||
info!("save_rebalance_stats: load err: {:?}", err);
|
info!("save_rebalance_stats: load err: {:?}", err);
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
match opt {
|
match opt {
|
||||||
RebalSaveOpt::Stats => {
|
RebalSaveOpt::Stats => {
|
||||||
|
|||||||
@@ -66,15 +66,15 @@ impl PeerRestClient {
|
|||||||
let mut remote = Vec::with_capacity(hosts.len());
|
let mut remote = Vec::with_capacity(hosts.len());
|
||||||
let mut all = vec![None; hosts.len()];
|
let mut all = vec![None; hosts.len()];
|
||||||
for (i, hs_host) in hosts.iter().enumerate() {
|
for (i, hs_host) in hosts.iter().enumerate() {
|
||||||
if let Some(host) = hs_host {
|
if let Some(host) = hs_host
|
||||||
if let Some(grid_host) = eps.find_grid_hosts_from_peer(host) {
|
&& let Some(grid_host) = eps.find_grid_hosts_from_peer(host)
|
||||||
|
{
|
||||||
let client = PeerRestClient::new(host.clone(), grid_host);
|
let client = PeerRestClient::new(host.clone(), grid_host);
|
||||||
|
|
||||||
all[i] = Some(client.clone());
|
all[i] = Some(client.clone());
|
||||||
remote.push(Some(client));
|
remote.push(Some(client));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if all.len() != remote.len() + 1 {
|
if all.len() != remote.len() + 1 {
|
||||||
warn!("Expected number of all hosts ({}) to be remote +1 ({})", all.len(), remote.len());
|
warn!("Expected number of all hosts ({}) to be remote +1 ({})", all.len(), remote.len());
|
||||||
|
|||||||
@@ -101,12 +101,12 @@ impl S3PeerSys {
|
|||||||
for pool_idx in 0..self.pools_count {
|
for pool_idx in 0..self.pools_count {
|
||||||
let mut per_pool_errs = vec![None; self.clients.len()];
|
let mut per_pool_errs = vec![None; self.clients.len()];
|
||||||
for (i, client) in self.clients.iter().enumerate() {
|
for (i, client) in self.clients.iter().enumerate() {
|
||||||
if let Some(v) = client.get_pools() {
|
if let Some(v) = client.get_pools()
|
||||||
if v.contains(&pool_idx) {
|
&& v.contains(&pool_idx)
|
||||||
|
{
|
||||||
per_pool_errs[i] = errs[i].clone();
|
per_pool_errs[i] = errs[i].clone();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
let qu = per_pool_errs.len() / 2;
|
let qu = per_pool_errs.len() / 2;
|
||||||
pool_errs.push(reduce_write_quorum_errs(&per_pool_errs, BUCKET_OP_IGNORED_ERRS, qu));
|
pool_errs.push(reduce_write_quorum_errs(&per_pool_errs, BUCKET_OP_IGNORED_ERRS, qu));
|
||||||
}
|
}
|
||||||
@@ -136,12 +136,12 @@ impl S3PeerSys {
|
|||||||
for pool_idx in 0..self.pools_count {
|
for pool_idx in 0..self.pools_count {
|
||||||
let mut per_pool_errs = vec![None; self.clients.len()];
|
let mut per_pool_errs = vec![None; self.clients.len()];
|
||||||
for (i, client) in self.clients.iter().enumerate() {
|
for (i, client) in self.clients.iter().enumerate() {
|
||||||
if let Some(v) = client.get_pools() {
|
if let Some(v) = client.get_pools()
|
||||||
if v.contains(&pool_idx) {
|
&& v.contains(&pool_idx)
|
||||||
|
{
|
||||||
per_pool_errs[i] = errs[i].clone();
|
per_pool_errs[i] = errs[i].clone();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
let qu = per_pool_errs.len() / 2;
|
let qu = per_pool_errs.len() / 2;
|
||||||
if let Some(pool_err) = reduce_write_quorum_errs(&per_pool_errs, BUCKET_OP_IGNORED_ERRS, qu) {
|
if let Some(pool_err) = reduce_write_quorum_errs(&per_pool_errs, BUCKET_OP_IGNORED_ERRS, qu) {
|
||||||
tracing::error!("heal_bucket per_pool_errs: {per_pool_errs:?}");
|
tracing::error!("heal_bucket per_pool_errs: {per_pool_errs:?}");
|
||||||
|
|||||||
@@ -266,12 +266,12 @@ impl SetDisks {
|
|||||||
let mut new_disk = Vec::with_capacity(disks.len());
|
let mut new_disk = Vec::with_capacity(disks.len());
|
||||||
|
|
||||||
for disk in disks.iter() {
|
for disk in disks.iter() {
|
||||||
if let Some(d) = disk {
|
if let Some(d) = disk
|
||||||
if d.is_online().await {
|
&& d.is_online().await
|
||||||
|
{
|
||||||
new_disk.push(disk.clone());
|
new_disk.push(disk.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let mut rng = rand::rng();
|
let mut rng = rand::rng();
|
||||||
|
|
||||||
@@ -1417,9 +1417,10 @@ impl SetDisks {
|
|||||||
let mut valid_obj_map = HashMap::new();
|
let mut valid_obj_map = HashMap::new();
|
||||||
|
|
||||||
for (i, op_hash) in meta_hashes.iter().enumerate() {
|
for (i, op_hash) in meta_hashes.iter().enumerate() {
|
||||||
if let Some(hash) = op_hash {
|
if let Some(hash) = op_hash
|
||||||
if let Some(max_hash) = max_val {
|
&& let Some(max_hash) = max_val
|
||||||
if hash == max_hash {
|
&& hash == max_hash
|
||||||
|
{
|
||||||
if metas[i].is_valid() && !found {
|
if metas[i].is_valid() && !found {
|
||||||
found_fi = Some(metas[i].clone());
|
found_fi = Some(metas[i].clone());
|
||||||
found = true;
|
found = true;
|
||||||
@@ -1433,8 +1434,6 @@ impl SetDisks {
|
|||||||
*valid_obj_map.entry(props).or_insert(0) += 1;
|
*valid_obj_map.entry(props).or_insert(0) += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if found {
|
if found {
|
||||||
let mut fi = found_fi.unwrap();
|
let mut fi = found_fi.unwrap();
|
||||||
@@ -3572,19 +3571,19 @@ impl SetDisks {
|
|||||||
let mut offline = 0;
|
let mut offline = 0;
|
||||||
for (i, err) in errs.iter().enumerate() {
|
for (i, err) in errs.iter().enumerate() {
|
||||||
let mut found = false;
|
let mut found = false;
|
||||||
if let Some(err) = err {
|
if let Some(err) = err
|
||||||
if err == &DiskError::DiskNotFound {
|
&& err == &DiskError::DiskNotFound
|
||||||
|
{
|
||||||
found = true;
|
found = true;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
for p in data_errs_by_part {
|
for p in data_errs_by_part {
|
||||||
if let Some(v) = p.1.get(i) {
|
if let Some(v) = p.1.get(i)
|
||||||
if *v == CHECK_PART_DISK_NOT_FOUND {
|
&& *v == CHECK_PART_DISK_NOT_FOUND
|
||||||
|
{
|
||||||
found = true;
|
found = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if found {
|
if found {
|
||||||
offline += 1;
|
offline += 1;
|
||||||
@@ -3838,11 +3837,11 @@ impl ObjectIO for SetDisks {
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(http_preconditions) = opts.http_preconditions.clone() {
|
if let Some(http_preconditions) = opts.http_preconditions.clone()
|
||||||
if let Some(err) = self.check_write_precondition(bucket, object, opts).await {
|
&& let Some(err) = self.check_write_precondition(bucket, object, opts).await
|
||||||
|
{
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let mut user_defined = opts.user_defined.clone();
|
let mut user_defined = opts.user_defined.clone();
|
||||||
|
|
||||||
@@ -4002,17 +4001,17 @@ impl ObjectIO for SetDisks {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if fi.checksum.is_none() {
|
if fi.checksum.is_none()
|
||||||
if let Some(content_hash) = data.as_hash_reader().content_hash() {
|
&& let Some(content_hash) = data.as_hash_reader().content_hash()
|
||||||
|
{
|
||||||
fi.checksum = Some(content_hash.to_bytes(&[]));
|
fi.checksum = Some(content_hash.to_bytes(&[]));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS) {
|
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS)
|
||||||
if sc == storageclass::STANDARD {
|
&& sc == storageclass::STANDARD
|
||||||
|
{
|
||||||
let _ = user_defined.remove(AMZ_STORAGE_CLASS);
|
let _ = user_defined.remove(AMZ_STORAGE_CLASS);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let mod_time = if let Some(mod_time) = opts.mod_time {
|
let mod_time = if let Some(mod_time) = opts.mod_time {
|
||||||
Some(mod_time)
|
Some(mod_time)
|
||||||
@@ -4062,13 +4061,13 @@ impl ObjectIO for SetDisks {
|
|||||||
self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?;
|
self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_dir).await?;
|
||||||
|
|
||||||
for (i, op_disk) in online_disks.iter().enumerate() {
|
for (i, op_disk) in online_disks.iter().enumerate() {
|
||||||
if let Some(disk) = op_disk {
|
if let Some(disk) = op_disk
|
||||||
if disk.is_online().await {
|
&& disk.is_online().await
|
||||||
|
{
|
||||||
fi = parts_metadatas[i].clone();
|
fi = parts_metadatas[i].clone();
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
fi.replication_state_internal = Some(opts.put_replication_state());
|
fi.replication_state_internal = Some(opts.put_replication_state());
|
||||||
|
|
||||||
@@ -5568,11 +5567,11 @@ impl StorageAPI for SetDisks {
|
|||||||
user_defined.insert("etag".to_owned(), etag.clone());
|
user_defined.insert("etag".to_owned(), etag.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS) {
|
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS)
|
||||||
if sc == storageclass::STANDARD {
|
&& sc == storageclass::STANDARD
|
||||||
|
{
|
||||||
let _ = user_defined.remove(AMZ_STORAGE_CLASS);
|
let _ = user_defined.remove(AMZ_STORAGE_CLASS);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let sc_parity_drives = {
|
let sc_parity_drives = {
|
||||||
if let Some(sc) = GLOBAL_STORAGE_CLASS.get() {
|
if let Some(sc) = GLOBAL_STORAGE_CLASS.get() {
|
||||||
@@ -5620,11 +5619,11 @@ impl StorageAPI for SetDisks {
|
|||||||
// TODO: get content-type
|
// TODO: get content-type
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS) {
|
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS)
|
||||||
if sc == storageclass::STANDARD {
|
&& sc == storageclass::STANDARD
|
||||||
|
{
|
||||||
let _ = user_defined.remove(AMZ_STORAGE_CLASS);
|
let _ = user_defined.remove(AMZ_STORAGE_CLASS);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(checksum) = &opts.want_checksum {
|
if let Some(checksum) = &opts.want_checksum {
|
||||||
user_defined.insert(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM.to_string(), checksum.checksum_type.to_string());
|
user_defined.insert(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM.to_string(), checksum.checksum_type.to_string());
|
||||||
@@ -5925,15 +5924,15 @@ impl StorageAPI for SetDisks {
|
|||||||
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
|
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
|
||||||
}
|
}
|
||||||
|
|
||||||
if checksum_type.full_object_requested() {
|
if checksum_type.full_object_requested()
|
||||||
if let Err(err) = checksum.add_part(&cs, ext_part.actual_size) {
|
&& let Err(err) = checksum.add_part(&cs, ext_part.actual_size)
|
||||||
|
{
|
||||||
error!(
|
error!(
|
||||||
"complete_multipart_upload checksum add_part failed part_id={}, bucket={}, object={}",
|
"complete_multipart_upload checksum add_part failed part_id={}, bucket={}, object={}",
|
||||||
p.part_num, bucket, object
|
p.part_num, bucket, object
|
||||||
);
|
);
|
||||||
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
|
return Err(Error::InvalidPart(p.part_num, ext_part.etag.clone(), p.etag.clone().unwrap_or_default()));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
checksum_combined.extend_from_slice(cs.raw.as_slice());
|
checksum_combined.extend_from_slice(cs.raw.as_slice());
|
||||||
}
|
}
|
||||||
@@ -6112,13 +6111,13 @@ impl StorageAPI for SetDisks {
|
|||||||
});
|
});
|
||||||
|
|
||||||
for (i, op_disk) in online_disks.iter().enumerate() {
|
for (i, op_disk) in online_disks.iter().enumerate() {
|
||||||
if let Some(disk) = op_disk {
|
if let Some(disk) = op_disk
|
||||||
if disk.is_online().await {
|
&& disk.is_online().await
|
||||||
|
{
|
||||||
fi = parts_metadatas[i].clone();
|
fi = parts_metadatas[i].clone();
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
fi.is_latest = true;
|
fi.is_latest = true;
|
||||||
|
|
||||||
@@ -6210,8 +6209,8 @@ impl StorageAPI for SetDisks {
|
|||||||
let _write_lock_guard = if !opts.no_lock {
|
let _write_lock_guard = if !opts.no_lock {
|
||||||
let key = rustfs_lock::fast_lock::types::ObjectKey::new(bucket, object);
|
let key = rustfs_lock::fast_lock::types::ObjectKey::new(bucket, object);
|
||||||
let mut skip_lock = false;
|
let mut skip_lock = false;
|
||||||
if let Some(lock_info) = self.fast_lock_manager.get_lock_info(&key) {
|
if let Some(lock_info) = self.fast_lock_manager.get_lock_info(&key)
|
||||||
if lock_info.owner.as_ref() == self.locker_owner.as_str()
|
&& lock_info.owner.as_ref() == self.locker_owner.as_str()
|
||||||
&& matches!(lock_info.mode, rustfs_lock::fast_lock::types::LockMode::Exclusive)
|
&& matches!(lock_info.mode, rustfs_lock::fast_lock::types::LockMode::Exclusive)
|
||||||
{
|
{
|
||||||
debug!(
|
debug!(
|
||||||
@@ -6220,7 +6219,6 @@ impl StorageAPI for SetDisks {
|
|||||||
);
|
);
|
||||||
skip_lock = true;
|
skip_lock = true;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if skip_lock {
|
if skip_lock {
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
@@ -6563,8 +6561,9 @@ async fn disks_with_all_parts(
|
|||||||
if err.is_some() {
|
if err.is_some() {
|
||||||
let part_err = conv_part_err_to_int(err);
|
let part_err = conv_part_err_to_int(err);
|
||||||
for p in 0..latest_meta.parts.len() {
|
for p in 0..latest_meta.parts.len() {
|
||||||
if let Some(vec) = data_errs_by_part.get_mut(&p) {
|
if let Some(vec) = data_errs_by_part.get_mut(&p)
|
||||||
if index < vec.len() {
|
&& index < vec.len()
|
||||||
|
{
|
||||||
info!(
|
info!(
|
||||||
"data_errs_by_part: copy meta errors to part errors: object_name={}, index: {index}, part: {p}, part_err: {part_err}",
|
"data_errs_by_part: copy meta errors to part errors: object_name={}, index: {index}, part: {p}, part_err: {part_err}",
|
||||||
object_name
|
object_name
|
||||||
@@ -6574,7 +6573,6 @@ async fn disks_with_all_parts(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Check data for each disk
|
// Check data for each disk
|
||||||
for (index, disk) in online_disks.iter().enumerate() {
|
for (index, disk) in online_disks.iter().enumerate() {
|
||||||
@@ -6609,8 +6607,9 @@ async fn disks_with_all_parts(
|
|||||||
.await
|
.await
|
||||||
.err();
|
.err();
|
||||||
|
|
||||||
if let Some(vec) = data_errs_by_part.get_mut(&0) {
|
if let Some(vec) = data_errs_by_part.get_mut(&0)
|
||||||
if index < vec.len() {
|
&& index < vec.len()
|
||||||
|
{
|
||||||
vec[index] = conv_part_err_to_int(&verify_err.map(|e| e.into()));
|
vec[index] = conv_part_err_to_int(&verify_err.map(|e| e.into()));
|
||||||
info!(
|
info!(
|
||||||
"data_errs_by_part:bitrot check result: object_name={}, index: {index}, result: {}",
|
"data_errs_by_part:bitrot check result: object_name={}, index: {index}, result: {}",
|
||||||
@@ -6618,7 +6617,6 @@ async fn disks_with_all_parts(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -6654,8 +6652,9 @@ async fn disks_with_all_parts(
|
|||||||
|
|
||||||
// Update dataErrsByPart for all parts
|
// Update dataErrsByPart for all parts
|
||||||
for p in 0..latest_meta.parts.len() {
|
for p in 0..latest_meta.parts.len() {
|
||||||
if let Some(vec) = data_errs_by_part.get_mut(&p) {
|
if let Some(vec) = data_errs_by_part.get_mut(&p)
|
||||||
if index < vec.len() {
|
&& index < vec.len()
|
||||||
|
{
|
||||||
if verify_err.is_some() {
|
if verify_err.is_some() {
|
||||||
info!(
|
info!(
|
||||||
"data_errs_by_part: verify_err: object_name={}, index: {index}, part: {p}, verify_err: {verify_err:?}",
|
"data_errs_by_part: verify_err: object_name={}, index: {index}, part: {p}, verify_err: {verify_err:?}",
|
||||||
@@ -6684,13 +6683,13 @@ async fn disks_with_all_parts(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Build dataErrsByDisk from dataErrsByPart
|
// Build dataErrsByDisk from dataErrsByPart
|
||||||
for (part, disks) in data_errs_by_part.iter() {
|
for (part, disks) in data_errs_by_part.iter() {
|
||||||
for (disk_idx, disk_err) in disks.iter().enumerate() {
|
for (disk_idx, disk_err) in disks.iter().enumerate() {
|
||||||
if let Some(vec) = data_errs_by_disk.get_mut(&disk_idx) {
|
if let Some(vec) = data_errs_by_disk.get_mut(&disk_idx)
|
||||||
if *part < vec.len() {
|
&& *part < vec.len()
|
||||||
|
{
|
||||||
vec[*part] = *disk_err;
|
vec[*part] = *disk_err;
|
||||||
info!(
|
info!(
|
||||||
"data_errs_by_disk: update data_errs_by_disk: object_name={}, part: {part}, disk_idx: {disk_idx}, disk_err: {disk_err}",
|
"data_errs_by_disk: update data_errs_by_disk: object_name={}, part: {part}, disk_idx: {disk_idx}, disk_err: {disk_err}",
|
||||||
@@ -6699,7 +6698,6 @@ async fn disks_with_all_parts(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate available_disks based on meta_errs and data_errs_by_disk
|
// Calculate available_disks based on meta_errs and data_errs_by_disk
|
||||||
for (i, disk) in online_disks.iter().enumerate() {
|
for (i, disk) in online_disks.iter().enumerate() {
|
||||||
@@ -6738,11 +6736,11 @@ pub fn should_heal_object_on_disk(
|
|||||||
meta: &FileInfo,
|
meta: &FileInfo,
|
||||||
latest_meta: &FileInfo,
|
latest_meta: &FileInfo,
|
||||||
) -> (bool, Option<DiskError>) {
|
) -> (bool, Option<DiskError>) {
|
||||||
if let Some(err) = err {
|
if let Some(err) = err
|
||||||
if err == &DiskError::FileNotFound || err == &DiskError::FileVersionNotFound || err == &DiskError::FileCorrupt {
|
&& (err == &DiskError::FileNotFound || err == &DiskError::FileVersionNotFound || err == &DiskError::FileCorrupt)
|
||||||
|
{
|
||||||
return (true, Some(err.clone()));
|
return (true, Some(err.clone()));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if latest_meta.volume != meta.volume
|
if latest_meta.volume != meta.volume
|
||||||
|| latest_meta.name != meta.name
|
|| latest_meta.name != meta.name
|
||||||
@@ -6906,16 +6904,16 @@ pub fn e_tag_matches(etag: &str, condition: &str) -> bool {
|
|||||||
pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<String>, if_match: Option<String>) -> bool {
|
pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<String>, if_match: Option<String>) -> bool {
|
||||||
match &oi.etag {
|
match &oi.etag {
|
||||||
Some(etag) => {
|
Some(etag) => {
|
||||||
if let Some(if_none_match) = if_none_match {
|
if let Some(if_none_match) = if_none_match
|
||||||
if e_tag_matches(etag, &if_none_match) {
|
&& e_tag_matches(etag, &if_none_match)
|
||||||
|
{
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
if let Some(if_match) = if_match
|
||||||
if let Some(if_match) = if_match {
|
&& !e_tag_matches(etag, &if_match)
|
||||||
if !e_tag_matches(etag, &if_match) {
|
{
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
// If we can't obtain the etag of the object, perevent the write only when we have at least one condition
|
// If we can't obtain the etag of the object, perevent the write only when we have at least one condition
|
||||||
|
|||||||
+13
-14
@@ -491,13 +491,13 @@ impl StorageAPI for Sets {
|
|||||||
let cp_src_dst_same = path_join_buf(&[src_bucket, src_object]) == path_join_buf(&[dst_bucket, dst_object]);
|
let cp_src_dst_same = path_join_buf(&[src_bucket, src_object]) == path_join_buf(&[dst_bucket, dst_object]);
|
||||||
|
|
||||||
if cp_src_dst_same {
|
if cp_src_dst_same {
|
||||||
if let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id) {
|
if let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id)
|
||||||
if src_vid == dst_vid {
|
&& src_vid == dst_vid
|
||||||
|
{
|
||||||
return src_set
|
return src_set
|
||||||
.copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
|
.copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if !dst_opts.versioned && src_opts.version_id.is_none() {
|
if !dst_opts.versioned && src_opts.version_id.is_none() {
|
||||||
return src_set
|
return src_set
|
||||||
@@ -823,11 +823,11 @@ impl StorageAPI for Sets {
|
|||||||
Ok((m, n)) => (m, n),
|
Ok((m, n)) => (m, n),
|
||||||
Err(_) => continue,
|
Err(_) => continue,
|
||||||
};
|
};
|
||||||
if let Some(set) = self.disk_set.get(m) {
|
if let Some(set) = self.disk_set.get(m)
|
||||||
if let Some(Some(disk)) = set.disks.read().await.get(n) {
|
&& let Some(Some(disk)) = set.disks.read().await.get(n)
|
||||||
|
{
|
||||||
let _ = disk.close().await;
|
let _ = disk.close().await;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(Some(disk)) = disks.get(index) {
|
if let Some(Some(disk)) = disks.get(index) {
|
||||||
self.disk_set[m].renew_disk(&disk.endpoint()).await;
|
self.disk_set[m].renew_disk(&disk.endpoint()).await;
|
||||||
@@ -980,8 +980,9 @@ fn new_heal_format_sets(
|
|||||||
let mut current_disks_info = vec![vec![DiskInfo::default(); set_drive_count]; set_count];
|
let mut current_disks_info = vec![vec![DiskInfo::default(); set_drive_count]; set_count];
|
||||||
for (i, set) in ref_format.erasure.sets.iter().enumerate() {
|
for (i, set) in ref_format.erasure.sets.iter().enumerate() {
|
||||||
for j in 0..set.len() {
|
for j in 0..set.len() {
|
||||||
if let Some(Some(err)) = errs.get(i * set_drive_count + j) {
|
if let Some(Some(err)) = errs.get(i * set_drive_count + j)
|
||||||
if *err == DiskError::UnformattedDisk {
|
&& *err == DiskError::UnformattedDisk
|
||||||
|
{
|
||||||
let mut fm = FormatV3::new(set_count, set_drive_count);
|
let mut fm = FormatV3::new(set_count, set_drive_count);
|
||||||
fm.id = ref_format.id;
|
fm.id = ref_format.id;
|
||||||
fm.format = ref_format.format.clone();
|
fm.format = ref_format.format.clone();
|
||||||
@@ -992,16 +993,14 @@ fn new_heal_format_sets(
|
|||||||
fm.erasure.distribution_algo = ref_format.erasure.distribution_algo.clone();
|
fm.erasure.distribution_algo = ref_format.erasure.distribution_algo.clone();
|
||||||
new_formats[i][j] = Some(fm);
|
new_formats[i][j] = Some(fm);
|
||||||
}
|
}
|
||||||
}
|
if let (Some(format), None) = (&formats[i * set_drive_count + j], &errs[i * set_drive_count + j])
|
||||||
if let (Some(format), None) = (&formats[i * set_drive_count + j], &errs[i * set_drive_count + j]) {
|
&& let Some(info) = &format.disk_info
|
||||||
if let Some(info) = &format.disk_info {
|
&& !info.endpoint.is_empty()
|
||||||
if !info.endpoint.is_empty() {
|
{
|
||||||
current_disks_info[i][j] = info.clone();
|
current_disks_info[i][j] = info.clone();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
(new_formats, current_disks_info)
|
(new_formats, current_disks_info)
|
||||||
}
|
}
|
||||||
|
|||||||
+25
-24
@@ -243,11 +243,11 @@ impl ECStore {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Only set it when the global deployment ID is not yet configured
|
// Only set it when the global deployment ID is not yet configured
|
||||||
if let Some(dep_id) = deployment_id {
|
if let Some(dep_id) = deployment_id
|
||||||
if get_global_deployment_id().is_none() {
|
&& get_global_deployment_id().is_none()
|
||||||
|
{
|
||||||
set_global_deployment_id(dep_id);
|
set_global_deployment_id(dep_id);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let wait_sec = 5;
|
let wait_sec = 5;
|
||||||
let mut exit_count = 0;
|
let mut exit_count = 0;
|
||||||
@@ -768,11 +768,11 @@ impl ECStore {
|
|||||||
def_pool = pinfo.clone();
|
def_pool = pinfo.clone();
|
||||||
has_def_pool = true;
|
has_def_pool = true;
|
||||||
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-deletes.html
|
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-deletes.html
|
||||||
if is_err_object_not_found(err) {
|
if is_err_object_not_found(err)
|
||||||
if let Err(err) = opts.precondition_check(&pinfo.object_info) {
|
&& let Err(err) = opts.precondition_check(&pinfo.object_info)
|
||||||
|
{
|
||||||
return Err(err.clone());
|
return Err(err.clone());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if !is_err_object_not_found(err) && !is_err_version_not_found(err) {
|
if !is_err_object_not_found(err) && !is_err_version_not_found(err) {
|
||||||
return Err(err.clone());
|
return Err(err.clone());
|
||||||
@@ -885,14 +885,15 @@ impl ECStore {
|
|||||||
return Ok((obj, res.idx));
|
return Ok((obj, res.idx));
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(err) = res.err {
|
if let Some(err) = res.err
|
||||||
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
|
&& !is_err_object_not_found(&err)
|
||||||
|
&& !is_err_version_not_found(&err)
|
||||||
|
{
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: delete marker
|
// TODO: delete marker
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let object = decode_dir_object(object);
|
let object = decode_dir_object(object);
|
||||||
|
|
||||||
@@ -918,13 +919,13 @@ impl ECStore {
|
|||||||
let mut derrs = Vec::new();
|
let mut derrs = Vec::new();
|
||||||
|
|
||||||
for pe in errs.iter() {
|
for pe in errs.iter() {
|
||||||
if let Some(err) = &pe.err {
|
if let Some(err) = &pe.err
|
||||||
if err == &StorageError::ErasureWriteQuorum {
|
&& err == &StorageError::ErasureWriteQuorum
|
||||||
|
{
|
||||||
objs.push(None);
|
objs.push(None);
|
||||||
derrs.push(Some(StorageError::ErasureWriteQuorum));
|
derrs.push(Some(StorageError::ErasureWriteQuorum));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(idx) = pe.index {
|
if let Some(idx) = pe.index {
|
||||||
match self.pools[idx].delete_object(bucket, object, opts.clone()).await {
|
match self.pools[idx].delete_object(bucket, object, opts.clone()).await {
|
||||||
@@ -1226,13 +1227,13 @@ impl StorageAPI for ECStore {
|
|||||||
|
|
||||||
#[instrument(skip(self))]
|
#[instrument(skip(self))]
|
||||||
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
|
async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> {
|
||||||
if !is_meta_bucketname(bucket) {
|
if !is_meta_bucketname(bucket)
|
||||||
if let Err(err) = check_valid_bucket_name_strict(bucket) {
|
&& let Err(err) = check_valid_bucket_name_strict(bucket)
|
||||||
|
{
|
||||||
return Err(StorageError::BucketNameInvalid(err.to_string()));
|
return Err(StorageError::BucketNameInvalid(err.to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: nslock
|
// TODO: nslock
|
||||||
}
|
|
||||||
|
|
||||||
if let Err(err) = self.peer_sys.make_bucket(bucket, opts).await {
|
if let Err(err) = self.peer_sys.make_bucket(bucket, opts).await {
|
||||||
let err = to_object_err(err.into(), vec![bucket]);
|
let err = to_object_err(err.into(), vec![bucket]);
|
||||||
@@ -1427,13 +1428,13 @@ impl StorageAPI for ECStore {
|
|||||||
let pool_idx = self.get_pool_idx_no_lock(src_bucket, &src_object, src_info.size).await?;
|
let pool_idx = self.get_pool_idx_no_lock(src_bucket, &src_object, src_info.size).await?;
|
||||||
|
|
||||||
if cp_src_dst_same {
|
if cp_src_dst_same {
|
||||||
if let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id) {
|
if let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id)
|
||||||
if src_vid == dst_vid {
|
&& src_vid == dst_vid
|
||||||
|
{
|
||||||
return self.pools[pool_idx]
|
return self.pools[pool_idx]
|
||||||
.copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, dst_opts)
|
.copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, dst_opts)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if !dst_opts.versioned && src_opts.version_id.is_none() {
|
if !dst_opts.versioned && src_opts.version_id.is_none() {
|
||||||
return self.pools[pool_idx]
|
return self.pools[pool_idx]
|
||||||
@@ -2433,14 +2434,14 @@ fn check_list_multipart_args(
|
|||||||
check_list_objs_args(bucket, prefix, key_marker)?;
|
check_list_objs_args(bucket, prefix, key_marker)?;
|
||||||
|
|
||||||
if let Some(upload_id_marker) = upload_id_marker {
|
if let Some(upload_id_marker) = upload_id_marker {
|
||||||
if let Some(key_marker) = key_marker {
|
if let Some(key_marker) = key_marker
|
||||||
if key_marker.ends_with('/') {
|
&& key_marker.ends_with('/')
|
||||||
|
{
|
||||||
return Err(StorageError::InvalidUploadIDKeyCombination(
|
return Err(StorageError::InvalidUploadIDKeyCombination(
|
||||||
upload_id_marker.to_string(),
|
upload_id_marker.to_string(),
|
||||||
key_marker.to_string(),
|
key_marker.to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Err(_e) = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id_marker.as_bytes()) {
|
if let Err(_e) = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id_marker.as_bytes()) {
|
||||||
return Err(StorageError::MalformedUploadID(upload_id_marker.to_owned()));
|
return Err(StorageError::MalformedUploadID(upload_id_marker.to_owned()));
|
||||||
@@ -2510,12 +2511,12 @@ pub async fn get_disk_infos(disks: &[Option<DiskStore>]) -> Vec<Option<DiskInfo>
|
|||||||
let opts = &DiskInfoOptions::default();
|
let opts = &DiskInfoOptions::default();
|
||||||
let mut res = vec![None; disks.len()];
|
let mut res = vec![None; disks.len()];
|
||||||
for (idx, disk_op) in disks.iter().enumerate() {
|
for (idx, disk_op) in disks.iter().enumerate() {
|
||||||
if let Some(disk) = disk_op {
|
if let Some(disk) = disk_op
|
||||||
if let Ok(info) = disk.disk_info(opts).await {
|
&& let Ok(info) = disk.disk_info(opts).await
|
||||||
|
{
|
||||||
res[idx] = Some(info);
|
res[idx] = Some(info);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
res
|
res
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -144,11 +144,11 @@ impl GetObjectReader {
|
|||||||
) -> Result<(Self, usize, i64)> {
|
) -> Result<(Self, usize, i64)> {
|
||||||
let mut rs = rs;
|
let mut rs = rs;
|
||||||
|
|
||||||
if let Some(part_number) = opts.part_number {
|
if let Some(part_number) = opts.part_number
|
||||||
if rs.is_none() {
|
&& rs.is_none()
|
||||||
|
{
|
||||||
rs = HTTPRangeSpec::from_object_info(oi, part_number);
|
rs = HTTPRangeSpec::from_object_info(oi, part_number);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// TODO:Encrypted
|
// TODO:Encrypted
|
||||||
|
|
||||||
@@ -462,33 +462,31 @@ impl ObjectOptions {
|
|||||||
pub fn precondition_check(&self, obj_info: &ObjectInfo) -> Result<()> {
|
pub fn precondition_check(&self, obj_info: &ObjectInfo) -> Result<()> {
|
||||||
let has_valid_mod_time = obj_info.mod_time.is_some_and(|t| t != OffsetDateTime::UNIX_EPOCH);
|
let has_valid_mod_time = obj_info.mod_time.is_some_and(|t| t != OffsetDateTime::UNIX_EPOCH);
|
||||||
|
|
||||||
if let Some(part_number) = self.part_number {
|
if let Some(part_number) = self.part_number
|
||||||
if part_number > 1 && !obj_info.parts.is_empty() {
|
&& part_number > 1
|
||||||
|
&& !obj_info.parts.is_empty()
|
||||||
|
{
|
||||||
let part_found = obj_info.parts.iter().any(|pi| pi.number == part_number);
|
let part_found = obj_info.parts.iter().any(|pi| pi.number == part_number);
|
||||||
if !part_found {
|
if !part_found {
|
||||||
return Err(Error::InvalidPartNumber(part_number));
|
return Err(Error::InvalidPartNumber(part_number));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(pre) = &self.http_preconditions {
|
if let Some(pre) = &self.http_preconditions {
|
||||||
if let Some(if_none_match) = &pre.if_none_match {
|
if let Some(if_none_match) = &pre.if_none_match
|
||||||
if let Some(etag) = &obj_info.etag {
|
&& let Some(etag) = &obj_info.etag
|
||||||
if is_etag_equal(etag, if_none_match) {
|
&& is_etag_equal(etag, if_none_match)
|
||||||
|
{
|
||||||
return Err(Error::NotModified);
|
return Err(Error::NotModified);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if has_valid_mod_time {
|
if has_valid_mod_time
|
||||||
if let Some(if_modified_since) = &pre.if_modified_since {
|
&& let Some(if_modified_since) = &pre.if_modified_since
|
||||||
if let Some(mod_time) = &obj_info.mod_time {
|
&& let Some(mod_time) = &obj_info.mod_time
|
||||||
if !is_modified_since(mod_time, if_modified_since) {
|
&& !is_modified_since(mod_time, if_modified_since)
|
||||||
|
{
|
||||||
return Err(Error::NotModified);
|
return Err(Error::NotModified);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(if_match) = &pre.if_match {
|
if let Some(if_match) = &pre.if_match {
|
||||||
if let Some(etag) = &obj_info.etag {
|
if let Some(etag) = &obj_info.etag {
|
||||||
@@ -499,16 +497,15 @@ impl ObjectOptions {
|
|||||||
return Err(Error::PreconditionFailed);
|
return Err(Error::PreconditionFailed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if has_valid_mod_time && pre.if_match.is_none() {
|
if has_valid_mod_time
|
||||||
if let Some(if_unmodified_since) = &pre.if_unmodified_since {
|
&& pre.if_match.is_none()
|
||||||
if let Some(mod_time) = &obj_info.mod_time {
|
&& let Some(if_unmodified_since) = &pre.if_unmodified_since
|
||||||
if is_modified_since(mod_time, if_unmodified_since) {
|
&& let Some(mod_time) = &obj_info.mod_time
|
||||||
|
&& is_modified_since(mod_time, if_unmodified_since)
|
||||||
|
{
|
||||||
return Err(Error::PreconditionFailed);
|
return Err(Error::PreconditionFailed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -698,13 +695,13 @@ impl ObjectInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if self.is_compressed() {
|
if self.is_compressed() {
|
||||||
if let Some(size_str) = self.user_defined.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size")) {
|
if let Some(size_str) = self.user_defined.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}actual-size"))
|
||||||
if !size_str.is_empty() {
|
&& !size_str.is_empty()
|
||||||
|
{
|
||||||
// Todo: deal with error
|
// Todo: deal with error
|
||||||
let size = size_str.parse::<i64>().map_err(|e| std::io::Error::other(e.to_string()))?;
|
let size = size_str.parse::<i64>().map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||||
return Ok(size);
|
return Ok(size);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
let mut actual_size = 0;
|
let mut actual_size = 0;
|
||||||
self.parts.iter().for_each(|part| {
|
self.parts.iter().for_each(|part| {
|
||||||
actual_size += part.actual_size;
|
actual_size += part.actual_size;
|
||||||
@@ -881,16 +878,17 @@ impl ObjectInfo {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if entry.is_dir() {
|
if entry.is_dir()
|
||||||
if let Some(delimiter) = &delimiter {
|
&& let Some(delimiter) = &delimiter
|
||||||
if let Some(idx) = {
|
&& let Some(idx) = {
|
||||||
let remaining = if entry.name.starts_with(prefix) {
|
let remaining = if entry.name.starts_with(prefix) {
|
||||||
&entry.name[prefix.len()..]
|
&entry.name[prefix.len()..]
|
||||||
} else {
|
} else {
|
||||||
entry.name.as_str()
|
entry.name.as_str()
|
||||||
};
|
};
|
||||||
remaining.find(delimiter.as_str())
|
remaining.find(delimiter.as_str())
|
||||||
} {
|
}
|
||||||
|
{
|
||||||
let idx = prefix.len() + idx + delimiter.len();
|
let idx = prefix.len() + idx + delimiter.len();
|
||||||
if let Some(curr_prefix) = entry.name.get(0..idx) {
|
if let Some(curr_prefix) = entry.name.get(0..idx) {
|
||||||
if curr_prefix == prev_prefix {
|
if curr_prefix == prev_prefix {
|
||||||
@@ -908,8 +906,6 @@ impl ObjectInfo {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
objects
|
objects
|
||||||
}
|
}
|
||||||
@@ -966,16 +962,17 @@ impl ObjectInfo {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if entry.is_dir() {
|
if entry.is_dir()
|
||||||
if let Some(delimiter) = &delimiter {
|
&& let Some(delimiter) = &delimiter
|
||||||
if let Some(idx) = {
|
&& let Some(idx) = {
|
||||||
let remaining = if entry.name.starts_with(prefix) {
|
let remaining = if entry.name.starts_with(prefix) {
|
||||||
&entry.name[prefix.len()..]
|
&entry.name[prefix.len()..]
|
||||||
} else {
|
} else {
|
||||||
entry.name.as_str()
|
entry.name.as_str()
|
||||||
};
|
};
|
||||||
remaining.find(delimiter.as_str())
|
remaining.find(delimiter.as_str())
|
||||||
} {
|
}
|
||||||
|
{
|
||||||
let idx = prefix.len() + idx + delimiter.len();
|
let idx = prefix.len() + idx + delimiter.len();
|
||||||
if let Some(curr_prefix) = entry.name.get(0..idx) {
|
if let Some(curr_prefix) = entry.name.get(0..idx) {
|
||||||
if curr_prefix == prev_prefix {
|
if curr_prefix == prev_prefix {
|
||||||
@@ -993,8 +990,6 @@ impl ObjectInfo {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
objects
|
objects
|
||||||
}
|
}
|
||||||
@@ -1026,11 +1021,11 @@ impl ObjectInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn decrypt_checksums(&self, part: usize, _headers: &HeaderMap) -> Result<(HashMap<String, String>, bool)> {
|
pub fn decrypt_checksums(&self, part: usize, _headers: &HeaderMap) -> Result<(HashMap<String, String>, bool)> {
|
||||||
if part > 0 {
|
if part > 0
|
||||||
if let Some(checksums) = self.parts.iter().find(|p| p.number == part).and_then(|p| p.checksums.clone()) {
|
&& let Some(checksums) = self.parts.iter().find(|p| p.number == part).and_then(|p| p.checksums.clone())
|
||||||
|
{
|
||||||
return Ok((checksums, true));
|
return Ok((checksums, true));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: decrypt checksums
|
// TODO: decrypt checksums
|
||||||
|
|
||||||
|
|||||||
@@ -302,11 +302,11 @@ impl ECStore {
|
|||||||
..Default::default()
|
..Default::default()
|
||||||
});
|
});
|
||||||
|
|
||||||
if let Some(err) = list_result.err.clone() {
|
if let Some(err) = list_result.err.clone()
|
||||||
if err != rustfs_filemeta::Error::Unexpected {
|
&& err != rustfs_filemeta::Error::Unexpected
|
||||||
|
{
|
||||||
return Err(to_object_err(err.into(), vec![bucket, prefix]));
|
return Err(to_object_err(err.into(), vec![bucket, prefix]));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(result) = list_result.entries.as_mut() {
|
if let Some(result) = list_result.entries.as_mut() {
|
||||||
result.forward_past(opts.marker);
|
result.forward_past(opts.marker);
|
||||||
@@ -418,11 +418,11 @@ impl ECStore {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(err) = list_result.err.clone() {
|
if let Some(err) = list_result.err.clone()
|
||||||
if err != rustfs_filemeta::Error::Unexpected {
|
&& err != rustfs_filemeta::Error::Unexpected
|
||||||
|
{
|
||||||
return Err(to_object_err(err.into(), vec![bucket, prefix]));
|
return Err(to_object_err(err.into(), vec![bucket, prefix]));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(result) = list_result.entries.as_mut() {
|
if let Some(result) = list_result.entries.as_mut() {
|
||||||
result.forward_past(opts.marker);
|
result.forward_past(opts.marker);
|
||||||
@@ -509,11 +509,12 @@ impl ECStore {
|
|||||||
let mut o = o.clone();
|
let mut o = o.clone();
|
||||||
o.marker = o.marker.filter(|v| v >= &o.prefix);
|
o.marker = o.marker.filter(|v| v >= &o.prefix);
|
||||||
|
|
||||||
if let Some(marker) = &o.marker {
|
if let Some(marker) = &o.marker
|
||||||
if !o.prefix.is_empty() && !marker.starts_with(&o.prefix) {
|
&& !o.prefix.is_empty()
|
||||||
|
&& !marker.starts_with(&o.prefix)
|
||||||
|
{
|
||||||
return Err(Error::Unexpected);
|
return Err(Error::Unexpected);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if o.limit == 0 {
|
if o.limit == 0 {
|
||||||
return Err(Error::Unexpected);
|
return Err(Error::Unexpected);
|
||||||
@@ -817,12 +818,12 @@ impl ECStore {
|
|||||||
let value = tx2.clone();
|
let value = tx2.clone();
|
||||||
let resolver = resolver.clone();
|
let resolver = resolver.clone();
|
||||||
async move {
|
async move {
|
||||||
if let Some(entry) = entries.resolve(resolver) {
|
if let Some(entry) = entries.resolve(resolver)
|
||||||
if let Err(err) = value.send(entry).await {
|
&& let Err(err) = value.send(entry).await
|
||||||
|
{
|
||||||
error!("list_path send fail {:?}", err);
|
error!("list_path send fail {:?}", err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
})
|
})
|
||||||
})),
|
})),
|
||||||
finished: None,
|
finished: None,
|
||||||
@@ -986,21 +987,22 @@ async fn gather_results(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(marker) = &opts.marker {
|
if let Some(marker) = &opts.marker
|
||||||
if &entry.name < marker {
|
&& &entry.name < marker
|
||||||
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if !entry.name.starts_with(&opts.prefix) {
|
if !entry.name.starts_with(&opts.prefix) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(separator) = &opts.separator {
|
if let Some(separator) = &opts.separator
|
||||||
if !opts.recursive && !entry.is_in_dir(&opts.prefix, separator) {
|
&& !opts.recursive
|
||||||
|
&& !entry.is_in_dir(&opts.prefix, separator)
|
||||||
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if !opts.incl_deleted && entry.is_object() && entry.is_latest_delete_marker() && !entry.is_object_dir() {
|
if !opts.incl_deleted && entry.is_object() && entry.is_latest_delete_marker() && !entry.is_object_dir() {
|
||||||
continue;
|
continue;
|
||||||
@@ -1200,29 +1202,29 @@ async fn merge_entry_channels(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(xl) = has_xl.as_mut() {
|
if let Some(xl) = has_xl.as_mut()
|
||||||
if !versions.is_empty() {
|
&& !versions.is_empty()
|
||||||
|
{
|
||||||
xl.versions = merge_file_meta_versions(read_quorum, true, 0, &versions);
|
xl.versions = merge_file_meta_versions(read_quorum, true, 0, &versions);
|
||||||
|
|
||||||
if let Ok(meta) = xl.marshal_msg() {
|
if let Ok(meta) = xl.marshal_msg()
|
||||||
if let Some(b) = best.as_mut() {
|
&& let Some(b) = best.as_mut()
|
||||||
|
{
|
||||||
b.metadata = meta;
|
b.metadata = meta;
|
||||||
b.cached = Some(xl.clone());
|
b.cached = Some(xl.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
to_merge.clear();
|
to_merge.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(best_entry) = &best {
|
if let Some(best_entry) = &best
|
||||||
if best_entry.name > last {
|
&& best_entry.name > last
|
||||||
|
{
|
||||||
out_channel.send(best_entry.clone()).await.map_err(Error::other)?;
|
out_channel.send(best_entry.clone()).await.map_err(Error::other)?;
|
||||||
last = best_entry.name.clone();
|
last = best_entry.name.clone();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
select_from(&mut in_channels, best_idx, &mut top, &mut n_done).await?;
|
select_from(&mut in_channels, best_idx, &mut top, &mut n_done).await?;
|
||||||
}
|
}
|
||||||
@@ -1307,12 +1309,12 @@ impl SetDisks {
|
|||||||
let value = tx2.clone();
|
let value = tx2.clone();
|
||||||
let resolver = resolver.clone();
|
let resolver = resolver.clone();
|
||||||
async move {
|
async move {
|
||||||
if let Some(entry) = entries.resolve(resolver) {
|
if let Some(entry) = entries.resolve(resolver)
|
||||||
if let Err(err) = value.send(entry).await {
|
&& let Err(err) = value.send(entry).await
|
||||||
|
{
|
||||||
error!("list_path send fail {:?}", err);
|
error!("list_path send fail {:?}", err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
})
|
})
|
||||||
})),
|
})),
|
||||||
finished: None,
|
finished: None,
|
||||||
|
|||||||
@@ -635,10 +635,10 @@ fn parse_restore_obj_status(restore_hdr: &str) -> Result<RestoreStatus> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn is_restored_object_on_disk(meta: &HashMap<String, String>) -> bool {
|
pub fn is_restored_object_on_disk(meta: &HashMap<String, String>) -> bool {
|
||||||
if let Some(restore_hdr) = meta.get(X_AMZ_RESTORE.as_str()) {
|
if let Some(restore_hdr) = meta.get(X_AMZ_RESTORE.as_str())
|
||||||
if let Ok(restore_status) = parse_restore_obj_status(restore_hdr) {
|
&& let Ok(restore_status) = parse_restore_obj_status(restore_hdr)
|
||||||
|
{
|
||||||
return restore_status.on_disk();
|
return restore_status.on_disk();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -575,15 +575,14 @@ impl FileMeta {
|
|||||||
let mod_time = version.get_mod_time();
|
let mod_time = version.get_mod_time();
|
||||||
|
|
||||||
for (idx, exist) in self.versions.iter().enumerate() {
|
for (idx, exist) in self.versions.iter().enumerate() {
|
||||||
if let Some(ref ex_mt) = exist.header.mod_time {
|
if let Some(ref ex_mt) = exist.header.mod_time
|
||||||
if let Some(ref in_md) = mod_time {
|
&& let Some(ref in_md) = mod_time
|
||||||
if ex_mt <= in_md {
|
&& ex_mt <= in_md
|
||||||
|
{
|
||||||
self.versions.insert(idx, FileMetaShallowVersion::try_from(version)?);
|
self.versions.insert(idx, FileMetaShallowVersion::try_from(version)?);
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(Error::other("add_version failed"))
|
Err(Error::other("add_version failed"))
|
||||||
|
|
||||||
// if !ver.valid() {
|
// if !ver.valid() {
|
||||||
@@ -657,8 +656,9 @@ impl FileMeta {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if fi.deleted {
|
if fi.deleted {
|
||||||
if !fi.delete_marker_replication_status().is_empty() {
|
if !fi.delete_marker_replication_status().is_empty()
|
||||||
if let Some(delete_marker) = ventry.delete_marker.as_mut() {
|
&& let Some(delete_marker) = ventry.delete_marker.as_mut()
|
||||||
|
{
|
||||||
if fi.delete_marker_replication_status() == ReplicationStatusType::Replica {
|
if fi.delete_marker_replication_status() == ReplicationStatusType::Replica {
|
||||||
delete_marker.meta_sys.insert(
|
delete_marker.meta_sys.insert(
|
||||||
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-status"),
|
format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, "replica-status"),
|
||||||
@@ -700,10 +700,10 @@ impl FileMeta {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if !fi.version_purge_status().is_empty() {
|
if !fi.version_purge_status().is_empty()
|
||||||
if let Some(delete_marker) = ventry.delete_marker.as_mut() {
|
&& let Some(delete_marker) = ventry.delete_marker.as_mut()
|
||||||
|
{
|
||||||
delete_marker.meta_sys.insert(
|
delete_marker.meta_sys.insert(
|
||||||
VERSION_PURGE_STATUS_KEY.to_string(),
|
VERSION_PURGE_STATUS_KEY.to_string(),
|
||||||
fi.replication_state_internal
|
fi.replication_state_internal
|
||||||
@@ -714,7 +714,6 @@ impl FileMeta {
|
|||||||
.to_vec(),
|
.to_vec(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(delete_marker) = ventry.delete_marker.as_mut() {
|
if let Some(delete_marker) = ventry.delete_marker.as_mut() {
|
||||||
for (k, v) in fi
|
for (k, v) in fi
|
||||||
@@ -1917,8 +1916,8 @@ impl MetaObject {
|
|||||||
if let Some(status) = self
|
if let Some(status) = self
|
||||||
.meta_sys
|
.meta_sys
|
||||||
.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}"))
|
.get(&format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}"))
|
||||||
|
&& *status == TRANSITION_COMPLETE.as_bytes().to_vec()
|
||||||
{
|
{
|
||||||
if *status == TRANSITION_COMPLETE.as_bytes().to_vec() {
|
|
||||||
let vid = Uuid::parse_str(&fi.tier_free_version_id());
|
let vid = Uuid::parse_str(&fi.tier_free_version_id());
|
||||||
if let Err(err) = vid {
|
if let Err(err) = vid {
|
||||||
panic!("Invalid Tier Object delete marker versionId {} {}", fi.tier_free_version_id(), err);
|
panic!("Invalid Tier Object delete marker versionId {} {}", fi.tier_free_version_id(), err);
|
||||||
@@ -1953,7 +1952,6 @@ impl MetaObject {
|
|||||||
}
|
}
|
||||||
return (free_entry, true);
|
return (free_entry, true);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
(FileMetaVersion::default(), false)
|
(FileMetaVersion::default(), false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3568,8 +3566,9 @@ impl FileMeta {
|
|||||||
match version.header.version_type {
|
match version.header.version_type {
|
||||||
VersionType::Object => {
|
VersionType::Object => {
|
||||||
stats.object_versions += 1;
|
stats.object_versions += 1;
|
||||||
if let Ok(ver) = FileMetaVersion::try_from(version.meta.as_slice()) {
|
if let Ok(ver) = FileMetaVersion::try_from(version.meta.as_slice())
|
||||||
if let Some(obj) = &ver.object {
|
&& let Some(obj) = &ver.object
|
||||||
|
{
|
||||||
stats.total_size += obj.size;
|
stats.total_size += obj.size;
|
||||||
if obj.uses_data_dir() {
|
if obj.uses_data_dir() {
|
||||||
stats.versions_with_data_dir += 1;
|
stats.versions_with_data_dir += 1;
|
||||||
@@ -3579,7 +3578,6 @@ impl FileMeta {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
VersionType::Delete => stats.delete_markers += 1,
|
VersionType::Delete => stats.delete_markers += 1,
|
||||||
VersionType::Legacy => stats.legacy_versions += 1,
|
VersionType::Legacy => stats.legacy_versions += 1,
|
||||||
VersionType::Invalid => stats.invalid_versions += 1,
|
VersionType::Invalid => stats.invalid_versions += 1,
|
||||||
|
|||||||
@@ -442,13 +442,13 @@ impl MetaCacheEntriesSorted {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn forward_past(&mut self, marker: Option<String>) {
|
pub fn forward_past(&mut self, marker: Option<String>) {
|
||||||
if let Some(val) = marker {
|
if let Some(val) = marker
|
||||||
if let Some(idx) = self.o.0.iter().flatten().position(|v| v.name > val) {
|
&& let Some(idx) = self.o.0.iter().flatten().position(|v| v.name > val)
|
||||||
|
{
|
||||||
self.o.0 = self.o.0.split_off(idx);
|
self.o.0 = self.o.0.split_off(idx);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
const METACACHE_STREAM_VERSION: u8 = 2;
|
const METACACHE_STREAM_VERSION: u8 = 2;
|
||||||
|
|
||||||
@@ -788,14 +788,16 @@ impl<T: Clone + Debug + Send + 'static> Cache<T> {
|
|||||||
.duration_since(UNIX_EPOCH)
|
.duration_since(UNIX_EPOCH)
|
||||||
.expect("Time went backwards")
|
.expect("Time went backwards")
|
||||||
.as_secs();
|
.as_secs();
|
||||||
if now - self.last_update_ms.load(AtomicOrdering::SeqCst) < self.ttl.as_secs() {
|
if now - self.last_update_ms.load(AtomicOrdering::SeqCst) < self.ttl.as_secs()
|
||||||
if let Some(v) = v {
|
&& let Some(v) = v
|
||||||
|
{
|
||||||
return Ok(v);
|
return Ok(v);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if self.opts.no_wait && now - self.last_update_ms.load(AtomicOrdering::SeqCst) < self.ttl.as_secs() * 2 {
|
if self.opts.no_wait
|
||||||
if let Some(value) = v {
|
&& now - self.last_update_ms.load(AtomicOrdering::SeqCst) < self.ttl.as_secs() * 2
|
||||||
|
&& let Some(value) = v
|
||||||
|
{
|
||||||
if self.updating.try_lock().is_ok() {
|
if self.updating.try_lock().is_ok() {
|
||||||
let this = Arc::clone(&self);
|
let this = Arc::clone(&self);
|
||||||
spawn(async move {
|
spawn(async move {
|
||||||
@@ -804,18 +806,16 @@ impl<T: Clone + Debug + Send + 'static> Cache<T> {
|
|||||||
}
|
}
|
||||||
return Ok(value);
|
return Ok(value);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let _ = self.updating.lock().await;
|
let _ = self.updating.lock().await;
|
||||||
|
|
||||||
if let (Ok(duration), Some(value)) = (
|
if let (Ok(duration), Some(value)) = (
|
||||||
SystemTime::now().duration_since(UNIX_EPOCH + Duration::from_secs(self.last_update_ms.load(AtomicOrdering::SeqCst))),
|
SystemTime::now().duration_since(UNIX_EPOCH + Duration::from_secs(self.last_update_ms.load(AtomicOrdering::SeqCst))),
|
||||||
v,
|
v,
|
||||||
) {
|
) && duration < self.ttl
|
||||||
if duration < self.ttl {
|
{
|
||||||
return Ok(value);
|
return Ok(value);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
match self.update().await {
|
match self.update().await {
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
|
|||||||
@@ -270,15 +270,13 @@ impl ReplicationState {
|
|||||||
return repl_status;
|
return repl_status;
|
||||||
}
|
}
|
||||||
|
|
||||||
if repl_status == ReplicationStatusType::Completed {
|
if repl_status == ReplicationStatusType::Completed
|
||||||
if let (Some(replica_timestamp), Some(replication_timestamp)) =
|
&& let (Some(replica_timestamp), Some(replication_timestamp)) =
|
||||||
(self.replica_timestamp, self.replication_timestamp)
|
(self.replica_timestamp, self.replication_timestamp)
|
||||||
|
&& replica_timestamp > replication_timestamp
|
||||||
{
|
{
|
||||||
if replica_timestamp > replication_timestamp {
|
|
||||||
return self.replica_status.clone();
|
return self.replica_status.clone();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return repl_status;
|
return repl_status;
|
||||||
}
|
}
|
||||||
|
|||||||
+39
-39
@@ -246,8 +246,9 @@ where
|
|||||||
}
|
}
|
||||||
|
|
||||||
let sts_user = has_sts_user.map(|sts| sts.credentials.access_key.clone());
|
let sts_user = has_sts_user.map(|sts| sts.credentials.access_key.clone());
|
||||||
if let Some(ref sts) = sts_user {
|
if let Some(ref sts) = sts_user
|
||||||
if let Some(plc) = sts_policy_map.get(sts) {
|
&& let Some(plc) = sts_policy_map.get(sts)
|
||||||
|
{
|
||||||
for p in plc.to_slice().iter() {
|
for p in plc.to_slice().iter() {
|
||||||
if !policy_docs_map.contains_key(p) {
|
if !policy_docs_map.contains_key(p) {
|
||||||
let _ = self.api.load_policy_doc(p, &mut policy_docs_map).await;
|
let _ = self.api.load_policy_doc(p, &mut policy_docs_map).await;
|
||||||
@@ -255,7 +256,6 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(plc) = user_policy_map.get(access_key) {
|
if let Some(plc) = user_policy_map.get(access_key) {
|
||||||
for p in plc.to_slice().iter() {
|
for p in plc.to_slice().iter() {
|
||||||
@@ -635,11 +635,11 @@ where
|
|||||||
}
|
}
|
||||||
|
|
||||||
let users = self.cache.users.load();
|
let users = self.cache.users.load();
|
||||||
if let Some(x) = users.get(&cred.access_key) {
|
if let Some(x) = users.get(&cred.access_key)
|
||||||
if x.credentials.is_service_account() {
|
&& x.credentials.is_service_account()
|
||||||
|
{
|
||||||
return Err(Error::IAMActionNotAllowed);
|
return Err(Error::IAMActionNotAllowed);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let u = UserIdentity::new(cred);
|
let u = UserIdentity::new(cred);
|
||||||
|
|
||||||
@@ -789,11 +789,11 @@ where
|
|||||||
|
|
||||||
if !policy_present {
|
if !policy_present {
|
||||||
let mut m = HashMap::new();
|
let mut m = HashMap::new();
|
||||||
if let Err(err) = self.api.load_mapped_policy(name, UserType::Reg, true, &mut m).await {
|
if let Err(err) = self.api.load_mapped_policy(name, UserType::Reg, true, &mut m).await
|
||||||
if !is_err_no_such_policy(&err) {
|
&& !is_err_no_such_policy(&err)
|
||||||
|
{
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if let Some(p) = m.get(name) {
|
if let Some(p) = m.get(name) {
|
||||||
Cache::add_or_update(&self.cache.group_policies, name, p, OffsetDateTime::now_utc());
|
Cache::add_or_update(&self.cache.group_policies, name, p, OffsetDateTime::now_utc());
|
||||||
return Ok((p.to_slice(), p.update_at));
|
return Ok((p.to_slice(), p.update_at));
|
||||||
@@ -815,11 +815,11 @@ where
|
|||||||
Some(p) => p.clone(),
|
Some(p) => p.clone(),
|
||||||
None => {
|
None => {
|
||||||
let mut m = HashMap::new();
|
let mut m = HashMap::new();
|
||||||
if let Err(err) = self.api.load_mapped_policy(name, UserType::Reg, false, &mut m).await {
|
if let Err(err) = self.api.load_mapped_policy(name, UserType::Reg, false, &mut m).await
|
||||||
if !is_err_no_such_policy(&err) {
|
&& !is_err_no_such_policy(&err)
|
||||||
|
{
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if let Some(p) = m.get(name) {
|
if let Some(p) = m.get(name) {
|
||||||
Cache::add_or_update(&self.cache.user_policies, name, p, OffsetDateTime::now_utc());
|
Cache::add_or_update(&self.cache.user_policies, name, p, OffsetDateTime::now_utc());
|
||||||
p.clone()
|
p.clone()
|
||||||
@@ -828,11 +828,11 @@ where
|
|||||||
Some(p) => p.clone(),
|
Some(p) => p.clone(),
|
||||||
None => {
|
None => {
|
||||||
let mut m = HashMap::new();
|
let mut m = HashMap::new();
|
||||||
if let Err(err) = self.api.load_mapped_policy(name, UserType::Sts, false, &mut m).await {
|
if let Err(err) = self.api.load_mapped_policy(name, UserType::Sts, false, &mut m).await
|
||||||
if !is_err_no_such_policy(&err) {
|
&& !is_err_no_such_policy(&err)
|
||||||
|
{
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if let Some(p) = m.get(name) {
|
if let Some(p) = m.get(name) {
|
||||||
Cache::add_or_update(&self.cache.sts_policies, name, p, OffsetDateTime::now_utc());
|
Cache::add_or_update(&self.cache.sts_policies, name, p, OffsetDateTime::now_utc());
|
||||||
p.clone()
|
p.clone()
|
||||||
@@ -864,11 +864,11 @@ where
|
|||||||
Some(p) => p.clone(),
|
Some(p) => p.clone(),
|
||||||
None => {
|
None => {
|
||||||
let mut m = HashMap::new();
|
let mut m = HashMap::new();
|
||||||
if let Err(err) = self.api.load_mapped_policy(group, UserType::Reg, true, &mut m).await {
|
if let Err(err) = self.api.load_mapped_policy(group, UserType::Reg, true, &mut m).await
|
||||||
if !is_err_no_such_policy(&err) {
|
&& !is_err_no_such_policy(&err)
|
||||||
|
{
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if let Some(p) = m.get(group) {
|
if let Some(p) = m.get(group) {
|
||||||
Cache::add_or_update(&self.cache.group_policies, group, p, OffsetDateTime::now_utc());
|
Cache::add_or_update(&self.cache.group_policies, group, p, OffsetDateTime::now_utc());
|
||||||
p.clone()
|
p.clone()
|
||||||
@@ -910,11 +910,11 @@ where
|
|||||||
Some(p) => p.clone(),
|
Some(p) => p.clone(),
|
||||||
None => {
|
None => {
|
||||||
let mut m = HashMap::new();
|
let mut m = HashMap::new();
|
||||||
if let Err(err) = self.api.load_mapped_policy(group, UserType::Reg, true, &mut m).await {
|
if let Err(err) = self.api.load_mapped_policy(group, UserType::Reg, true, &mut m).await
|
||||||
if !is_err_no_such_policy(&err) {
|
&& !is_err_no_such_policy(&err)
|
||||||
|
{
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if let Some(p) = m.get(group) {
|
if let Some(p) = m.get(group) {
|
||||||
Cache::add_or_update(&self.cache.group_policies, group, p, OffsetDateTime::now_utc());
|
Cache::add_or_update(&self.cache.group_policies, group, p, OffsetDateTime::now_utc());
|
||||||
p.clone()
|
p.clone()
|
||||||
@@ -937,11 +937,11 @@ where
|
|||||||
}
|
}
|
||||||
|
|
||||||
if policy.is_empty() {
|
if policy.is_empty() {
|
||||||
if let Err(err) = self.api.delete_mapped_policy(name, user_type, is_group).await {
|
if let Err(err) = self.api.delete_mapped_policy(name, user_type, is_group).await
|
||||||
if !is_err_no_such_policy(&err) {
|
&& !is_err_no_such_policy(&err)
|
||||||
|
{
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if is_group {
|
if is_group {
|
||||||
Cache::delete(&self.cache.group_policies, name, OffsetDateTime::now_utc());
|
Cache::delete(&self.cache.group_policies, name, OffsetDateTime::now_utc());
|
||||||
@@ -1220,11 +1220,11 @@ where
|
|||||||
|
|
||||||
Cache::delete(&self.cache.user_policies, access_key, OffsetDateTime::now_utc());
|
Cache::delete(&self.cache.user_policies, access_key, OffsetDateTime::now_utc());
|
||||||
|
|
||||||
if let Err(err) = self.api.delete_user_identity(access_key, utype).await {
|
if let Err(err) = self.api.delete_user_identity(access_key, utype).await
|
||||||
if !is_err_no_such_user(&err) {
|
&& !is_err_no_such_user(&err)
|
||||||
|
{
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if utype == UserType::Sts {
|
if utype == UserType::Sts {
|
||||||
Cache::delete(&self.cache.sts_accounts, access_key, OffsetDateTime::now_utc());
|
Cache::delete(&self.cache.sts_accounts, access_key, OffsetDateTime::now_utc());
|
||||||
@@ -1532,17 +1532,17 @@ where
|
|||||||
}
|
}
|
||||||
|
|
||||||
if members.is_empty() {
|
if members.is_empty() {
|
||||||
if let Err(err) = self.api.delete_mapped_policy(group, UserType::Reg, true).await {
|
if let Err(err) = self.api.delete_mapped_policy(group, UserType::Reg, true).await
|
||||||
if !is_err_no_such_policy(&err) {
|
&& !is_err_no_such_policy(&err)
|
||||||
|
{
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Err(err) = self.api.delete_group_info(group).await {
|
if let Err(err) = self.api.delete_group_info(group).await
|
||||||
if !is_err_no_such_group(&err) {
|
&& !is_err_no_such_group(&err)
|
||||||
|
{
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Cache::delete(&self.cache.groups, group, OffsetDateTime::now_utc());
|
Cache::delete(&self.cache.groups, group, OffsetDateTime::now_utc());
|
||||||
Cache::delete(&self.cache.group_policies, group, OffsetDateTime::now_utc());
|
Cache::delete(&self.cache.group_policies, group, OffsetDateTime::now_utc());
|
||||||
@@ -1691,13 +1691,13 @@ where
|
|||||||
let member_of = self.cache.user_group_memberships.load();
|
let member_of = self.cache.user_group_memberships.load();
|
||||||
if let Some(m) = member_of.get(name) {
|
if let Some(m) = member_of.get(name) {
|
||||||
for group in m.iter() {
|
for group in m.iter() {
|
||||||
if let Err(err) = self.remove_members_from_group(group, vec![name.to_string()], true).await {
|
if let Err(err) = self.remove_members_from_group(group, vec![name.to_string()], true).await
|
||||||
if !is_err_no_such_group(&err) {
|
&& !is_err_no_such_group(&err)
|
||||||
|
{
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if user_type == UserType::Reg {
|
if user_type == UserType::Reg {
|
||||||
let users_cache = self.cache.users.load();
|
let users_cache = self.cache.users.load();
|
||||||
@@ -1859,13 +1859,13 @@ fn filter_policies(cache: &Cache, policy_name: &str, bucket_name: &str) -> (Stri
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(p) = cache.policy_docs.load().get(&policy) {
|
if let Some(p) = cache.policy_docs.load().get(&policy)
|
||||||
if bucket_name.is_empty() || pollster::block_on(p.policy.match_resource(bucket_name)) {
|
&& (bucket_name.is_empty() || pollster::block_on(p.policy.match_resource(bucket_name)))
|
||||||
|
{
|
||||||
policies.push(policy);
|
policies.push(policy);
|
||||||
to_merge.push(p.policy.clone());
|
to_merge.push(p.policy.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
(policies.join(","), Policy::merge_policies(to_merge))
|
(policies.join(","), Policy::merge_policies(to_merge))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -633,13 +633,13 @@ impl Store for ObjectStore {
|
|||||||
|
|
||||||
if let Some(item) = v.item {
|
if let Some(item) = v.item {
|
||||||
let name = rustfs_utils::path::dir(&item);
|
let name = rustfs_utils::path::dir(&item);
|
||||||
if let Err(err) = self.load_group(&name, m).await {
|
if let Err(err) = self.load_group(&name, m).await
|
||||||
if !is_err_no_such_group(&err) {
|
&& !is_err_no_such_group(&err)
|
||||||
|
{
|
||||||
return Err(err);
|
return Err(err);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
let _ = ctx.cancel();
|
let _ = ctx.cancel();
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -936,10 +936,10 @@ impl Store for ObjectStore {
|
|||||||
let name = item.trim_end_matches(".json");
|
let name = item.trim_end_matches(".json");
|
||||||
|
|
||||||
info!("load group policy: {}", name);
|
info!("load group policy: {}", name);
|
||||||
if let Err(err) = self.load_mapped_policy(name, UserType::Reg, true, &mut items_cache).await {
|
if let Err(err) = self.load_mapped_policy(name, UserType::Reg, true, &mut items_cache).await
|
||||||
if !is_err_no_such_policy(&err) {
|
&& !is_err_no_such_policy(&err)
|
||||||
|
{
|
||||||
return Err(Error::other(format!("load group policy failed: {err}")));
|
return Err(Error::other(format!("load group policy failed: {err}")));
|
||||||
}
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -955,10 +955,10 @@ impl Store for ObjectStore {
|
|||||||
for item in item_name_list.iter() {
|
for item in item_name_list.iter() {
|
||||||
let name = rustfs_utils::path::dir(item);
|
let name = rustfs_utils::path::dir(item);
|
||||||
info!("load svc user: {}", name);
|
info!("load svc user: {}", name);
|
||||||
if let Err(err) = self.load_user(&name, UserType::Svc, &mut items_cache).await {
|
if let Err(err) = self.load_user(&name, UserType::Svc, &mut items_cache).await
|
||||||
if !is_err_no_such_user(&err) {
|
&& !is_err_no_such_user(&err)
|
||||||
|
{
|
||||||
return Err(Error::other(format!("load svc user failed: {err}")));
|
return Err(Error::other(format!("load svc user failed: {err}")));
|
||||||
}
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -969,13 +969,12 @@ impl Store for ObjectStore {
|
|||||||
if let Err(err) = self
|
if let Err(err) = self
|
||||||
.load_mapped_policy(&parent, UserType::Sts, false, &mut sts_policies_cache)
|
.load_mapped_policy(&parent, UserType::Sts, false, &mut sts_policies_cache)
|
||||||
.await
|
.await
|
||||||
|
&& !is_err_no_such_policy(&err)
|
||||||
{
|
{
|
||||||
if !is_err_no_such_policy(&err) {
|
|
||||||
return Err(Error::other(format!("load_mapped_policy failed: {err}")));
|
return Err(Error::other(format!("load_mapped_policy failed: {err}")));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Merge items_cache to user_items_cache
|
// Merge items_cache to user_items_cache
|
||||||
user_items_cache.extend(items_cache);
|
user_items_cache.extend(items_cache);
|
||||||
|
|||||||
+27
-27
@@ -203,8 +203,9 @@ impl<T: Store> IamSys<T> {
|
|||||||
pub async fn set_policy(&self, name: &str, policy: Policy) -> Result<OffsetDateTime> {
|
pub async fn set_policy(&self, name: &str, policy: Policy) -> Result<OffsetDateTime> {
|
||||||
let updated_at = self.store.set_policy(name, policy).await?;
|
let updated_at = self.store.set_policy(name, policy).await?;
|
||||||
|
|
||||||
if !self.has_watcher() {
|
if !self.has_watcher()
|
||||||
if let Some(notification_sys) = get_global_notification_sys() {
|
&& let Some(notification_sys) = get_global_notification_sys()
|
||||||
|
{
|
||||||
let resp = notification_sys.load_policy(name).await;
|
let resp = notification_sys.load_policy(name).await;
|
||||||
for r in resp {
|
for r in resp {
|
||||||
if let Some(err) = r.err {
|
if let Some(err) = r.err {
|
||||||
@@ -212,7 +213,6 @@ impl<T: Store> IamSys<T> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok(updated_at)
|
Ok(updated_at)
|
||||||
}
|
}
|
||||||
@@ -232,8 +232,10 @@ impl<T: Store> IamSys<T> {
|
|||||||
pub async fn delete_user(&self, name: &str, notify: bool) -> Result<()> {
|
pub async fn delete_user(&self, name: &str, notify: bool) -> Result<()> {
|
||||||
self.store.delete_user(name, UserType::Reg).await?;
|
self.store.delete_user(name, UserType::Reg).await?;
|
||||||
|
|
||||||
if notify && !self.has_watcher() {
|
if notify
|
||||||
if let Some(notification_sys) = get_global_notification_sys() {
|
&& !self.has_watcher()
|
||||||
|
&& let Some(notification_sys) = get_global_notification_sys()
|
||||||
|
{
|
||||||
let resp = notification_sys.delete_user(name).await;
|
let resp = notification_sys.delete_user(name).await;
|
||||||
for r in resp {
|
for r in resp {
|
||||||
if let Some(err) = r.err {
|
if let Some(err) = r.err {
|
||||||
@@ -241,7 +243,6 @@ impl<T: Store> IamSys<T> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -476,14 +477,13 @@ impl<T: Store> IamSys<T> {
|
|||||||
|
|
||||||
let op_pt = claims.get(&iam_policy_claim_name_sa());
|
let op_pt = claims.get(&iam_policy_claim_name_sa());
|
||||||
let op_sp = claims.get(SESSION_POLICY_NAME);
|
let op_sp = claims.get(SESSION_POLICY_NAME);
|
||||||
if let (Some(pt), Some(sp)) = (op_pt, op_sp) {
|
if let (Some(pt), Some(sp)) = (op_pt, op_sp)
|
||||||
if pt == EMBEDDED_POLICY_TYPE {
|
&& pt == EMBEDDED_POLICY_TYPE
|
||||||
let policy = serde_json::from_slice(
|
{
|
||||||
&base64_simd::URL_SAFE_NO_PAD.decode_to_vec(sp.as_str().unwrap_or_default().as_bytes())?,
|
let policy =
|
||||||
)?;
|
serde_json::from_slice(&base64_simd::URL_SAFE_NO_PAD.decode_to_vec(sp.as_str().unwrap_or_default().as_bytes())?)?;
|
||||||
return Ok((sa, Some(policy)));
|
return Ok((sa, Some(policy)));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok((sa, None))
|
Ok((sa, None))
|
||||||
}
|
}
|
||||||
@@ -537,14 +537,13 @@ impl<T: Store> IamSys<T> {
|
|||||||
|
|
||||||
let op_pt = claims.get(&iam_policy_claim_name_sa());
|
let op_pt = claims.get(&iam_policy_claim_name_sa());
|
||||||
let op_sp = claims.get(SESSION_POLICY_NAME);
|
let op_sp = claims.get(SESSION_POLICY_NAME);
|
||||||
if let (Some(pt), Some(sp)) = (op_pt, op_sp) {
|
if let (Some(pt), Some(sp)) = (op_pt, op_sp)
|
||||||
if pt == EMBEDDED_POLICY_TYPE {
|
&& pt == EMBEDDED_POLICY_TYPE
|
||||||
let policy = serde_json::from_slice(
|
{
|
||||||
&base64_simd::URL_SAFE_NO_PAD.decode_to_vec(sp.as_str().unwrap_or_default().as_bytes())?,
|
let policy =
|
||||||
)?;
|
serde_json::from_slice(&base64_simd::URL_SAFE_NO_PAD.decode_to_vec(sp.as_str().unwrap_or_default().as_bytes())?)?;
|
||||||
return Ok((sa, Some(policy)));
|
return Ok((sa, Some(policy)));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok((sa, None))
|
Ok((sa, None))
|
||||||
}
|
}
|
||||||
@@ -572,8 +571,10 @@ impl<T: Store> IamSys<T> {
|
|||||||
|
|
||||||
self.store.delete_user(access_key, UserType::Svc).await?;
|
self.store.delete_user(access_key, UserType::Svc).await?;
|
||||||
|
|
||||||
if notify && !self.has_watcher() {
|
if notify
|
||||||
if let Some(notification_sys) = get_global_notification_sys() {
|
&& !self.has_watcher()
|
||||||
|
&& let Some(notification_sys) = get_global_notification_sys()
|
||||||
|
{
|
||||||
let resp = notification_sys.delete_service_account(access_key).await;
|
let resp = notification_sys.delete_service_account(access_key).await;
|
||||||
for r in resp {
|
for r in resp {
|
||||||
if let Some(err) = r.err {
|
if let Some(err) = r.err {
|
||||||
@@ -581,7 +582,6 @@ impl<T: Store> IamSys<T> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -651,11 +651,11 @@ impl<T: Store> IamSys<T> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn check_key(&self, access_key: &str) -> Result<(Option<UserIdentity>, bool)> {
|
pub async fn check_key(&self, access_key: &str) -> Result<(Option<UserIdentity>, bool)> {
|
||||||
if let Some(sys_cred) = get_global_action_cred() {
|
if let Some(sys_cred) = get_global_action_cred()
|
||||||
if sys_cred.access_key == access_key {
|
&& sys_cred.access_key == access_key
|
||||||
|
{
|
||||||
return Ok((Some(UserIdentity::new(sys_cred)), true));
|
return Ok((Some(UserIdentity::new(sys_cred)), true));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
match self.store.get_user(access_key).await {
|
match self.store.get_user(access_key).await {
|
||||||
Some(res) => {
|
Some(res) => {
|
||||||
@@ -725,8 +725,9 @@ impl<T: Store> IamSys<T> {
|
|||||||
pub async fn policy_db_set(&self, name: &str, user_type: UserType, is_group: bool, policy: &str) -> Result<OffsetDateTime> {
|
pub async fn policy_db_set(&self, name: &str, user_type: UserType, is_group: bool, policy: &str) -> Result<OffsetDateTime> {
|
||||||
let updated_at = self.store.policy_db_set(name, user_type, is_group, policy).await?;
|
let updated_at = self.store.policy_db_set(name, user_type, is_group, policy).await?;
|
||||||
|
|
||||||
if !self.has_watcher() {
|
if !self.has_watcher()
|
||||||
if let Some(notification_sys) = get_global_notification_sys() {
|
&& let Some(notification_sys) = get_global_notification_sys()
|
||||||
|
{
|
||||||
let resp = notification_sys.load_policy_mapping(name, user_type.to_u64(), is_group).await;
|
let resp = notification_sys.load_policy_mapping(name, user_type.to_u64(), is_group).await;
|
||||||
for r in resp {
|
for r in resp {
|
||||||
if let Some(err) = r.err {
|
if let Some(err) = r.err {
|
||||||
@@ -734,7 +735,6 @@ impl<T: Store> IamSys<T> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok(updated_at)
|
Ok(updated_at)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -452,29 +452,27 @@ impl KmsClient for LocalKmsClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let path = entry.path();
|
let path = entry.path();
|
||||||
if path.extension().is_some_and(|ext| ext == "key") {
|
if path.extension().is_some_and(|ext| ext == "key")
|
||||||
if let Some(stem) = path.file_stem() {
|
&& let Some(stem) = path.file_stem()
|
||||||
if let Some(key_id) = stem.to_str() {
|
&& let Some(key_id) = stem.to_str()
|
||||||
if let Ok(key_info) = self.describe_key(key_id, None).await {
|
&& let Ok(key_info) = self.describe_key(key_id, None).await
|
||||||
|
{
|
||||||
// Apply filters
|
// Apply filters
|
||||||
if let Some(ref status_filter) = request.status_filter {
|
if let Some(ref status_filter) = request.status_filter
|
||||||
if &key_info.status != status_filter {
|
&& &key_info.status != status_filter
|
||||||
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
if let Some(ref usage_filter) = request.usage_filter
|
||||||
if let Some(ref usage_filter) = request.usage_filter {
|
&& &key_info.usage != usage_filter
|
||||||
if &key_info.usage != usage_filter {
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
keys.push(key_info);
|
keys.push(key_info);
|
||||||
count += 1;
|
count += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(ListKeysResponse {
|
Ok(ListKeysResponse {
|
||||||
keys,
|
keys,
|
||||||
|
|||||||
@@ -279,9 +279,10 @@ impl KmsConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Validate TLS configuration if using HTTPS
|
// Validate TLS configuration if using HTTPS
|
||||||
if config.address.starts_with("https://") {
|
if config.address.starts_with("https://")
|
||||||
if let Some(ref tls) = config.tls {
|
&& let Some(ref tls) = config.tls
|
||||||
if !tls.skip_verify {
|
&& !tls.skip_verify
|
||||||
|
{
|
||||||
// In production, we should have proper TLS configuration
|
// In production, we should have proper TLS configuration
|
||||||
if tls.ca_cert_path.is_none() && tls.client_cert_path.is_none() {
|
if tls.ca_cert_path.is_none() && tls.client_cert_path.is_none() {
|
||||||
tracing::warn!("Using HTTPS without custom TLS configuration - relying on system CA");
|
tracing::warn!("Using HTTPS without custom TLS configuration - relying on system CA");
|
||||||
@@ -289,8 +290,6 @@ impl KmsConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate cache configuration
|
// Validate cache configuration
|
||||||
if self.enable_cache && self.cache_config.max_keys == 0 {
|
if self.enable_cache && self.cache_config.max_keys == 0 {
|
||||||
|
|||||||
@@ -74,8 +74,9 @@ impl KmsManager {
|
|||||||
// Check cache first if enabled
|
// Check cache first if enabled
|
||||||
if self.config.enable_cache {
|
if self.config.enable_cache {
|
||||||
let cache = self.cache.read().await;
|
let cache = self.cache.read().await;
|
||||||
if let Some(cached_key) = cache.get_data_key(&request.key_id).await {
|
if let Some(cached_key) = cache.get_data_key(&request.key_id).await
|
||||||
if cached_key.key_spec == request.key_spec {
|
&& cached_key.key_spec == request.key_spec
|
||||||
|
{
|
||||||
return Ok(GenerateDataKeyResponse {
|
return Ok(GenerateDataKeyResponse {
|
||||||
key_id: request.key_id.clone(),
|
key_id: request.key_id.clone(),
|
||||||
plaintext_key: cached_key.plaintext.clone(),
|
plaintext_key: cached_key.plaintext.clone(),
|
||||||
@@ -83,7 +84,6 @@ impl KmsManager {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Generate new data key from backend
|
// Generate new data key from backend
|
||||||
let response = self.backend.generate_data_key(request).await?;
|
let response = self.backend.generate_data_key(request).await?;
|
||||||
|
|||||||
@@ -104,8 +104,9 @@ mod tests {
|
|||||||
let response = client.acquire_exclusive(&request).await;
|
let response = client.acquire_exclusive(&request).await;
|
||||||
assert!(response.is_ok());
|
assert!(response.is_ok());
|
||||||
|
|
||||||
if let Ok(response) = response {
|
if let Ok(response) = response
|
||||||
if response.success {
|
&& response.success
|
||||||
|
{
|
||||||
let lock_info = response.lock_info.unwrap();
|
let lock_info = response.lock_info.unwrap();
|
||||||
|
|
||||||
// Test status check
|
// Test status check
|
||||||
@@ -120,4 +121,3 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -953,12 +953,12 @@ mod tests {
|
|||||||
// Wait for all operations to complete
|
// Wait for all operations to complete
|
||||||
let mut successful_operations = 0;
|
let mut successful_operations = 0;
|
||||||
for handle in handles {
|
for handle in handles {
|
||||||
if let Ok(success) = handle.await {
|
if let Ok(success) = handle.await
|
||||||
if success {
|
&& success
|
||||||
|
{
|
||||||
successful_operations += 1;
|
successful_operations += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// We expect most operations to succeed with the new timeouts and optimizations
|
// We expect most operations to succeed with the new timeouts and optimizations
|
||||||
let total_operations = concurrent_queries + 5;
|
let total_operations = concurrent_queries + 5;
|
||||||
|
|||||||
@@ -414,13 +414,13 @@ impl Default for FastObjectLockManager {
|
|||||||
impl Drop for FastObjectLockManager {
|
impl Drop for FastObjectLockManager {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
// Note: We can't use async in Drop, so we just abort the cleanup task
|
// Note: We can't use async in Drop, so we just abort the cleanup task
|
||||||
if let Ok(handle_guard) = self.cleanup_handle.try_read() {
|
if let Ok(handle_guard) = self.cleanup_handle.try_read()
|
||||||
if let Some(handle) = handle_guard.as_ref() {
|
&& let Some(handle) = handle_guard.as_ref()
|
||||||
|
{
|
||||||
handle.abort();
|
handle.abort();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
impl Clone for FastObjectLockManager {
|
impl Clone for FastObjectLockManager {
|
||||||
fn clone(&self) -> Self {
|
fn clone(&self) -> Self {
|
||||||
|
|||||||
@@ -69,11 +69,11 @@ impl LockShard {
|
|||||||
/// Try fast path only (without fallback to slow path)
|
/// Try fast path only (without fallback to slow path)
|
||||||
pub fn try_fast_path_only(&self, request: &ObjectLockRequest) -> bool {
|
pub fn try_fast_path_only(&self, request: &ObjectLockRequest) -> bool {
|
||||||
// Early check to avoid unnecessary lock contention
|
// Early check to avoid unnecessary lock contention
|
||||||
if let Some(state) = self.objects.read().get(&request.key) {
|
if let Some(state) = self.objects.read().get(&request.key)
|
||||||
if !state.atomic_state.is_fast_path_available(request.mode) {
|
&& !state.atomic_state.is_fast_path_available(request.mode)
|
||||||
|
{
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
self.try_fast_path(request).is_some()
|
self.try_fast_path(request).is_some()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -441,8 +441,9 @@ impl LockShard {
|
|||||||
/// Get lock information for monitoring
|
/// Get lock information for monitoring
|
||||||
pub fn get_lock_info(&self, key: &ObjectKey) -> Option<crate::fast_lock::types::ObjectLockInfo> {
|
pub fn get_lock_info(&self, key: &ObjectKey) -> Option<crate::fast_lock::types::ObjectLockInfo> {
|
||||||
let objects = self.objects.read();
|
let objects = self.objects.read();
|
||||||
if let Some(state) = objects.get(key) {
|
if let Some(state) = objects.get(key)
|
||||||
if let Some(mode) = state.current_mode() {
|
&& let Some(mode) = state.current_mode()
|
||||||
|
{
|
||||||
let (owner, acquired_at, lock_timeout) = match mode {
|
let (owner, acquired_at, lock_timeout) = match mode {
|
||||||
LockMode::Exclusive => {
|
LockMode::Exclusive => {
|
||||||
let current_owner = state.current_owner.read();
|
let current_owner = state.current_owner.read();
|
||||||
@@ -471,7 +472,6 @@ impl LockShard {
|
|||||||
priority,
|
priority,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -165,12 +165,12 @@ impl NamespaceLock {
|
|||||||
let mut successful_clients = Vec::new();
|
let mut successful_clients = Vec::new();
|
||||||
|
|
||||||
for (idx, res) in results {
|
for (idx, res) in results {
|
||||||
if let Ok(resp) = res {
|
if let Ok(resp) = res
|
||||||
if resp.success {
|
&& resp.success
|
||||||
|
{
|
||||||
successful_clients.push(idx);
|
successful_clients.push(idx);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if successful_clients.len() >= self.quorum {
|
if successful_clients.len() >= self.quorum {
|
||||||
let resp = LockResponse::success(
|
let resp = LockResponse::success(
|
||||||
|
|||||||
@@ -204,12 +204,12 @@ impl TargetFactory for MQTTTargetFactory {
|
|||||||
if !std::path::Path::new(&queue_dir).is_absolute() {
|
if !std::path::Path::new(&queue_dir).is_absolute() {
|
||||||
return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string()));
|
return Err(TargetError::Configuration("MQTT queue directory must be an absolute path".to_string()));
|
||||||
}
|
}
|
||||||
if let Some(qos_str) = config.lookup(MQTT_QOS) {
|
if let Some(qos_str) = config.lookup(MQTT_QOS)
|
||||||
if qos_str == "0" {
|
&& qos_str == "0"
|
||||||
|
{
|
||||||
warn!("Using queue_dir with QoS 0 may result in event loss");
|
warn!("Using queue_dir with QoS 0 may result in event loss");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -345,12 +345,12 @@ impl TargetList {
|
|||||||
pub async fn clear_targets_only(&mut self) {
|
pub async fn clear_targets_only(&mut self) {
|
||||||
let target_ids_to_clear: Vec<TargetID> = self.targets.keys().cloned().collect();
|
let target_ids_to_clear: Vec<TargetID> = self.targets.keys().cloned().collect();
|
||||||
for id in target_ids_to_clear {
|
for id in target_ids_to_clear {
|
||||||
if let Some(target_arc) = self.targets.remove(&id) {
|
if let Some(target_arc) = self.targets.remove(&id)
|
||||||
if let Err(e) = target_arc.close().await {
|
&& let Err(e) = target_arc.close().await
|
||||||
|
{
|
||||||
error!("Failed to close target {} during clear: {}", id, e);
|
error!("Failed to close target {} during clear: {}", id, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
self.targets.clear();
|
self.targets.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -119,14 +119,13 @@ impl TargetRegistry {
|
|||||||
format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}")
|
format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}")
|
||||||
.to_uppercase();
|
.to_uppercase();
|
||||||
for (key, value) in &all_env {
|
for (key, value) in &all_env {
|
||||||
if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false) {
|
if EnableState::from_str(value).ok().map(|s| s.is_enabled()).unwrap_or(false)
|
||||||
if let Some(id) = key.strip_prefix(&enable_prefix) {
|
&& let Some(id) = key.strip_prefix(&enable_prefix)
|
||||||
if !id.is_empty() {
|
&& !id.is_empty()
|
||||||
|
{
|
||||||
instance_ids_from_env.insert(id.to_lowercase());
|
instance_ids_from_env.insert(id.to_lowercase());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3.2. Parse all relevant environment variable configurations
|
// 3.2. Parse all relevant environment variable configurations
|
||||||
// 3.2.1. Build environment variable prefixes such as 'RUSTFS_NOTIFY_WEBHOOK_'
|
// 3.2.1. Build environment variable prefixes such as 'RUSTFS_NOTIFY_WEBHOOK_'
|
||||||
@@ -273,11 +272,11 @@ impl TargetRegistry {
|
|||||||
for section in sections {
|
for section in sections {
|
||||||
let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
|
let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
|
||||||
// Add default item
|
// Add default item
|
||||||
if let Some(default_kvs) = section_defaults.get(§ion) {
|
if let Some(default_kvs) = section_defaults.get(§ion)
|
||||||
if !default_kvs.is_empty() {
|
&& !default_kvs.is_empty()
|
||||||
|
{
|
||||||
section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
|
section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Add successful instance item
|
// Add successful instance item
|
||||||
if let Some(instances) = successes_by_section.get(§ion) {
|
if let Some(instances) = successes_by_section.get(§ion) {
|
||||||
|
|||||||
@@ -24,18 +24,19 @@ pub fn new_pattern(prefix: Option<&str>, suffix: Option<&str>) -> String {
|
|||||||
let mut pattern = String::new();
|
let mut pattern = String::new();
|
||||||
|
|
||||||
// Process the prefix part
|
// Process the prefix part
|
||||||
if let Some(p) = prefix {
|
if let Some(p) = prefix
|
||||||
if !p.is_empty() {
|
&& !p.is_empty()
|
||||||
|
{
|
||||||
pattern.push_str(p);
|
pattern.push_str(p);
|
||||||
if !p.ends_with('*') {
|
if !p.ends_with('*') {
|
||||||
pattern.push('*');
|
pattern.push('*');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Process the suffix part
|
// Process the suffix part
|
||||||
if let Some(s) = suffix {
|
if let Some(s) = suffix
|
||||||
if !s.is_empty() {
|
&& !s.is_empty()
|
||||||
|
{
|
||||||
let mut s_to_append = s.to_string();
|
let mut s_to_append = s.to_string();
|
||||||
if !s.starts_with('*') {
|
if !s.starts_with('*') {
|
||||||
s_to_append.insert(0, '*');
|
s_to_append.insert(0, '*');
|
||||||
@@ -49,7 +50,6 @@ pub fn new_pattern(prefix: Option<&str>, suffix: Option<&str>) -> String {
|
|||||||
pattern.push_str(&s_to_append);
|
pattern.push_str(&s_to_append);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Replace "**" with "*"
|
// Replace "**" with "*"
|
||||||
pattern = pattern.replace("**", "*");
|
pattern = pattern.replace("**", "*");
|
||||||
|
|||||||
@@ -86,22 +86,22 @@ impl std::fmt::Debug for OtelGuard {
|
|||||||
|
|
||||||
impl Drop for OtelGuard {
|
impl Drop for OtelGuard {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
if let Some(provider) = self.tracer_provider.take() {
|
if let Some(provider) = self.tracer_provider.take()
|
||||||
if let Err(err) = provider.shutdown() {
|
&& let Err(err) = provider.shutdown()
|
||||||
|
{
|
||||||
eprintln!("Tracer shutdown error: {err:?}");
|
eprintln!("Tracer shutdown error: {err:?}");
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(provider) = self.meter_provider.take() {
|
if let Some(provider) = self.meter_provider.take()
|
||||||
if let Err(err) = provider.shutdown() {
|
&& let Err(err) = provider.shutdown()
|
||||||
|
{
|
||||||
eprintln!("Meter shutdown error: {err:?}");
|
eprintln!("Meter shutdown error: {err:?}");
|
||||||
}
|
}
|
||||||
}
|
if let Some(provider) = self.logger_provider.take()
|
||||||
if let Some(provider) = self.logger_provider.take() {
|
&& let Err(err) = provider.shutdown()
|
||||||
if let Err(err) = provider.shutdown() {
|
{
|
||||||
eprintln!("Logger shutdown error: {err:?}");
|
eprintln!("Logger shutdown error: {err:?}");
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(handle) = self.flexi_logger_handles.take() {
|
if let Some(handle) = self.flexi_logger_handles.take() {
|
||||||
handle.shutdown();
|
handle.shutdown();
|
||||||
|
|||||||
@@ -122,13 +122,13 @@ impl Resource {
|
|||||||
// Apply condition substitutions
|
// Apply condition substitutions
|
||||||
if !conditions.is_empty() {
|
if !conditions.is_empty() {
|
||||||
for key in KeyName::COMMON_KEYS {
|
for key in KeyName::COMMON_KEYS {
|
||||||
if let Some(rvalue) = conditions.get(key.name()) {
|
if let Some(rvalue) = conditions.get(key.name())
|
||||||
if matches!(rvalue.first().map(|c| !c.is_empty()), Some(true)) {
|
&& matches!(rvalue.first().map(|c| !c.is_empty()), Some(true))
|
||||||
|
{
|
||||||
resolved_pattern = resolved_pattern.replace(&key.var_name(), &rvalue[0]);
|
resolved_pattern = resolved_pattern.replace(&key.var_name(), &rvalue[0]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let cp = path::clean(resource);
|
let cp = path::clean(resource);
|
||||||
if cp != "." && cp == resolved_pattern.as_str() {
|
if cp != "." && cp == resolved_pattern.as_str() {
|
||||||
|
|||||||
+13
-13
@@ -387,7 +387,7 @@ impl Checksum {
|
|||||||
|
|
||||||
// Ensure we don't divide by 0
|
// Ensure we don't divide by 0
|
||||||
let raw_len = self.checksum_type.raw_byte_len();
|
let raw_len = self.checksum_type.raw_byte_len();
|
||||||
if raw_len == 0 || parts.len() % raw_len != 0 {
|
if raw_len == 0 || !parts.len().is_multiple_of(raw_len) {
|
||||||
checksums = 0;
|
checksums = 0;
|
||||||
} else if !parts.is_empty() {
|
} else if !parts.is_empty() {
|
||||||
checksums = (parts.len() / raw_len) as i32;
|
checksums = (parts.len() / raw_len) as i32;
|
||||||
@@ -506,8 +506,9 @@ pub fn get_content_checksum(headers: &HeaderMap) -> Result<Option<Checksum>, std
|
|||||||
for header in trailing_headers {
|
for header in trailing_headers {
|
||||||
let mut duplicates = false;
|
let mut duplicates = false;
|
||||||
for &checksum_type in crate::checksum::BASE_CHECKSUM_TYPES {
|
for &checksum_type in crate::checksum::BASE_CHECKSUM_TYPES {
|
||||||
if let Some(key) = checksum_type.key() {
|
if let Some(key) = checksum_type.key()
|
||||||
if header.eq_ignore_ascii_case(key) {
|
&& header.eq_ignore_ascii_case(key)
|
||||||
|
{
|
||||||
duplicates = result.is_some();
|
duplicates = result.is_some();
|
||||||
result = Some(Checksum {
|
result = Some(Checksum {
|
||||||
checksum_type: ChecksumType(checksum_type.0 | ChecksumType::TRAILING.0),
|
checksum_type: ChecksumType(checksum_type.0 | ChecksumType::TRAILING.0),
|
||||||
@@ -517,7 +518,6 @@ pub fn get_content_checksum(headers: &HeaderMap) -> Result<Option<Checksum>, std
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if duplicates {
|
if duplicates {
|
||||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid checksum"));
|
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid checksum"));
|
||||||
}
|
}
|
||||||
@@ -567,22 +567,23 @@ fn get_content_checksum_direct(headers: &HeaderMap) -> (ChecksumType, String) {
|
|||||||
checksum_type = ChecksumType(checksum_type.0 | ChecksumType::FULL_OBJECT.0);
|
checksum_type = ChecksumType(checksum_type.0 | ChecksumType::FULL_OBJECT.0);
|
||||||
}
|
}
|
||||||
|
|
||||||
if checksum_type.is_set() {
|
if checksum_type.is_set()
|
||||||
if let Some(key) = checksum_type.key() {
|
&& let Some(key) = checksum_type.key()
|
||||||
|
{
|
||||||
if let Some(value) = headers.get(key).and_then(|v| v.to_str().ok()) {
|
if let Some(value) = headers.get(key).and_then(|v| v.to_str().ok()) {
|
||||||
return (checksum_type, value.to_string());
|
return (checksum_type, value.to_string());
|
||||||
} else {
|
} else {
|
||||||
return (ChecksumType::NONE, String::new());
|
return (ChecksumType::NONE, String::new());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
return (checksum_type, String::new());
|
return (checksum_type, String::new());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check individual checksum headers
|
// Check individual checksum headers
|
||||||
for &ct in crate::checksum::BASE_CHECKSUM_TYPES {
|
for &ct in crate::checksum::BASE_CHECKSUM_TYPES {
|
||||||
if let Some(key) = ct.key() {
|
if let Some(key) = ct.key()
|
||||||
if let Some(value) = headers.get(key).and_then(|v| v.to_str().ok()) {
|
&& let Some(value) = headers.get(key).and_then(|v| v.to_str().ok())
|
||||||
|
{
|
||||||
// If already set, invalid
|
// If already set, invalid
|
||||||
if checksum_type != ChecksumType::NONE {
|
if checksum_type != ChecksumType::NONE {
|
||||||
return (ChecksumType::INVALID, String::new());
|
return (ChecksumType::INVALID, String::new());
|
||||||
@@ -598,7 +599,6 @@ fn get_content_checksum_direct(headers: &HeaderMap) -> (ChecksumType, String) {
|
|||||||
return (checksum_type, value.to_string());
|
return (checksum_type, value.to_string());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
(checksum_type, String::new())
|
(checksum_type, String::new())
|
||||||
}
|
}
|
||||||
@@ -965,11 +965,11 @@ fn gf2_matrix_times(mat: &[u64], mut vec: u64) -> u64 {
|
|||||||
let mut mat_iter = mat.iter();
|
let mut mat_iter = mat.iter();
|
||||||
|
|
||||||
while vec != 0 {
|
while vec != 0 {
|
||||||
if vec & 1 != 0 {
|
if vec & 1 != 0
|
||||||
if let Some(&m) = mat_iter.next() {
|
&& let Some(&m) = mat_iter.next()
|
||||||
|
{
|
||||||
sum ^= m;
|
sum ^= m;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
vec >>= 1;
|
vec >>= 1;
|
||||||
mat_iter.next();
|
mat_iter.next();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -178,13 +178,12 @@ impl HashReader {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(checksum) = existing_hash_reader.checksum() {
|
if let Some(checksum) = existing_hash_reader.checksum()
|
||||||
if let Some(ref md5) = md5hex {
|
&& let Some(ref md5) = md5hex
|
||||||
if checksum != md5 {
|
&& checksum != md5
|
||||||
|
{
|
||||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "HashReader checksum mismatch"));
|
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "HashReader checksum mismatch"));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if existing_hash_reader.size() > 0 && size > 0 && existing_hash_reader.size() != size {
|
if existing_hash_reader.size() > 0 && size > 0 && existing_hash_reader.size() != size {
|
||||||
return Err(std::io::Error::new(
|
return Err(std::io::Error::new(
|
||||||
@@ -359,16 +358,16 @@ impl HashReader {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if checksum.checksum_type.trailing() {
|
if checksum.checksum_type.trailing() {
|
||||||
if let Some(trailer) = self.trailer_s3s.as_ref() {
|
if let Some(trailer) = self.trailer_s3s.as_ref()
|
||||||
if let Some(Some(checksum_str)) = trailer.read(|headers| {
|
&& let Some(Some(checksum_str)) = trailer.read(|headers| {
|
||||||
checksum
|
checksum
|
||||||
.checksum_type
|
.checksum_type
|
||||||
.key()
|
.key()
|
||||||
.and_then(|key| headers.get(key).and_then(|value| value.to_str().ok().map(|s| s.to_string())))
|
.and_then(|key| headers.get(key).and_then(|value| value.to_str().ok().map(|s| s.to_string())))
|
||||||
}) {
|
})
|
||||||
|
{
|
||||||
map.insert(checksum.checksum_type.to_string(), checksum_str);
|
map.insert(checksum.checksum_type.to_string(), checksum_str);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
return map;
|
return map;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -450,20 +449,20 @@ impl AsyncRead for HashReader {
|
|||||||
|
|
||||||
if filled > 0 {
|
if filled > 0 {
|
||||||
// Update SHA256 hasher
|
// Update SHA256 hasher
|
||||||
if let Some(hasher) = this.content_sha256_hasher {
|
if let Some(hasher) = this.content_sha256_hasher
|
||||||
if let Err(e) = hasher.write_all(data) {
|
&& let Err(e) = hasher.write_all(data)
|
||||||
|
{
|
||||||
error!("SHA256 hasher write error, error={:?}", e);
|
error!("SHA256 hasher write error, error={:?}", e);
|
||||||
return Poll::Ready(Err(std::io::Error::other(e)));
|
return Poll::Ready(Err(std::io::Error::other(e)));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Update content hasher
|
// Update content hasher
|
||||||
if let Some(hasher) = this.content_hasher {
|
if let Some(hasher) = this.content_hasher
|
||||||
if let Err(e) = hasher.write_all(data) {
|
&& let Err(e) = hasher.write_all(data)
|
||||||
|
{
|
||||||
return Poll::Ready(Err(std::io::Error::other(e)));
|
return Poll::Ready(Err(std::io::Error::other(e)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if filled == 0 && !*this.checksum_on_finish {
|
if filled == 0 && !*this.checksum_on_finish {
|
||||||
// check SHA256
|
// check SHA256
|
||||||
@@ -477,13 +476,15 @@ impl AsyncRead for HashReader {
|
|||||||
|
|
||||||
// check content hasher
|
// check content hasher
|
||||||
if let (Some(hasher), Some(expected_content_hash)) = (this.content_hasher, this.content_hash) {
|
if let (Some(hasher), Some(expected_content_hash)) = (this.content_hasher, this.content_hash) {
|
||||||
if expected_content_hash.checksum_type.trailing() {
|
if expected_content_hash.checksum_type.trailing()
|
||||||
if let Some(trailer) = this.trailer_s3s.as_ref() {
|
&& let Some(trailer) = this.trailer_s3s.as_ref()
|
||||||
if let Some(Some(checksum_str)) = trailer.read(|headers| {
|
&& let Some(Some(checksum_str)) = trailer.read(|headers| {
|
||||||
expected_content_hash.checksum_type.key().and_then(|key| {
|
expected_content_hash
|
||||||
headers.get(key).and_then(|value| value.to_str().ok().map(|s| s.to_string()))
|
.checksum_type
|
||||||
|
.key()
|
||||||
|
.and_then(|key| headers.get(key).and_then(|value| value.to_str().ok().map(|s| s.to_string())))
|
||||||
})
|
})
|
||||||
}) {
|
{
|
||||||
expected_content_hash.encoded = checksum_str;
|
expected_content_hash.encoded = checksum_str;
|
||||||
expected_content_hash.raw = general_purpose::STANDARD
|
expected_content_hash.raw = general_purpose::STANDARD
|
||||||
.decode(&expected_content_hash.encoded)
|
.decode(&expected_content_hash.encoded)
|
||||||
@@ -493,8 +494,6 @@ impl AsyncRead for HashReader {
|
|||||||
return Poll::Ready(Err(std::io::Error::other("Content hash mismatch")));
|
return Poll::Ready(Err(std::io::Error::other("Content hash mismatch")));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let content_hash = hasher.finalize();
|
let content_hash = hasher.finalize();
|
||||||
|
|
||||||
|
|||||||
@@ -175,11 +175,11 @@ impl SimpleQueryDispatcher {
|
|||||||
.clone()
|
.clone()
|
||||||
.map(|e| e.as_bytes().first().copied().unwrap_or_default()),
|
.map(|e| e.as_bytes().first().copied().unwrap_or_default()),
|
||||||
);
|
);
|
||||||
if let Some(delimiter) = csv.field_delimiter.as_ref() {
|
if let Some(delimiter) = csv.field_delimiter.as_ref()
|
||||||
if delimiter.len() == 1 {
|
&& delimiter.len() == 1
|
||||||
|
{
|
||||||
file_format = file_format.with_delimiter(delimiter.as_bytes()[0]);
|
file_format = file_format.with_delimiter(delimiter.as_bytes()[0]);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
// TODO waiting for processing @junxiang Mu
|
// TODO waiting for processing @junxiang Mu
|
||||||
// if csv.file_header_info.is_some() {}
|
// if csv.file_header_info.is_some() {}
|
||||||
match csv.file_header_info.as_ref() {
|
match csv.file_header_info.as_ref() {
|
||||||
|
|||||||
@@ -25,11 +25,11 @@ pub fn get_host_addr(req: &request::Request<Body>) -> String {
|
|||||||
} else {
|
} else {
|
||||||
req_host = uri.host().unwrap().to_string();
|
req_host = uri.host().unwrap().to_string();
|
||||||
}
|
}
|
||||||
if let Some(host) = host {
|
if let Some(host) = host
|
||||||
if req_host != *host.to_str().unwrap() {
|
&& req_host != *host.to_str().unwrap()
|
||||||
|
{
|
||||||
return (*host.to_str().unwrap()).to_string();
|
return (*host.to_str().unwrap()).to_string();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
/*if req.uri_ref().unwrap().host().is_some() {
|
/*if req.uri_ref().unwrap().host().is_some() {
|
||||||
return req.uri_ref().unwrap().host().unwrap();
|
return req.uri_ref().unwrap().host().unwrap();
|
||||||
}*/
|
}*/
|
||||||
|
|||||||
@@ -97,12 +97,12 @@ pub fn parse_key(s: &str) -> Key {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Number of batch items parsed
|
// Number of batch items parsed
|
||||||
if let Some(colon_pos) = name.find(':') {
|
if let Some(colon_pos) = name.find(':')
|
||||||
if let Ok(count) = name[..colon_pos].parse::<usize>() {
|
&& let Ok(count) = name[..colon_pos].parse::<usize>()
|
||||||
|
{
|
||||||
item_count = count;
|
item_count = count;
|
||||||
name = name[colon_pos + 1..].to_string();
|
name = name[colon_pos + 1..].to_string();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Resolve extension
|
// Resolve extension
|
||||||
if let Some(dot_pos) = name.rfind('.') {
|
if let Some(dot_pos) = name.rfind('.') {
|
||||||
|
|||||||
@@ -225,17 +225,20 @@ where
|
|||||||
|
|
||||||
match tokio::time::timeout(DEFAULT_CONNECTION_TIMEOUT, async {
|
match tokio::time::timeout(DEFAULT_CONNECTION_TIMEOUT, async {
|
||||||
while !self.connected.load(Ordering::SeqCst) {
|
while !self.connected.load(Ordering::SeqCst) {
|
||||||
if let Some(handle) = self.bg_task_manager.init_cell.get() {
|
if let Some(handle) = self.bg_task_manager.init_cell.get()
|
||||||
if handle.is_finished() && !self.connected.load(Ordering::SeqCst) {
|
&& handle.is_finished()
|
||||||
|
&& !self.connected.load(Ordering::SeqCst)
|
||||||
|
{
|
||||||
error!(target_id = %self.id, "MQTT background task exited prematurely before connection was established.");
|
error!(target_id = %self.id, "MQTT background task exited prematurely before connection was established.");
|
||||||
return Err(TargetError::Network("MQTT background task exited prematurely".to_string()));
|
return Err(TargetError::Network("MQTT background task exited prematurely".to_string()));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||||
}
|
}
|
||||||
debug!(target_id = %self.id, "MQTT target connected successfully.");
|
debug!(target_id = %self.id, "MQTT target connected successfully.");
|
||||||
Ok(())
|
Ok(())
|
||||||
}).await {
|
})
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(Ok(_)) => {
|
Ok(Ok(_)) => {
|
||||||
info!(target_id = %self.id, "MQTT target initialized and connected.");
|
info!(target_id = %self.id, "MQTT target initialized and connected.");
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -243,9 +246,7 @@ where
|
|||||||
Ok(Err(e)) => Err(e),
|
Ok(Err(e)) => Err(e),
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
error!(target_id = %self.id, "Timeout waiting for MQTT connection after task spawn.");
|
error!(target_id = %self.id, "Timeout waiting for MQTT connection after task spawn.");
|
||||||
Err(TargetError::Network(
|
Err(TargetError::Network("Timeout waiting for MQTT connection".to_string()))
|
||||||
"Timeout waiting for MQTT connection".to_string(),
|
|
||||||
))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -470,12 +471,12 @@ where
|
|||||||
debug!(target_id = %self.id, "Checking if MQTT target is active.");
|
debug!(target_id = %self.id, "Checking if MQTT target is active.");
|
||||||
if self.client.lock().await.is_none() && !self.connected.load(Ordering::SeqCst) {
|
if self.client.lock().await.is_none() && !self.connected.load(Ordering::SeqCst) {
|
||||||
// Check if the background task is running and has not panicked
|
// Check if the background task is running and has not panicked
|
||||||
if let Some(handle) = self.bg_task_manager.init_cell.get() {
|
if let Some(handle) = self.bg_task_manager.init_cell.get()
|
||||||
if handle.is_finished() {
|
&& handle.is_finished()
|
||||||
|
{
|
||||||
error!(target_id = %self.id, "MQTT background task has finished, possibly due to an error. Target is not active.");
|
error!(target_id = %self.id, "MQTT background task has finished, possibly due to an error. Target is not active.");
|
||||||
return Err(TargetError::Network("MQTT background task terminated".to_string()));
|
return Err(TargetError::Network("MQTT background task terminated".to_string()));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
debug!(target_id = %self.id, "MQTT client not yet initialized or task not running/connected.");
|
debug!(target_id = %self.id, "MQTT client not yet initialized or task not running/connected.");
|
||||||
return Err(TargetError::Configuration(
|
return Err(TargetError::Configuration(
|
||||||
"MQTT client not available or not initialized/connected".to_string(),
|
"MQTT client not available or not initialized/connected".to_string(),
|
||||||
|
|||||||
@@ -292,7 +292,7 @@ pub fn create_multi_cert_resolver(
|
|||||||
|
|
||||||
for (domain, (certs, key)) in cert_key_pairs {
|
for (domain, (certs, key)) in cert_key_pairs {
|
||||||
// create a signature
|
// create a signature
|
||||||
let signing_key = rustls::crypto::ring::sign::any_supported_type(&key)
|
let signing_key = rustls::crypto::aws_lc_rs::sign::any_supported_type(&key)
|
||||||
.map_err(|e| certs_error(format!("unsupported private key types:{domain}, err:{e:?}")))?;
|
.map_err(|e| certs_error(format!("unsupported private key types:{domain}, err:{e:?}")))?;
|
||||||
|
|
||||||
// create a CertifiedKey
|
// create a CertifiedKey
|
||||||
|
|||||||
+25
-26
@@ -56,20 +56,21 @@ fn is_xff_header_enabled() -> bool {
|
|||||||
///
|
///
|
||||||
pub fn get_source_scheme(headers: &HeaderMap) -> Option<String> {
|
pub fn get_source_scheme(headers: &HeaderMap) -> Option<String> {
|
||||||
// Retrieve the scheme from X-Forwarded-Proto.
|
// Retrieve the scheme from X-Forwarded-Proto.
|
||||||
if let Some(proto) = headers.get(X_FORWARDED_PROTO) {
|
if let Some(proto) = headers.get(X_FORWARDED_PROTO)
|
||||||
if let Ok(proto_str) = proto.to_str() {
|
&& let Ok(proto_str) = proto.to_str()
|
||||||
|
{
|
||||||
return Some(proto_str.to_lowercase());
|
return Some(proto_str.to_lowercase());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(proto) = headers.get(X_FORWARDED_SCHEME) {
|
if let Some(proto) = headers.get(X_FORWARDED_SCHEME)
|
||||||
if let Ok(proto_str) = proto.to_str() {
|
&& let Ok(proto_str) = proto.to_str()
|
||||||
|
{
|
||||||
return Some(proto_str.to_lowercase());
|
return Some(proto_str.to_lowercase());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(forwarded) = headers.get(FORWARDED) {
|
if let Some(forwarded) = headers.get(FORWARDED)
|
||||||
if let Ok(forwarded_str) = forwarded.to_str() {
|
&& let Ok(forwarded_str) = forwarded.to_str()
|
||||||
|
{
|
||||||
// match should contain at least two elements if the protocol was
|
// match should contain at least two elements if the protocol was
|
||||||
// specified in the Forwarded header. The first element will always be
|
// specified in the Forwarded header. The first element will always be
|
||||||
// the 'for=', which we ignore, subsequently we proceed to look for
|
// the 'for=', which we ignore, subsequently we proceed to look for
|
||||||
@@ -77,18 +78,17 @@ pub fn get_source_scheme(headers: &HeaderMap) -> Option<String> {
|
|||||||
// we simply ignore the values and return empty. This is in line
|
// we simply ignore the values and return empty. This is in line
|
||||||
// with the approach we took for returning first ip from multiple
|
// with the approach we took for returning first ip from multiple
|
||||||
// params.
|
// params.
|
||||||
if let Some(for_match) = FOR_REGEX.captures(forwarded_str) {
|
if let Some(for_match) = FOR_REGEX.captures(forwarded_str)
|
||||||
if for_match.len() > 1 {
|
&& for_match.len() > 1
|
||||||
|
{
|
||||||
let remaining = &for_match[2];
|
let remaining = &for_match[2];
|
||||||
if let Some(proto_match) = PROTO_REGEX.captures(remaining) {
|
if let Some(proto_match) = PROTO_REGEX.captures(remaining)
|
||||||
if proto_match.len() > 1 {
|
&& proto_match.len() > 1
|
||||||
|
{
|
||||||
return Some(proto_match[2].to_lowercase());
|
return Some(proto_match[2].to_lowercase());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
@@ -105,9 +105,10 @@ pub fn get_source_scheme(headers: &HeaderMap) -> Option<String> {
|
|||||||
pub fn get_source_ip_from_headers(headers: &HeaderMap) -> Option<String> {
|
pub fn get_source_ip_from_headers(headers: &HeaderMap) -> Option<String> {
|
||||||
let mut addr = None;
|
let mut addr = None;
|
||||||
|
|
||||||
if is_xff_header_enabled() {
|
if is_xff_header_enabled()
|
||||||
if let Some(forwarded_for) = headers.get(X_FORWARDED_FOR) {
|
&& let Some(forwarded_for) = headers.get(X_FORWARDED_FOR)
|
||||||
if let Ok(forwarded_str) = forwarded_for.to_str() {
|
&& let Ok(forwarded_str) = forwarded_for.to_str()
|
||||||
|
{
|
||||||
// Only grab the first (client) address. Note that '192.168.0.1,
|
// Only grab the first (client) address. Note that '192.168.0.1,
|
||||||
// 10.1.1.1' is a valid key for X-Forwarded-For where addresses after
|
// 10.1.1.1' is a valid key for X-Forwarded-For where addresses after
|
||||||
// the first may represent forwarding proxies earlier in the chain.
|
// the first may represent forwarding proxies earlier in the chain.
|
||||||
@@ -115,8 +116,6 @@ pub fn get_source_ip_from_headers(headers: &HeaderMap) -> Option<String> {
|
|||||||
let end = first_comma.unwrap_or(forwarded_str.len());
|
let end = first_comma.unwrap_or(forwarded_str.len());
|
||||||
addr = Some(forwarded_str[..end].to_string());
|
addr = Some(forwarded_str[..end].to_string());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if addr.is_none() {
|
if addr.is_none() {
|
||||||
if let Some(real_ip) = headers.get(X_REAL_IP) {
|
if let Some(real_ip) = headers.get(X_REAL_IP) {
|
||||||
@@ -125,15 +124,17 @@ pub fn get_source_ip_from_headers(headers: &HeaderMap) -> Option<String> {
|
|||||||
// request).
|
// request).
|
||||||
addr = Some(real_ip_str.to_string());
|
addr = Some(real_ip_str.to_string());
|
||||||
}
|
}
|
||||||
} else if let Some(forwarded) = headers.get(FORWARDED) {
|
} else if let Some(forwarded) = headers.get(FORWARDED)
|
||||||
if let Ok(forwarded_str) = forwarded.to_str() {
|
&& let Ok(forwarded_str) = forwarded.to_str()
|
||||||
|
{
|
||||||
// match should contain at least two elements if the protocol was
|
// match should contain at least two elements if the protocol was
|
||||||
// specified in the Forwarded header. The first element will always be
|
// specified in the Forwarded header. The first element will always be
|
||||||
// the 'for=' capture, which we ignore. In the case of multiple IP
|
// the 'for=' capture, which we ignore. In the case of multiple IP
|
||||||
// addresses (for=8.8.8.8, 8.8.4.4, 172.16.1.20 is valid) we only
|
// addresses (for=8.8.8.8, 8.8.4.4, 172.16.1.20 is valid) we only
|
||||||
// extract the first, which should be the client IP.
|
// extract the first, which should be the client IP.
|
||||||
if let Some(for_match) = FOR_REGEX.captures(forwarded_str) {
|
if let Some(for_match) = FOR_REGEX.captures(forwarded_str)
|
||||||
if for_match.len() > 1 {
|
&& for_match.len() > 1
|
||||||
|
{
|
||||||
// IPv6 addresses in Forwarded headers are quoted-strings. We strip
|
// IPv6 addresses in Forwarded headers are quoted-strings. We strip
|
||||||
// these quotes.
|
// these quotes.
|
||||||
let ip = for_match[1].trim_matches('"');
|
let ip = for_match[1].trim_matches('"');
|
||||||
@@ -141,8 +142,6 @@ pub fn get_source_ip_from_headers(headers: &HeaderMap) -> Option<String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
addr
|
addr
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -174,17 +174,16 @@ pub async fn get_host_ip(host: Host<&str>) -> std::io::Result<HashSet<IpAddr>> {
|
|||||||
match host {
|
match host {
|
||||||
Host::Domain(domain) => {
|
Host::Domain(domain) => {
|
||||||
// Check cache first
|
// Check cache first
|
||||||
if CUSTOM_DNS_RESOLVER.read().unwrap().is_none() {
|
if CUSTOM_DNS_RESOLVER.read().unwrap().is_none()
|
||||||
if let Ok(mut cache) = DNS_CACHE.lock() {
|
&& let Ok(mut cache) = DNS_CACHE.lock()
|
||||||
if let Some(entry) = cache.get(domain) {
|
&& let Some(entry) = cache.get(domain)
|
||||||
|
{
|
||||||
if !entry.is_expired(DNS_CACHE_TTL) {
|
if !entry.is_expired(DNS_CACHE_TTL) {
|
||||||
return Ok(entry.ips.clone());
|
return Ok(entry.ips.clone());
|
||||||
}
|
}
|
||||||
// Remove expired entry
|
// Remove expired entry
|
||||||
cache.remove(domain);
|
cache.remove(domain);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
info!("Cache miss for domain {domain}, querying system resolver.");
|
info!("Cache miss for domain {domain}, querying system resolver.");
|
||||||
|
|
||||||
|
|||||||
@@ -196,14 +196,13 @@ impl ParsedURL {
|
|||||||
impl std::fmt::Display for ParsedURL {
|
impl std::fmt::Display for ParsedURL {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
let mut url = self.0.clone();
|
let mut url = self.0.clone();
|
||||||
if let Some(host) = url.host_str().map(|h| h.to_string()) {
|
if let Some(host) = url.host_str().map(|h| h.to_string())
|
||||||
if let Some(port) = url.port() {
|
&& let Some(port) = url.port()
|
||||||
if (url.scheme() == "http" && port == 80) || (url.scheme() == "https" && port == 443) {
|
&& ((url.scheme() == "http" && port == 80) || (url.scheme() == "https" && port == 443))
|
||||||
|
{
|
||||||
url.set_host(Some(&host)).unwrap();
|
url.set_host(Some(&host)).unwrap();
|
||||||
url.set_port(None).unwrap();
|
url.set_port(None).unwrap();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
let mut s = url.to_string();
|
let mut s = url.to_string();
|
||||||
|
|
||||||
// If the URL ends with a slash and the path is just "/", remove the trailing slash.
|
// If the URL ends with a slash and the path is just "/", remove the trailing slash.
|
||||||
@@ -251,14 +250,14 @@ impl<'de> serde::Deserialize<'de> for ParsedURL {
|
|||||||
/// Returns NetError if parsing fails or host is invalid.
|
/// Returns NetError if parsing fails or host is invalid.
|
||||||
///
|
///
|
||||||
pub fn parse_url(s: &str) -> Result<ParsedURL, NetError> {
|
pub fn parse_url(s: &str) -> Result<ParsedURL, NetError> {
|
||||||
if let Some(scheme_end) = s.find("://") {
|
if let Some(scheme_end) = s.find("://")
|
||||||
if s[scheme_end + 3..].starts_with('/') {
|
&& s[scheme_end + 3..].starts_with('/')
|
||||||
|
{
|
||||||
let scheme = &s[..scheme_end];
|
let scheme = &s[..scheme_end];
|
||||||
if !scheme.is_empty() {
|
if !scheme.is_empty() {
|
||||||
return Err(NetError::SchemeWithEmptyHost);
|
return Err(NetError::SchemeWithEmptyHost);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let mut uu = Url::parse(s).map_err(|e| NetError::ParseError(e.to_string()))?;
|
let mut uu = Url::parse(s).map_err(|e| NetError::ParseError(e.to_string()))?;
|
||||||
if uu.host_str().is_none_or(|h| h.is_empty()) {
|
if uu.host_str().is_none_or(|h| h.is_empty()) {
|
||||||
|
|||||||
@@ -367,11 +367,12 @@ async fn _setup_console_tls_config(tls_path: Option<&String>) -> Result<Option<R
|
|||||||
debug!("Found TLS directory for console, checking for certificates");
|
debug!("Found TLS directory for console, checking for certificates");
|
||||||
|
|
||||||
// Make sure to use a modern encryption suite
|
// Make sure to use a modern encryption suite
|
||||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||||
|
|
||||||
// 1. Attempt to load all certificates in the directory (multi-certificate support, for SNI)
|
// 1. Attempt to load all certificates in the directory (multi-certificate support, for SNI)
|
||||||
if let Ok(cert_key_pairs) = rustfs_utils::load_all_certs_from_directory(tls_path) {
|
if let Ok(cert_key_pairs) = rustfs_utils::load_all_certs_from_directory(tls_path)
|
||||||
if !cert_key_pairs.is_empty() {
|
&& !cert_key_pairs.is_empty()
|
||||||
|
{
|
||||||
debug!(
|
debug!(
|
||||||
"Found {} certificates for console, creating SNI-aware multi-cert resolver",
|
"Found {} certificates for console, creating SNI-aware multi-cert resolver",
|
||||||
cert_key_pairs.len()
|
cert_key_pairs.len()
|
||||||
@@ -396,7 +397,6 @@ async fn _setup_console_tls_config(tls_path: Option<&String>) -> Result<Option<R
|
|||||||
info!(target: "rustfs::console::tls", "Console TLS enabled with multi-certificate SNI support");
|
info!(target: "rustfs::console::tls", "Console TLS enabled with multi-certificate SNI support");
|
||||||
return Ok(Some(RustlsConfig::from_config(Arc::new(server_config))));
|
return Ok(Some(RustlsConfig::from_config(Arc::new(server_config))));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Revert to the traditional single-certificate mode
|
// 2. Revert to the traditional single-certificate mode
|
||||||
let key_path = format!("{tls_path}/{RUSTFS_TLS_KEY}");
|
let key_path = format!("{tls_path}/{RUSTFS_TLS_KEY}");
|
||||||
|
|||||||
@@ -636,54 +636,54 @@ fn extract_metrics_init_params(uri: &Uri) -> MetricsParams {
|
|||||||
for param in params {
|
for param in params {
|
||||||
let mut parts = param.split('=');
|
let mut parts = param.split('=');
|
||||||
if let Some(key) = parts.next() {
|
if let Some(key) = parts.next() {
|
||||||
if key == "disks" {
|
if key == "disks"
|
||||||
if let Some(value) = parts.next() {
|
&& let Some(value) = parts.next()
|
||||||
|
{
|
||||||
mp.disks = value.to_string();
|
mp.disks = value.to_string();
|
||||||
}
|
}
|
||||||
}
|
if key == "hosts"
|
||||||
if key == "hosts" {
|
&& let Some(value) = parts.next()
|
||||||
if let Some(value) = parts.next() {
|
{
|
||||||
mp.hosts = value.to_string();
|
mp.hosts = value.to_string();
|
||||||
}
|
}
|
||||||
}
|
if key == "interval"
|
||||||
if key == "interval" {
|
&& let Some(value) = parts.next()
|
||||||
if let Some(value) = parts.next() {
|
{
|
||||||
mp.tick = value.to_string();
|
mp.tick = value.to_string();
|
||||||
}
|
}
|
||||||
}
|
if key == "n"
|
||||||
if key == "n" {
|
&& let Some(value) = parts.next()
|
||||||
if let Some(value) = parts.next() {
|
{
|
||||||
mp.n = value.parse::<u64>().unwrap_or(u64::MAX);
|
mp.n = value.parse::<u64>().unwrap_or(u64::MAX);
|
||||||
}
|
}
|
||||||
}
|
if key == "types"
|
||||||
if key == "types" {
|
&& let Some(value) = parts.next()
|
||||||
if let Some(value) = parts.next() {
|
{
|
||||||
mp.types = value.parse::<u32>().unwrap_or_default();
|
mp.types = value.parse::<u32>().unwrap_or_default();
|
||||||
}
|
}
|
||||||
}
|
if key == "by-disk"
|
||||||
if key == "by-disk" {
|
&& let Some(value) = parts.next()
|
||||||
if let Some(value) = parts.next() {
|
{
|
||||||
mp.by_disk = value.to_string();
|
mp.by_disk = value.to_string();
|
||||||
}
|
}
|
||||||
}
|
if key == "by-host"
|
||||||
if key == "by-host" {
|
&& let Some(value) = parts.next()
|
||||||
if let Some(value) = parts.next() {
|
{
|
||||||
mp.by_host = value.to_string();
|
mp.by_host = value.to_string();
|
||||||
}
|
}
|
||||||
}
|
if key == "by-jobID"
|
||||||
if key == "by-jobID" {
|
&& let Some(value) = parts.next()
|
||||||
if let Some(value) = parts.next() {
|
{
|
||||||
mp.by_job_id = value.to_string();
|
mp.by_job_id = value.to_string();
|
||||||
}
|
}
|
||||||
}
|
if key == "by-depID"
|
||||||
if key == "by-depID" {
|
&& let Some(value) = parts.next()
|
||||||
if let Some(value) = parts.next() {
|
{
|
||||||
mp.by_dep_id = value.to_string();
|
mp.by_dep_id = value.to_string();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
mp
|
mp
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -830,11 +830,11 @@ fn extract_heal_init_params(body: &Bytes, uri: &Uri, params: Params<'_, '_>) ->
|
|||||||
for param in params {
|
for param in params {
|
||||||
let mut parts = param.split('=');
|
let mut parts = param.split('=');
|
||||||
if let Some(key) = parts.next() {
|
if let Some(key) = parts.next() {
|
||||||
if key == "clientToken" {
|
if key == "clientToken"
|
||||||
if let Some(value) = parts.next() {
|
&& let Some(value) = parts.next()
|
||||||
|
{
|
||||||
hip.client_token = value.to_string();
|
hip.client_token = value.to_string();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if key == "forceStart" && parts.next().is_some() {
|
if key == "forceStart" && parts.next().is_some() {
|
||||||
hip.force_start = true;
|
hip.force_start = true;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -277,11 +277,12 @@ impl Operation for UpdateGroupMembers {
|
|||||||
} else {
|
} else {
|
||||||
warn!("add group members");
|
warn!("add group members");
|
||||||
|
|
||||||
if let Err(err) = iam_store.get_group_description(&args.group).await {
|
if let Err(err) = iam_store.get_group_description(&args.group).await
|
||||||
if is_err_no_such_group(&err) && has_space_be(&args.group) {
|
&& is_err_no_such_group(&err)
|
||||||
|
&& has_space_be(&args.group)
|
||||||
|
{
|
||||||
return Err(s3_error!(InvalidArgument, "not such group"));
|
return Err(s3_error!(InvalidArgument, "not such group"));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
iam_store.add_users_to_group(&args.group, args.members).await.map_err(|e| {
|
iam_store.add_users_to_group(&args.group, args.members).await.map_err(|e| {
|
||||||
warn!("add group members failed, e: {:?}", e);
|
warn!("add group members failed, e: {:?}", e);
|
||||||
|
|||||||
@@ -96,11 +96,11 @@ impl Operation for AddUser {
|
|||||||
return Err(s3_error!(InvalidArgument, "access key is empty"));
|
return Err(s3_error!(InvalidArgument, "access key is empty"));
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(sys_cred) = get_global_action_cred() {
|
if let Some(sys_cred) = get_global_action_cred()
|
||||||
if constant_time_eq(&sys_cred.access_key, ak) {
|
&& constant_time_eq(&sys_cred.access_key, ak)
|
||||||
|
{
|
||||||
return Err(s3_error!(InvalidArgument, "can't create user with system access key"));
|
return Err(s3_error!(InvalidArgument, "can't create user with system access key"));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let Ok(iam_store) = rustfs_iam::get() else {
|
let Ok(iam_store) = rustfs_iam::get() else {
|
||||||
return Err(s3_error!(InvalidRequest, "iam not init"));
|
return Err(s3_error!(InvalidRequest, "iam not init"));
|
||||||
@@ -777,11 +777,11 @@ impl Operation for ImportIam {
|
|||||||
let groups: HashMap<String, GroupInfo> = serde_json::from_slice(&file_content)
|
let groups: HashMap<String, GroupInfo> = serde_json::from_slice(&file_content)
|
||||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?;
|
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?;
|
||||||
for (group_name, group_info) in groups {
|
for (group_name, group_info) in groups {
|
||||||
if let Err(e) = iam_store.get_group_description(&group_name).await {
|
if let Err(e) = iam_store.get_group_description(&group_name).await
|
||||||
if matches!(e, rustfs_iam::error::Error::NoSuchGroup(_)) || has_space_be(&group_name) {
|
&& (matches!(e, rustfs_iam::error::Error::NoSuchGroup(_)) || has_space_be(&group_name))
|
||||||
|
{
|
||||||
return Err(s3_error!(InvalidArgument, "group not found or has space be"));
|
return Err(s3_error!(InvalidArgument, "group not found or has space be"));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Err(e) = iam_store.add_users_to_group(&group_name, group_info.members.clone()).await {
|
if let Err(e) = iam_store.add_users_to_group(&group_name, group_info.members.clone()).await {
|
||||||
failed.groups.push(IAMErrEntity {
|
failed.groups.push(IAMErrEntity {
|
||||||
|
|||||||
+39
-39
@@ -175,11 +175,11 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
|
|||||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("check claims failed1 {e}")))?;
|
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("check claims failed1 {e}")))?;
|
||||||
|
|
||||||
if !ok {
|
if !ok {
|
||||||
if let Some(u) = u {
|
if let Some(u) = u
|
||||||
if u.credentials.status == "off" {
|
&& u.credentials.status == "off"
|
||||||
|
{
|
||||||
return Err(s3_error!(InvalidRequest, "ErrAccessKeyDisabled"));
|
return Err(s3_error!(InvalidRequest, "ErrAccessKeyDisabled"));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return Err(s3_error!(InvalidRequest, "ErrAccessKeyDisabled"));
|
return Err(s3_error!(InvalidRequest, "ErrAccessKeyDisabled"));
|
||||||
}
|
}
|
||||||
@@ -200,11 +200,11 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result<
|
|||||||
constant_time_eq(&sys_cred.access_key, &cred.access_key) || constant_time_eq(&cred.parent_user, &sys_cred.access_key);
|
constant_time_eq(&sys_cred.access_key, &cred.access_key) || constant_time_eq(&cred.parent_user, &sys_cred.access_key);
|
||||||
|
|
||||||
// permitRootAccess
|
// permitRootAccess
|
||||||
if let Some(claims) = &cred.claims {
|
if let Some(claims) = &cred.claims
|
||||||
if claims.contains_key(SESSION_POLICY_NAME) {
|
&& claims.contains_key(SESSION_POLICY_NAME)
|
||||||
|
{
|
||||||
owner = false
|
owner = false
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok((cred, owner))
|
Ok((cred, owner))
|
||||||
}
|
}
|
||||||
@@ -358,11 +358,11 @@ pub fn get_condition_values(
|
|||||||
args.insert("authType".to_owned(), vec![auth_type]);
|
args.insert("authType".to_owned(), vec![auth_type]);
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(lc) = region {
|
if let Some(lc) = region
|
||||||
if !lc.is_empty() {
|
&& !lc.is_empty()
|
||||||
|
{
|
||||||
args.insert("LocationConstraint".to_owned(), vec![lc.to_string()]);
|
args.insert("LocationConstraint".to_owned(), vec![lc.to_string()]);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let mut clone_header = header.clone();
|
let mut clone_header = header.clone();
|
||||||
if let Some(v) = clone_header.get("x-amz-signature-age") {
|
if let Some(v) = clone_header.get("x-amz-signature-age") {
|
||||||
@@ -411,8 +411,9 @@ pub fn get_condition_values(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(grps_val) = claims.get("groups") {
|
if let Some(grps_val) = claims.get("groups")
|
||||||
if let Some(grps_is) = grps_val.as_array() {
|
&& let Some(grps_is) = grps_val.as_array()
|
||||||
|
{
|
||||||
let grps = grps_is
|
let grps = grps_is
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|g| g.as_str().map(|s| s.to_string()))
|
.filter_map(|g| g.as_str().map(|s| s.to_string()))
|
||||||
@@ -422,13 +423,12 @@ pub fn get_condition_values(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(groups) = &cred.groups {
|
if let Some(groups) = &cred.groups
|
||||||
if !args.contains_key("groups") {
|
&& !args.contains_key("groups")
|
||||||
|
{
|
||||||
args.insert("groups".to_string(), groups.clone());
|
args.insert("groups".to_string(), groups.clone());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
args
|
args
|
||||||
}
|
}
|
||||||
@@ -502,11 +502,11 @@ fn determine_auth_type_and_version(header: &HeaderMap) -> (String, String) {
|
|||||||
/// # Returns
|
/// # Returns
|
||||||
/// * `bool` - True if request has JWT, false otherwise
|
/// * `bool` - True if request has JWT, false otherwise
|
||||||
fn is_request_jwt(header: &HeaderMap) -> bool {
|
fn is_request_jwt(header: &HeaderMap) -> bool {
|
||||||
if let Some(auth) = header.get("authorization") {
|
if let Some(auth) = header.get("authorization")
|
||||||
if let Ok(auth_str) = auth.to_str() {
|
&& let Ok(auth_str) = auth.to_str()
|
||||||
|
{
|
||||||
return auth_str.starts_with(JWT_ALGORITHM);
|
return auth_str.starts_with(JWT_ALGORITHM);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -518,11 +518,11 @@ fn is_request_jwt(header: &HeaderMap) -> bool {
|
|||||||
/// # Returns
|
/// # Returns
|
||||||
/// * `bool` - True if request has AWS Signature Version '4', false otherwise
|
/// * `bool` - True if request has AWS Signature Version '4', false otherwise
|
||||||
fn is_request_signature_v4(header: &HeaderMap) -> bool {
|
fn is_request_signature_v4(header: &HeaderMap) -> bool {
|
||||||
if let Some(auth) = header.get("authorization") {
|
if let Some(auth) = header.get("authorization")
|
||||||
if let Ok(auth_str) = auth.to_str() {
|
&& let Ok(auth_str) = auth.to_str()
|
||||||
|
{
|
||||||
return auth_str.starts_with(SIGN_V4_ALGORITHM);
|
return auth_str.starts_with(SIGN_V4_ALGORITHM);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -534,11 +534,11 @@ fn is_request_signature_v4(header: &HeaderMap) -> bool {
|
|||||||
/// # Returns
|
/// # Returns
|
||||||
/// * `bool` - True if request has AWS Signature Version '2', false otherwise
|
/// * `bool` - True if request has AWS Signature Version '2', false otherwise
|
||||||
fn is_request_signature_v2(header: &HeaderMap) -> bool {
|
fn is_request_signature_v2(header: &HeaderMap) -> bool {
|
||||||
if let Some(auth) = header.get("authorization") {
|
if let Some(auth) = header.get("authorization")
|
||||||
if let Ok(auth_str) = auth.to_str() {
|
&& let Ok(auth_str) = auth.to_str()
|
||||||
|
{
|
||||||
return !auth_str.starts_with(SIGN_V4_ALGORITHM) && auth_str.starts_with(SIGN_V2_ALGORITHM);
|
return !auth_str.starts_with(SIGN_V4_ALGORITHM) && auth_str.starts_with(SIGN_V2_ALGORITHM);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -578,41 +578,41 @@ fn is_request_presigned_signature_v2(header: &HeaderMap) -> bool {
|
|||||||
/// # Returns
|
/// # Returns
|
||||||
/// * `bool` - True if request has AWS Post policy Signature Version '4', false otherwise
|
/// * `bool` - True if request has AWS Post policy Signature Version '4', false otherwise
|
||||||
fn is_request_post_policy_signature_v4(header: &HeaderMap) -> bool {
|
fn is_request_post_policy_signature_v4(header: &HeaderMap) -> bool {
|
||||||
if let Some(content_type) = header.get("content-type") {
|
if let Some(content_type) = header.get("content-type")
|
||||||
if let Ok(ct) = content_type.to_str() {
|
&& let Ok(ct) = content_type.to_str()
|
||||||
|
{
|
||||||
return ct.contains("multipart/form-data");
|
return ct.contains("multipart/form-data");
|
||||||
}
|
}
|
||||||
}
|
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Verify if the request has AWS Streaming Signature Version '4'
|
/// Verify if the request has AWS Streaming Signature Version '4'
|
||||||
fn is_request_sign_streaming_v4(header: &HeaderMap) -> bool {
|
fn is_request_sign_streaming_v4(header: &HeaderMap) -> bool {
|
||||||
if let Some(content_sha256) = header.get("x-amz-content-sha256") {
|
if let Some(content_sha256) = header.get("x-amz-content-sha256")
|
||||||
if let Ok(sha256_str) = content_sha256.to_str() {
|
&& let Ok(sha256_str) = content_sha256.to_str()
|
||||||
|
{
|
||||||
return sha256_str == STREAMING_CONTENT_SHA256;
|
return sha256_str == STREAMING_CONTENT_SHA256;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify if the request has AWS Streaming Signature Version '4' with trailer
|
// Verify if the request has AWS Streaming Signature Version '4' with trailer
|
||||||
fn is_request_sign_streaming_trailer_v4(header: &HeaderMap) -> bool {
|
fn is_request_sign_streaming_trailer_v4(header: &HeaderMap) -> bool {
|
||||||
if let Some(content_sha256) = header.get("x-amz-content-sha256") {
|
if let Some(content_sha256) = header.get("x-amz-content-sha256")
|
||||||
if let Ok(sha256_str) = content_sha256.to_str() {
|
&& let Ok(sha256_str) = content_sha256.to_str()
|
||||||
|
{
|
||||||
return sha256_str == STREAMING_CONTENT_SHA256_TRAILER;
|
return sha256_str == STREAMING_CONTENT_SHA256_TRAILER;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify if the request has AWS Streaming Signature Version '4' with unsigned content and trailer
|
// Verify if the request has AWS Streaming Signature Version '4' with unsigned content and trailer
|
||||||
fn is_request_unsigned_trailer_v4(header: &HeaderMap) -> bool {
|
fn is_request_unsigned_trailer_v4(header: &HeaderMap) -> bool {
|
||||||
if let Some(content_sha256) = header.get("x-amz-content-sha256") {
|
if let Some(content_sha256) = header.get("x-amz-content-sha256")
|
||||||
if let Ok(sha256_str) = content_sha256.to_str() {
|
&& let Ok(sha256_str) = content_sha256.to_str()
|
||||||
|
{
|
||||||
return sha256_str == UNSIGNED_PAYLOAD_TRAILER;
|
return sha256_str == UNSIGNED_PAYLOAD_TRAILER;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -634,12 +634,12 @@ pub fn get_query_param<'a>(query: &'a str, param_name: &str) -> Option<&'a str>
|
|||||||
|
|
||||||
for pair in query.split('&') {
|
for pair in query.split('&') {
|
||||||
let mut parts = pair.splitn(2, '=');
|
let mut parts = pair.splitn(2, '=');
|
||||||
if let (Some(key), Some(value)) = (parts.next(), parts.next()) {
|
if let (Some(key), Some(value)) = (parts.next(), parts.next())
|
||||||
if key.to_lowercase() == param_name {
|
&& key.to_lowercase() == param_name
|
||||||
|
{
|
||||||
return Some(value);
|
return Some(value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+4
-6
@@ -193,10 +193,10 @@ impl From<ApiError> for S3Error {
|
|||||||
impl From<StorageError> for ApiError {
|
impl From<StorageError> for ApiError {
|
||||||
fn from(err: StorageError) -> Self {
|
fn from(err: StorageError) -> Self {
|
||||||
// Special handling for Io errors that may contain ChecksumMismatch
|
// Special handling for Io errors that may contain ChecksumMismatch
|
||||||
if let StorageError::Io(ref io_err) = err {
|
if let StorageError::Io(ref io_err) = err
|
||||||
if let Some(inner) = io_err.get_ref() {
|
&& let Some(inner) = io_err.get_ref()
|
||||||
if inner.downcast_ref::<rustfs_rio::ChecksumMismatch>().is_some()
|
&& (inner.downcast_ref::<rustfs_rio::ChecksumMismatch>().is_some()
|
||||||
|| inner.downcast_ref::<rustfs_rio::BadDigest>().is_some()
|
|| inner.downcast_ref::<rustfs_rio::BadDigest>().is_some())
|
||||||
{
|
{
|
||||||
return ApiError {
|
return ApiError {
|
||||||
code: S3ErrorCode::BadDigest,
|
code: S3ErrorCode::BadDigest,
|
||||||
@@ -204,8 +204,6 @@ impl From<StorageError> for ApiError {
|
|||||||
source: Some(Box::new(err)),
|
source: Some(Box::new(err)),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let code = match &err {
|
let code = match &err {
|
||||||
StorageError::NotImplemented => S3ErrorCode::NotImplemented,
|
StorageError::NotImplemented => S3ErrorCode::NotImplemented,
|
||||||
|
|||||||
@@ -735,8 +735,9 @@ impl StorageBackend<super::server::FtpsUser> for FtpsDriver {
|
|||||||
|
|
||||||
match s3_client.list_objects_v2(list_input).await {
|
match s3_client.list_objects_v2(list_input).await {
|
||||||
Ok(output) => {
|
Ok(output) => {
|
||||||
if let Some(objects) = output.contents {
|
if let Some(objects) = output.contents
|
||||||
if !objects.is_empty() {
|
&& !objects.is_empty()
|
||||||
|
{
|
||||||
debug!("FTPS RMD - bucket '{}' is not empty, cannot delete", bucket);
|
debug!("FTPS RMD - bucket '{}' is not empty, cannot delete", bucket);
|
||||||
return Err(Error::new(
|
return Err(Error::new(
|
||||||
ErrorKind::PermanentFileNotAvailable,
|
ErrorKind::PermanentFileNotAvailable,
|
||||||
@@ -744,7 +745,6 @@ impl StorageBackend<super::server::FtpsUser> for FtpsDriver {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
debug!("FTPS RMD - failed to list objects: {}", e);
|
debug!("FTPS RMD - failed to list objects: {}", e);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -98,17 +98,17 @@ impl FtpsConfig {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(path) = &self.cert_file {
|
if let Some(path) = &self.cert_file
|
||||||
if !tokio::fs::try_exists(path).await.unwrap_or(false) {
|
&& !tokio::fs::try_exists(path).await.unwrap_or(false)
|
||||||
|
{
|
||||||
return Err(FtpsInitError::InvalidConfig(format!("Certificate file not found: {}", path)));
|
return Err(FtpsInitError::InvalidConfig(format!("Certificate file not found: {}", path)));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(path) = &self.key_file {
|
if let Some(path) = &self.key_file
|
||||||
if !tokio::fs::try_exists(path).await.unwrap_or(false) {
|
&& !tokio::fs::try_exists(path).await.unwrap_or(false)
|
||||||
|
{
|
||||||
return Err(FtpsInitError::InvalidConfig(format!("Key file not found: {}", path)));
|
return Err(FtpsInitError::InvalidConfig(format!("Key file not found: {}", path)));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Validate passive ports format
|
// Validate passive ports format
|
||||||
if self.passive_ports.is_some() {
|
if self.passive_ports.is_some() {
|
||||||
|
|||||||
@@ -753,8 +753,9 @@ impl Handler for SftpHandler {
|
|||||||
|
|
||||||
match s3_client.list_objects_v2(list_input).await {
|
match s3_client.list_objects_v2(list_input).await {
|
||||||
Ok(output) => {
|
Ok(output) => {
|
||||||
if let Some(objects) = output.contents {
|
if let Some(objects) = output.contents
|
||||||
if !objects.is_empty() {
|
&& !objects.is_empty()
|
||||||
|
{
|
||||||
debug!("SFTP REMOVE - bucket '{}' is not empty, cannot delete", bucket);
|
debug!("SFTP REMOVE - bucket '{}' is not empty, cannot delete", bucket);
|
||||||
return Ok(Status {
|
return Ok(Status {
|
||||||
id,
|
id,
|
||||||
@@ -764,7 +765,6 @@ impl Handler for SftpHandler {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
debug!("SFTP REMOVE - failed to list objects: {}", e);
|
debug!("SFTP REMOVE - failed to list objects: {}", e);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -696,11 +696,11 @@ fn compare_keys(stored_key: &str, client_key_base64: &str) -> bool {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Ok(stored_bytes) = BASE64.decode(stored_key_data) {
|
if let Ok(stored_bytes) = BASE64.decode(stored_key_data)
|
||||||
if let Ok(client_bytes) = BASE64.decode(client_key_base64) {
|
&& let Ok(client_bytes) = BASE64.decode(client_key_base64)
|
||||||
|
{
|
||||||
return stored_bytes == client_bytes;
|
return stored_bytes == client_bytes;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -260,14 +260,14 @@ async fn walk_dir(path: PathBuf, cert_name: &str, cert_data: &mut Vec<u8>) {
|
|||||||
// Only check direct subdirectories, no deeper recursion
|
// Only check direct subdirectories, no deeper recursion
|
||||||
if let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await {
|
if let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await {
|
||||||
while let Ok(Some(sub_entry)) = sub_rd.next_entry().await {
|
while let Ok(Some(sub_entry)) = sub_rd.next_entry().await {
|
||||||
if let Ok(sub_ft) = sub_entry.file_type().await {
|
if let Ok(sub_ft) = sub_entry.file_type().await
|
||||||
if sub_ft.is_file() {
|
&& sub_ft.is_file()
|
||||||
|
{
|
||||||
load_if_matches(&sub_entry, cert_name, cert_data).await;
|
load_if_matches(&sub_entry, cert_name, cert_data).await;
|
||||||
}
|
}
|
||||||
// Ignore subdirectories and symlinks in subdirs to limit to one level
|
// Ignore subdirectories and symlinks in subdirs to limit to one level
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
} else if ft.is_symlink() {
|
} else if ft.is_symlink() {
|
||||||
// Follow symlink and treat target as file or directory, but limit to one level
|
// Follow symlink and treat target as file or directory, but limit to one level
|
||||||
if let Ok(meta) = tokio::fs::metadata(&entry.path()).await {
|
if let Ok(meta) = tokio::fs::metadata(&entry.path()).await {
|
||||||
@@ -277,8 +277,9 @@ async fn walk_dir(path: PathBuf, cert_name: &str, cert_data: &mut Vec<u8>) {
|
|||||||
// Treat as directory but only check its direct contents
|
// Treat as directory but only check its direct contents
|
||||||
if let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await {
|
if let Ok(mut sub_rd) = tokio::fs::read_dir(&entry.path()).await {
|
||||||
while let Ok(Some(sub_entry)) = sub_rd.next_entry().await {
|
while let Ok(Some(sub_entry)) = sub_rd.next_entry().await {
|
||||||
if let Ok(sub_ft) = sub_entry.file_type().await {
|
if let Ok(sub_ft) = sub_entry.file_type().await
|
||||||
if sub_ft.is_file() {
|
&& sub_ft.is_file()
|
||||||
|
{
|
||||||
load_if_matches(&sub_entry, cert_name, cert_data).await;
|
load_if_matches(&sub_entry, cert_name, cert_data).await;
|
||||||
}
|
}
|
||||||
// Ignore deeper levels
|
// Ignore deeper levels
|
||||||
@@ -289,7 +290,6 @@ async fn walk_dir(path: PathBuf, cert_name: &str, cert_data: &mut Vec<u8>) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
debug!("Certificate directory not found: {}", path.display());
|
debug!("Certificate directory not found: {}", path.display());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -282,41 +282,36 @@ impl Predicate for CompressionPredicate {
|
|||||||
// CompressionLayer before calling this predicate, so we don't need to check them here.
|
// CompressionLayer before calling this predicate, so we don't need to check them here.
|
||||||
|
|
||||||
// Check Content-Length header for minimum size threshold
|
// Check Content-Length header for minimum size threshold
|
||||||
if let Some(content_length) = response.headers().get(http::header::CONTENT_LENGTH) {
|
if let Some(content_length) = response.headers().get(http::header::CONTENT_LENGTH)
|
||||||
if let Ok(length_str) = content_length.to_str() {
|
&& let Ok(length_str) = content_length.to_str()
|
||||||
if let Ok(length) = length_str.parse::<u64>() {
|
&& let Ok(length) = length_str.parse::<u64>()
|
||||||
if length < self.config.min_size {
|
&& length < self.config.min_size
|
||||||
|
{
|
||||||
debug!(
|
debug!(
|
||||||
"Skipping compression for small response: size={} bytes, min_size={}",
|
"Skipping compression for small response: size={} bytes, min_size={}",
|
||||||
length, self.config.min_size
|
length, self.config.min_size
|
||||||
);
|
);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if the response matches configured extension via Content-Disposition
|
// Check if the response matches configured extension via Content-Disposition
|
||||||
if let Some(content_disposition) = response.headers().get(http::header::CONTENT_DISPOSITION) {
|
if let Some(content_disposition) = response.headers().get(http::header::CONTENT_DISPOSITION)
|
||||||
if let Ok(cd) = content_disposition.to_str() {
|
&& let Ok(cd) = content_disposition.to_str()
|
||||||
if let Some(filename) = CompressionConfig::extract_filename_from_content_disposition(cd) {
|
&& let Some(filename) = CompressionConfig::extract_filename_from_content_disposition(cd)
|
||||||
if self.config.matches_extension(&filename) {
|
&& self.config.matches_extension(&filename)
|
||||||
|
{
|
||||||
debug!("Compressing response: filename '{}' matches configured extension", filename);
|
debug!("Compressing response: filename '{}' matches configured extension", filename);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if the response matches configured MIME type
|
// Check if the response matches configured MIME type
|
||||||
if let Some(content_type) = response.headers().get(http::header::CONTENT_TYPE) {
|
if let Some(content_type) = response.headers().get(http::header::CONTENT_TYPE)
|
||||||
if let Ok(ct) = content_type.to_str() {
|
&& let Ok(ct) = content_type.to_str()
|
||||||
if self.config.matches_mime_type(ct) {
|
&& self.config.matches_mime_type(ct)
|
||||||
|
{
|
||||||
debug!("Compressing response: Content-Type '{}' matches configured MIME pattern", ct);
|
debug!("Compressing response: Content-Type '{}' matches configured MIME pattern", ct);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Default: don't compress (whitelist approach)
|
// Default: don't compress (whitelist approach)
|
||||||
debug!("Skipping compression: response does not match any configured extension or MIME type");
|
debug!("Skipping compression: response does not match any configured extension or MIME type");
|
||||||
|
|||||||
@@ -139,14 +139,14 @@ pub async fn start_http_server(
|
|||||||
};
|
};
|
||||||
|
|
||||||
// If address is IPv6 try to enable dual-stack; on failure, switch to IPv4 socket.
|
// If address is IPv6 try to enable dual-stack; on failure, switch to IPv4 socket.
|
||||||
if server_addr.is_ipv6() {
|
if server_addr.is_ipv6()
|
||||||
if let Err(e) = socket.set_only_v6(false) {
|
&& let Err(e) = socket.set_only_v6(false)
|
||||||
|
{
|
||||||
warn!("Failed to set IPV6_V6ONLY=false, attempting IPv4 fallback: {}", e);
|
warn!("Failed to set IPV6_V6ONLY=false, attempting IPv4 fallback: {}", e);
|
||||||
let ipv4_addr = SocketAddr::new(std::net::Ipv4Addr::UNSPECIFIED.into(), server_addr.port());
|
let ipv4_addr = SocketAddr::new(std::net::Ipv4Addr::UNSPECIFIED.into(), server_addr.port());
|
||||||
server_addr = ipv4_addr;
|
server_addr = ipv4_addr;
|
||||||
socket = socket2::Socket::new(socket2::Domain::IPV4, socket2::Type::STREAM, Some(socket2::Protocol::TCP))?;
|
socket = socket2::Socket::new(socket2::Domain::IPV4, socket2::Type::STREAM, Some(socket2::Protocol::TCP))?;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Common setup for both IPv4 and successful dual-stack IPv6
|
// Common setup for both IPv4 and successful dual-stack IPv6
|
||||||
let backlog = get_listen_backlog();
|
let backlog = get_listen_backlog();
|
||||||
@@ -434,12 +434,13 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result<Option<TlsAcceptor>> {
|
|||||||
debug!("Found TLS directory, checking for certificates");
|
debug!("Found TLS directory, checking for certificates");
|
||||||
|
|
||||||
// Make sure to use a modern encryption suite
|
// Make sure to use a modern encryption suite
|
||||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||||
let mtls_verifier = rustfs_utils::build_webpki_client_verifier(tls_path)?;
|
let mtls_verifier = rustfs_utils::build_webpki_client_verifier(tls_path)?;
|
||||||
|
|
||||||
// 1. Attempt to load all certificates in the directory (multi-certificate support, for SNI)
|
// 1. Attempt to load all certificates in the directory (multi-certificate support, for SNI)
|
||||||
if let Ok(cert_key_pairs) = rustfs_utils::load_all_certs_from_directory(tls_path) {
|
if let Ok(cert_key_pairs) = rustfs_utils::load_all_certs_from_directory(tls_path)
|
||||||
if !cert_key_pairs.is_empty() {
|
&& !cert_key_pairs.is_empty()
|
||||||
|
{
|
||||||
debug!("Found {} certificates, creating SNI-aware multi-cert resolver", cert_key_pairs.len());
|
debug!("Found {} certificates, creating SNI-aware multi-cert resolver", cert_key_pairs.len());
|
||||||
|
|
||||||
// Create an SNI-enabled certificate resolver
|
// Create an SNI-enabled certificate resolver
|
||||||
@@ -466,7 +467,6 @@ async fn setup_tls_acceptor(tls_path: &str) -> Result<Option<TlsAcceptor>> {
|
|||||||
|
|
||||||
return Ok(Some(TlsAcceptor::from(Arc::new(server_config))));
|
return Ok(Some(TlsAcceptor::from(Arc::new(server_config))));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Revert to the traditional single-certificate mode
|
// 2. Revert to the traditional single-certificate mode
|
||||||
let key_path = format!("{tls_path}/{RUSTFS_TLS_KEY}");
|
let key_path = format!("{tls_path}/{RUSTFS_TLS_KEY}");
|
||||||
@@ -520,7 +520,8 @@ struct ConnectionContext {
|
|||||||
/// 2. Build a complete service stack for this connection, including S3, RPC services, and all middleware.
|
/// 2. Build a complete service stack for this connection, including S3, RPC services, and all middleware.
|
||||||
/// 3. Use Hyper to handle HTTP requests on this connection.
|
/// 3. Use Hyper to handle HTTP requests on this connection.
|
||||||
/// 4. Incorporate connections into the management of elegant closures.
|
/// 4. Incorporate connections into the management of elegant closures.
|
||||||
#[instrument(skip_all, fields(peer_addr = %socket.peer_addr().map(|a| a.to_string()).unwrap_or_else(|_| "unknown".to_string())))]
|
#[instrument(skip_all, fields(peer_addr = %socket.peer_addr().map(|a| a.to_string()).unwrap_or_else(|_| "unknown".to_string())
|
||||||
|
))]
|
||||||
fn process_connection(
|
fn process_connection(
|
||||||
socket: TcpStream,
|
socket: TcpStream,
|
||||||
tls_acceptor: Option<Arc<TlsAcceptor>>,
|
tls_acceptor: Option<Arc<TlsAcceptor>>,
|
||||||
|
|||||||
+91
-109
@@ -519,14 +519,14 @@ fn validate_list_object_unordered_with_delimiter(delimiter: Option<&Delimiter>,
|
|||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Ok(params) = from_bytes::<ListObjectUnorderedQuery>(query.as_bytes()) {
|
if let Ok(params) = from_bytes::<ListObjectUnorderedQuery>(query.as_bytes())
|
||||||
if params.allow_unordered.as_deref() == Some("true") {
|
&& params.allow_unordered.as_deref() == Some("true")
|
||||||
|
{
|
||||||
return Err(S3Error::with_message(
|
return Err(S3Error::with_message(
|
||||||
S3ErrorCode::InvalidArgument,
|
S3ErrorCode::InvalidArgument,
|
||||||
"The allow-unordered parameter cannot be used when delimiter is specified.".to_string(),
|
"The allow-unordered parameter cannot be used when delimiter is specified.".to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -735,8 +735,8 @@ impl FS {
|
|||||||
let mut checksum_sha256 = input.checksum_sha256;
|
let mut checksum_sha256 = input.checksum_sha256;
|
||||||
let mut checksum_crc64nvme = input.checksum_crc64nvme;
|
let mut checksum_crc64nvme = input.checksum_crc64nvme;
|
||||||
|
|
||||||
if let Some(alg) = &input.checksum_algorithm {
|
if let Some(alg) = &input.checksum_algorithm
|
||||||
if let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| {
|
&& let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| {
|
||||||
let key = match alg.as_str() {
|
let key = match alg.as_str() {
|
||||||
ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(),
|
ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(),
|
||||||
ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(),
|
ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(),
|
||||||
@@ -750,7 +750,8 @@ impl FS {
|
|||||||
.get(key.unwrap_or_default())
|
.get(key.unwrap_or_default())
|
||||||
.and_then(|value| value.to_str().ok().map(|s| s.to_string()))
|
.and_then(|value| value.to_str().ok().map(|s| s.to_string()))
|
||||||
})
|
})
|
||||||
}) {
|
})
|
||||||
|
{
|
||||||
match alg.as_str() {
|
match alg.as_str() {
|
||||||
ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str,
|
ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str,
|
||||||
ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str,
|
ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str,
|
||||||
@@ -760,7 +761,6 @@ impl FS {
|
|||||||
_ => (),
|
_ => (),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
warn!(
|
warn!(
|
||||||
"put object extract checksum_crc32={checksum_crc32:?}, checksum_crc32c={checksum_crc32c:?}, checksum_sha1={checksum_sha1:?}, checksum_sha256={checksum_sha256:?}, checksum_crc64nvme={checksum_crc64nvme:?}",
|
"put object extract checksum_crc32={checksum_crc32:?}, checksum_crc32c={checksum_crc32c:?}, checksum_sha1={checksum_sha1:?}, checksum_sha256={checksum_sha256:?}, checksum_crc64nvme={checksum_crc64nvme:?}",
|
||||||
@@ -977,8 +977,9 @@ impl S3 for FS {
|
|||||||
|
|
||||||
let mut reader = HashReader::new(reader, length, actual_size, None, None, false).map_err(ApiError::from)?;
|
let mut reader = HashReader::new(reader, length, actual_size, None, None, false).map_err(ApiError::from)?;
|
||||||
|
|
||||||
if let Some(ref sse_alg) = effective_sse {
|
if let Some(ref sse_alg) = effective_sse
|
||||||
if is_managed_sse(sse_alg) {
|
&& is_managed_sse(sse_alg)
|
||||||
|
{
|
||||||
let material =
|
let material =
|
||||||
create_managed_encryption_material(&bucket, &key, sse_alg, effective_kms_key_id.clone(), actual_size).await?;
|
create_managed_encryption_material(&bucket, &key, sse_alg, effective_kms_key_id.clone(), actual_size).await?;
|
||||||
|
|
||||||
@@ -997,12 +998,11 @@ impl S3 for FS {
|
|||||||
let encrypt_reader = EncryptReader::new(reader, key_bytes, nonce);
|
let encrypt_reader = EncryptReader::new(reader, key_bytes, nonce);
|
||||||
reader = HashReader::new(Box::new(encrypt_reader), -1, actual_size, None, None, false).map_err(ApiError::from)?;
|
reader = HashReader::new(Box::new(encrypt_reader), -1, actual_size, None, None, false).map_err(ApiError::from)?;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Apply SSE-C encryption if customer-provided key is specified
|
// Apply SSE-C encryption if customer-provided key is specified
|
||||||
if let (Some(sse_alg), Some(sse_key), Some(sse_md5)) = (&sse_customer_algorithm, &sse_customer_key, &sse_customer_key_md5)
|
if let (Some(sse_alg), Some(sse_key), Some(sse_md5)) = (&sse_customer_algorithm, &sse_customer_key, &sse_customer_key_md5)
|
||||||
|
&& sse_alg.as_str() == "AES256"
|
||||||
{
|
{
|
||||||
if sse_alg.as_str() == "AES256" {
|
|
||||||
let key_bytes = BASE64_STANDARD.decode(sse_key.as_str()).map_err(|e| {
|
let key_bytes = BASE64_STANDARD.decode(sse_key.as_str()).map_err(|e| {
|
||||||
error!("Failed to decode SSE-C key: {}", e);
|
error!("Failed to decode SSE-C key: {}", e);
|
||||||
ApiError::from(StorageError::other("Invalid SSE-C key"))
|
ApiError::from(StorageError::other("Invalid SSE-C key"))
|
||||||
@@ -1035,7 +1035,6 @@ impl S3 for FS {
|
|||||||
let encrypt_reader = EncryptReader::new(reader, key_array, nonce);
|
let encrypt_reader = EncryptReader::new(reader, key_array, nonce);
|
||||||
reader = HashReader::new(Box::new(encrypt_reader), -1, actual_size, None, None, false).map_err(ApiError::from)?;
|
reader = HashReader::new(Box::new(encrypt_reader), -1, actual_size, None, None, false).map_err(ApiError::from)?;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
src_info.put_object_reader = Some(PutObjReader::new(reader));
|
src_info.put_object_reader = Some(PutObjReader::new(reader));
|
||||||
|
|
||||||
@@ -1246,16 +1245,15 @@ impl S3 for FS {
|
|||||||
|
|
||||||
let restore_object = Uuid::new_v4().to_string();
|
let restore_object = Uuid::new_v4().to_string();
|
||||||
//if let Some(rreq) = rreq {
|
//if let Some(rreq) = rreq {
|
||||||
if let Some(output_location) = &rreq.output_location {
|
if let Some(output_location) = &rreq.output_location
|
||||||
if let Some(s3) = &output_location.s3 {
|
&& let Some(s3) = &output_location.s3
|
||||||
if !s3.bucket_name.is_empty() {
|
&& !s3.bucket_name.is_empty()
|
||||||
|
{
|
||||||
header.insert(
|
header.insert(
|
||||||
X_AMZ_RESTORE_OUTPUT_PATH,
|
X_AMZ_RESTORE_OUTPUT_PATH,
|
||||||
format!("{}{}{}", s3.bucket_name, s3.prefix, restore_object).parse().unwrap(),
|
format!("{}{}{}", s3.bucket_name, s3.prefix, restore_object).parse().unwrap(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
//}
|
//}
|
||||||
/*send_event(EventArgs {
|
/*send_event(EventArgs {
|
||||||
event_name: event::ObjectRestorePost,
|
event_name: event::ObjectRestorePost,
|
||||||
@@ -1730,8 +1728,8 @@ impl S3 for FS {
|
|||||||
};
|
};
|
||||||
|
|
||||||
for dobjs in delete_results.iter() {
|
for dobjs in delete_results.iter() {
|
||||||
if let Some(dobj) = &dobjs.delete_object {
|
if let Some(dobj) = &dobjs.delete_object
|
||||||
if replicate_deletes
|
&& replicate_deletes
|
||||||
&& (dobj.delete_marker_replication_status() == ReplicationStatusType::Pending
|
&& (dobj.delete_marker_replication_status() == ReplicationStatusType::Pending
|
||||||
|| dobj.version_purge_status() == VersionPurgeStatusType::Pending)
|
|| dobj.version_purge_status() == VersionPurgeStatusType::Pending)
|
||||||
{
|
{
|
||||||
@@ -1749,7 +1747,6 @@ impl S3 for FS {
|
|||||||
schedule_replication_delete(deleted_object).await;
|
schedule_replication_delete(deleted_object).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let req_headers = req.headers.clone();
|
let req_headers = req.headers.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
@@ -1854,8 +1851,11 @@ impl S3 for FS {
|
|||||||
let cache_key = ConcurrencyManager::make_cache_key(&bucket, &key, version_id.as_deref());
|
let cache_key = ConcurrencyManager::make_cache_key(&bucket, &key, version_id.as_deref());
|
||||||
|
|
||||||
// Only attempt cache lookup if caching is enabled and for objects without range/part requests
|
// Only attempt cache lookup if caching is enabled and for objects without range/part requests
|
||||||
if manager.is_cache_enabled() && part_number.is_none() && range.is_none() {
|
if manager.is_cache_enabled()
|
||||||
if let Some(cached) = manager.get_cached_object(&cache_key).await {
|
&& part_number.is_none()
|
||||||
|
&& range.is_none()
|
||||||
|
&& let Some(cached) = manager.get_cached_object(&cache_key).await
|
||||||
|
{
|
||||||
let cache_serve_duration = request_start.elapsed();
|
let cache_serve_duration = request_start.elapsed();
|
||||||
|
|
||||||
debug!("Serving object from response cache: {} (latency: {:?})", cache_key, cache_serve_duration);
|
debug!("Serving object from response cache: {} (latency: {:?})", cache_key, cache_serve_duration);
|
||||||
@@ -1944,7 +1944,6 @@ impl S3 for FS {
|
|||||||
let _ = helper.complete(&result);
|
let _ = helper.complete(&result);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: getObjectInArchiveFileHandler object = xxx.zip/xxx/xxx.xxx
|
// TODO: getObjectInArchiveFileHandler object = xxx.zip/xxx/xxx.xxx
|
||||||
|
|
||||||
@@ -1954,11 +1953,11 @@ impl S3 for FS {
|
|||||||
|
|
||||||
let part_number = part_number.map(|v| v as usize);
|
let part_number = part_number.map(|v| v as usize);
|
||||||
|
|
||||||
if let Some(part_num) = part_number {
|
if let Some(part_num) = part_number
|
||||||
if part_num == 0 {
|
&& part_num == 0
|
||||||
|
{
|
||||||
return Err(s3_error!(InvalidArgument, "Invalid part number: part number must be greater than 0"));
|
return Err(s3_error!(InvalidArgument, "Invalid part number: part number must be greater than 0"));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let rs = range.map(|v| match v {
|
let rs = range.map(|v| match v {
|
||||||
Range::Int { first, last } => HTTPRangeSpec {
|
Range::Int { first, last } => HTTPRangeSpec {
|
||||||
@@ -2065,11 +2064,11 @@ impl S3 for FS {
|
|||||||
|
|
||||||
let mut rs = rs;
|
let mut rs = rs;
|
||||||
|
|
||||||
if let Some(part_number) = part_number {
|
if let Some(part_number) = part_number
|
||||||
if rs.is_none() {
|
&& rs.is_none()
|
||||||
|
{
|
||||||
rs = HTTPRangeSpec::from_object_info(&info, part_number);
|
rs = HTTPRangeSpec::from_object_info(&info, part_number);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let mut content_length = info.get_actual_size().map_err(ApiError::from)?;
|
let mut content_length = info.get_actual_size().map_err(ApiError::from)?;
|
||||||
|
|
||||||
@@ -2183,8 +2182,8 @@ impl S3 for FS {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if stored_sse_algorithm.is_none() {
|
if stored_sse_algorithm.is_none()
|
||||||
if let Some((key_bytes, nonce, original_size)) =
|
&& let Some((key_bytes, nonce, original_size)) =
|
||||||
decrypt_managed_encryption_key(&bucket, &key, &info.user_defined).await?
|
decrypt_managed_encryption_key(&bucket, &key, &info.user_defined).await?
|
||||||
{
|
{
|
||||||
if info.parts.len() > 1 {
|
if info.parts.len() > 1 {
|
||||||
@@ -2201,7 +2200,6 @@ impl S3 for FS {
|
|||||||
}
|
}
|
||||||
managed_encryption_applied = true;
|
managed_encryption_applied = true;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// For SSE-C encrypted objects, use the original size instead of encrypted size
|
// For SSE-C encrypted objects, use the original size instead of encrypted size
|
||||||
let response_content_length = if stored_sse_algorithm.is_some() {
|
let response_content_length = if stored_sse_algorithm.is_some() {
|
||||||
@@ -2518,11 +2516,11 @@ impl S3 for FS {
|
|||||||
|
|
||||||
let part_number = part_number.map(|v| v as usize);
|
let part_number = part_number.map(|v| v as usize);
|
||||||
|
|
||||||
if let Some(part_num) = part_number {
|
if let Some(part_num) = part_number
|
||||||
if part_num == 0 {
|
&& part_num == 0
|
||||||
|
{
|
||||||
return Err(s3_error!(InvalidArgument, "part_number invalid"));
|
return Err(s3_error!(InvalidArgument, "part_number invalid"));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let rs = range.map(|v| match v {
|
let rs = range.map(|v| match v {
|
||||||
Range::Int { first, last } => HTTPRangeSpec {
|
Range::Int { first, last } => HTTPRangeSpec {
|
||||||
@@ -2558,17 +2556,15 @@ impl S3 for FS {
|
|||||||
return Err(S3Error::new(S3ErrorCode::MethodNotAllowed));
|
return Err(S3Error::new(S3ErrorCode::MethodNotAllowed));
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(match_etag) = if_none_match {
|
if let Some(match_etag) = if_none_match
|
||||||
if let Some(strong_etag) = match_etag.into_etag() {
|
&& let Some(strong_etag) = match_etag.into_etag()
|
||||||
if info
|
&& info
|
||||||
.etag
|
.etag
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag)
|
.is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag)
|
||||||
{
|
{
|
||||||
return Err(S3Error::new(S3ErrorCode::NotModified));
|
return Err(S3Error::new(S3ErrorCode::NotModified));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(modified_since) = if_modified_since {
|
if let Some(modified_since) = if_modified_since {
|
||||||
// obj_time < givenTime + 1s
|
// obj_time < givenTime + 1s
|
||||||
@@ -2581,23 +2577,22 @@ impl S3 for FS {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some(match_etag) = if_match {
|
if let Some(match_etag) = if_match {
|
||||||
if let Some(strong_etag) = match_etag.into_etag() {
|
if let Some(strong_etag) = match_etag.into_etag()
|
||||||
if info
|
&& info
|
||||||
.etag
|
.etag
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag)
|
.is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag)
|
||||||
{
|
{
|
||||||
return Err(S3Error::new(S3ErrorCode::PreconditionFailed));
|
return Err(S3Error::new(S3ErrorCode::PreconditionFailed));
|
||||||
}
|
}
|
||||||
}
|
} else if let Some(unmodified_since) = if_unmodified_since
|
||||||
} else if let Some(unmodified_since) = if_unmodified_since {
|
&& info.mod_time.is_some_and(|mod_time| {
|
||||||
if info.mod_time.is_some_and(|mod_time| {
|
|
||||||
let give_time: OffsetDateTime = unmodified_since.into();
|
let give_time: OffsetDateTime = unmodified_since.into();
|
||||||
mod_time > give_time.add(time::Duration::seconds(1))
|
mod_time > give_time.add(time::Duration::seconds(1))
|
||||||
}) {
|
})
|
||||||
|
{
|
||||||
return Err(S3Error::new(S3ErrorCode::PreconditionFailed));
|
return Err(S3Error::new(S3ErrorCode::PreconditionFailed));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let event_info = info.clone();
|
let event_info = info.clone();
|
||||||
let content_type = {
|
let content_type = {
|
||||||
@@ -3080,11 +3075,11 @@ impl S3 for FS {
|
|||||||
let input = req.input;
|
let input = req.input;
|
||||||
|
|
||||||
// Save SSE-C parameters before moving input
|
// Save SSE-C parameters before moving input
|
||||||
if let Some(ref storage_class) = input.storage_class {
|
if let Some(ref storage_class) = input.storage_class
|
||||||
if !is_valid_storage_class(storage_class.as_str()) {
|
&& !is_valid_storage_class(storage_class.as_str())
|
||||||
|
{
|
||||||
return Err(s3_error!(InvalidStorageClass));
|
return Err(s3_error!(InvalidStorageClass));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
let PutObjectInput {
|
let PutObjectInput {
|
||||||
body,
|
body,
|
||||||
bucket,
|
bucket,
|
||||||
@@ -3116,20 +3111,18 @@ impl S3 for FS {
|
|||||||
match store.get_object_info(&bucket, &key, &ObjectOptions::default()).await {
|
match store.get_object_info(&bucket, &key, &ObjectOptions::default()).await {
|
||||||
Ok(info) => {
|
Ok(info) => {
|
||||||
if !info.delete_marker {
|
if !info.delete_marker {
|
||||||
if let Some(ifmatch) = if_match {
|
if let Some(ifmatch) = if_match
|
||||||
if let Some(strong_etag) = ifmatch.into_etag() {
|
&& let Some(strong_etag) = ifmatch.into_etag()
|
||||||
if info
|
&& info
|
||||||
.etag
|
.etag
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag)
|
.is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag)
|
||||||
{
|
{
|
||||||
return Err(s3_error!(PreconditionFailed));
|
return Err(s3_error!(PreconditionFailed));
|
||||||
}
|
}
|
||||||
}
|
if let Some(ifnonematch) = if_none_match
|
||||||
}
|
&& let Some(strong_etag) = ifnonematch.into_etag()
|
||||||
if let Some(ifnonematch) = if_none_match {
|
&& info
|
||||||
if let Some(strong_etag) = ifnonematch.into_etag() {
|
|
||||||
if info
|
|
||||||
.etag
|
.etag
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag)
|
.is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag)
|
||||||
@@ -3138,8 +3131,6 @@ impl S3 for FS {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
|
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
|
||||||
return Err(ApiError::from(err).into());
|
return Err(ApiError::from(err).into());
|
||||||
@@ -3344,12 +3335,12 @@ impl S3 for FS {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Apply managed SSE (SSE-S3 or SSE-KMS) when requested
|
// Apply managed SSE (SSE-S3 or SSE-KMS) when requested
|
||||||
if sse_customer_algorithm.is_none() {
|
if sse_customer_algorithm.is_none()
|
||||||
if let Some(sse_alg) = &effective_sse {
|
&& let Some(sse_alg) = &effective_sse
|
||||||
if is_managed_sse(sse_alg) {
|
&& is_managed_sse(sse_alg)
|
||||||
|
{
|
||||||
let material =
|
let material =
|
||||||
create_managed_encryption_material(&bucket, &key, sse_alg, effective_kms_key_id.clone(), actual_size)
|
create_managed_encryption_material(&bucket, &key, sse_alg, effective_kms_key_id.clone(), actual_size).await?;
|
||||||
.await?;
|
|
||||||
|
|
||||||
let ManagedEncryptionMaterial {
|
let ManagedEncryptionMaterial {
|
||||||
data_key,
|
data_key,
|
||||||
@@ -3364,10 +3355,7 @@ impl S3 for FS {
|
|||||||
effective_kms_key_id = Some(kms_key_used.clone());
|
effective_kms_key_id = Some(kms_key_used.clone());
|
||||||
|
|
||||||
let encrypt_reader = EncryptReader::new(reader, key_bytes, nonce);
|
let encrypt_reader = EncryptReader::new(reader, key_bytes, nonce);
|
||||||
reader =
|
reader = HashReader::new(Box::new(encrypt_reader), -1, actual_size, None, None, false).map_err(ApiError::from)?;
|
||||||
HashReader::new(Box::new(encrypt_reader), -1, actual_size, None, None, false).map_err(ApiError::from)?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut reader = PutObjReader::new(reader);
|
let mut reader = PutObjReader::new(reader);
|
||||||
@@ -3428,8 +3416,8 @@ impl S3 for FS {
|
|||||||
let mut checksum_sha256 = input.checksum_sha256;
|
let mut checksum_sha256 = input.checksum_sha256;
|
||||||
let mut checksum_crc64nvme = input.checksum_crc64nvme;
|
let mut checksum_crc64nvme = input.checksum_crc64nvme;
|
||||||
|
|
||||||
if let Some(alg) = &input.checksum_algorithm {
|
if let Some(alg) = &input.checksum_algorithm
|
||||||
if let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| {
|
&& let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| {
|
||||||
let key = match alg.as_str() {
|
let key = match alg.as_str() {
|
||||||
ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(),
|
ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(),
|
||||||
ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(),
|
ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(),
|
||||||
@@ -3443,7 +3431,8 @@ impl S3 for FS {
|
|||||||
.get(key.unwrap_or_default())
|
.get(key.unwrap_or_default())
|
||||||
.and_then(|value| value.to_str().ok().map(|s| s.to_string()))
|
.and_then(|value| value.to_str().ok().map(|s| s.to_string()))
|
||||||
})
|
})
|
||||||
}) {
|
})
|
||||||
|
{
|
||||||
match alg.as_str() {
|
match alg.as_str() {
|
||||||
ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str,
|
ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str,
|
||||||
ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str,
|
ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str,
|
||||||
@@ -3453,7 +3442,6 @@ impl S3 for FS {
|
|||||||
_ => (),
|
_ => (),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let output = PutObjectOutput {
|
let output = PutObjectOutput {
|
||||||
e_tag,
|
e_tag,
|
||||||
@@ -3495,11 +3483,11 @@ impl S3 for FS {
|
|||||||
} = req.input.clone();
|
} = req.input.clone();
|
||||||
|
|
||||||
// Validate storage class if provided
|
// Validate storage class if provided
|
||||||
if let Some(ref storage_class) = storage_class {
|
if let Some(ref storage_class) = storage_class
|
||||||
if !is_valid_storage_class(storage_class.as_str()) {
|
&& !is_valid_storage_class(storage_class.as_str())
|
||||||
|
{
|
||||||
return Err(s3_error!(InvalidStorageClass));
|
return Err(s3_error!(InvalidStorageClass));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// mc cp step 3
|
// mc cp step 3
|
||||||
|
|
||||||
@@ -3654,11 +3642,11 @@ impl S3 for FS {
|
|||||||
let mut body_stream = body.ok_or_else(|| s3_error!(IncompleteBody))?;
|
let mut body_stream = body.ok_or_else(|| s3_error!(IncompleteBody))?;
|
||||||
|
|
||||||
if size.is_none() {
|
if size.is_none() {
|
||||||
if let Some(val) = req.headers.get(AMZ_DECODED_CONTENT_LENGTH) {
|
if let Some(val) = req.headers.get(AMZ_DECODED_CONTENT_LENGTH)
|
||||||
if let Some(x) = atoi::atoi::<i64>(val.as_bytes()) {
|
&& let Some(x) = atoi::atoi::<i64>(val.as_bytes())
|
||||||
|
{
|
||||||
size = Some(x);
|
size = Some(x);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if size.is_none() {
|
if size.is_none() {
|
||||||
let mut total = 0i64;
|
let mut total = 0i64;
|
||||||
@@ -3828,8 +3816,8 @@ impl S3 for FS {
|
|||||||
let mut checksum_sha256 = input.checksum_sha256;
|
let mut checksum_sha256 = input.checksum_sha256;
|
||||||
let mut checksum_crc64nvme = input.checksum_crc64nvme;
|
let mut checksum_crc64nvme = input.checksum_crc64nvme;
|
||||||
|
|
||||||
if let Some(alg) = &input.checksum_algorithm {
|
if let Some(alg) = &input.checksum_algorithm
|
||||||
if let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| {
|
&& let Some(Some(checksum_str)) = req.trailing_headers.as_ref().map(|trailer| {
|
||||||
let key = match alg.as_str() {
|
let key = match alg.as_str() {
|
||||||
ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(),
|
ChecksumAlgorithm::CRC32 => rustfs_rio::ChecksumType::CRC32.key(),
|
||||||
ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(),
|
ChecksumAlgorithm::CRC32C => rustfs_rio::ChecksumType::CRC32C.key(),
|
||||||
@@ -3843,7 +3831,8 @@ impl S3 for FS {
|
|||||||
.get(key.unwrap_or_default())
|
.get(key.unwrap_or_default())
|
||||||
.and_then(|value| value.to_str().ok().map(|s| s.to_string()))
|
.and_then(|value| value.to_str().ok().map(|s| s.to_string()))
|
||||||
})
|
})
|
||||||
}) {
|
})
|
||||||
|
{
|
||||||
match alg.as_str() {
|
match alg.as_str() {
|
||||||
ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str,
|
ChecksumAlgorithm::CRC32 => checksum_crc32 = checksum_str,
|
||||||
ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str,
|
ChecksumAlgorithm::CRC32C => checksum_crc32c = checksum_str,
|
||||||
@@ -3853,7 +3842,6 @@ impl S3 for FS {
|
|||||||
_ => (),
|
_ => (),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let output = UploadPartOutput {
|
let output = UploadPartOutput {
|
||||||
checksum_crc32,
|
checksum_crc32,
|
||||||
@@ -3949,16 +3937,14 @@ impl S3 for FS {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(if_none_match) = copy_source_if_none_match {
|
if let Some(if_none_match) = copy_source_if_none_match
|
||||||
if let Some(ref etag) = src_info.etag {
|
&& let Some(ref etag) = src_info.etag
|
||||||
if let Some(strong_etag) = if_none_match.into_etag() {
|
&& let Some(strong_etag) = if_none_match.into_etag()
|
||||||
if ETag::Strong(etag.clone()) == strong_etag {
|
&& ETag::Strong(etag.clone()) == strong_etag
|
||||||
|
{
|
||||||
return Err(s3_error!(PreconditionFailed));
|
return Err(s3_error!(PreconditionFailed));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
// Weak ETag in If-None-Match is ignored (doesn't match)
|
// Weak ETag in If-None-Match is ignored (doesn't match)
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: Implement proper time comparison for if_modified_since and if_unmodified_since
|
// TODO: Implement proper time comparison for if_modified_since and if_unmodified_since
|
||||||
// For now, we'll skip these conditions
|
// For now, we'll skip these conditions
|
||||||
@@ -4157,11 +4143,11 @@ impl S3 for FS {
|
|||||||
|
|
||||||
let max_uploads = max_uploads.map(|x| x as usize).unwrap_or(MAX_PARTS_COUNT);
|
let max_uploads = max_uploads.map(|x| x as usize).unwrap_or(MAX_PARTS_COUNT);
|
||||||
|
|
||||||
if let Some(key_marker) = &key_marker {
|
if let Some(key_marker) = &key_marker
|
||||||
if !key_marker.starts_with(prefix.as_str()) {
|
&& !key_marker.starts_with(prefix.as_str())
|
||||||
|
{
|
||||||
return Err(s3_error!(NotImplemented, "Invalid key marker"));
|
return Err(s3_error!(NotImplemented, "Invalid key marker"));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let result = store
|
let result = store
|
||||||
.list_multipart_uploads(&bucket, &prefix, delimiter, key_marker, upload_id_marker, max_uploads)
|
.list_multipart_uploads(&bucket, &prefix, delimiter, key_marker, upload_id_marker, max_uploads)
|
||||||
@@ -4227,20 +4213,18 @@ impl S3 for FS {
|
|||||||
match store.get_object_info(&bucket, &key, &ObjectOptions::default()).await {
|
match store.get_object_info(&bucket, &key, &ObjectOptions::default()).await {
|
||||||
Ok(info) => {
|
Ok(info) => {
|
||||||
if !info.delete_marker {
|
if !info.delete_marker {
|
||||||
if let Some(ifmatch) = if_match {
|
if let Some(ifmatch) = if_match
|
||||||
if let Some(strong_etag) = ifmatch.into_etag() {
|
&& let Some(strong_etag) = ifmatch.into_etag()
|
||||||
if info
|
&& info
|
||||||
.etag
|
.etag
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag)
|
.is_some_and(|etag| ETag::Strong(etag.clone()) != strong_etag)
|
||||||
{
|
{
|
||||||
return Err(s3_error!(PreconditionFailed));
|
return Err(s3_error!(PreconditionFailed));
|
||||||
}
|
}
|
||||||
}
|
if let Some(ifnonematch) = if_none_match
|
||||||
}
|
&& let Some(strong_etag) = ifnonematch.into_etag()
|
||||||
if let Some(ifnonematch) = if_none_match {
|
&& info
|
||||||
if let Some(strong_etag) = ifnonematch.into_etag() {
|
|
||||||
if info
|
|
||||||
.etag
|
.etag
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag)
|
.is_some_and(|etag| ETag::Strong(etag.clone()) == strong_etag)
|
||||||
@@ -4249,8 +4233,6 @@ impl S3 for FS {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
|
if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
|
||||||
return Err(ApiError::from(err).into());
|
return Err(ApiError::from(err).into());
|
||||||
@@ -4852,12 +4834,12 @@ impl S3 for FS {
|
|||||||
let Some(input_cfg) = lifecycle_configuration else { return Err(s3_error!(InvalidArgument)) };
|
let Some(input_cfg) = lifecycle_configuration else { return Err(s3_error!(InvalidArgument)) };
|
||||||
|
|
||||||
let rcfg = metadata_sys::get_object_lock_config(&bucket).await;
|
let rcfg = metadata_sys::get_object_lock_config(&bucket).await;
|
||||||
if let Ok(rcfg) = rcfg {
|
if let Ok(rcfg) = rcfg
|
||||||
if let Err(err) = input_cfg.validate(&rcfg.0).await {
|
&& let Err(err) = input_cfg.validate(&rcfg.0).await
|
||||||
|
{
|
||||||
//return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockValidateFailed".into()), err.to_string()));
|
//return Err(S3Error::with_message(S3ErrorCode::Custom("BucketLockValidateFailed".into()), err.to_string()));
|
||||||
return Err(S3Error::with_message(S3ErrorCode::Custom("ValidateFailed".into()), err.to_string()));
|
return Err(S3Error::with_message(S3ErrorCode::Custom("ValidateFailed".into()), err.to_string()));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if let Err(err) = validate_transition_tier(&input_cfg).await {
|
if let Err(err) = validate_transition_tier(&input_cfg).await {
|
||||||
//warn!("lifecycle_configuration add failed, err: {:?}", err);
|
//warn!("lifecycle_configuration add failed, err: {:?}", err);
|
||||||
@@ -5735,8 +5717,9 @@ impl S3 for FS {
|
|||||||
|
|
||||||
/// Auxiliary functions: extract prefixes and suffixes
|
/// Auxiliary functions: extract prefixes and suffixes
|
||||||
fn extract_prefix_suffix(filter: Option<&NotificationConfigurationFilter>) -> (String, String) {
|
fn extract_prefix_suffix(filter: Option<&NotificationConfigurationFilter>) -> (String, String) {
|
||||||
if let Some(filter) = filter {
|
if let Some(filter) = filter
|
||||||
if let Some(filter_rules) = &filter.key {
|
&& let Some(filter_rules) = &filter.key
|
||||||
|
{
|
||||||
let mut prefix = String::new();
|
let mut prefix = String::new();
|
||||||
let mut suffix = String::new();
|
let mut suffix = String::new();
|
||||||
if let Some(rules) = &filter_rules.filter_rules {
|
if let Some(rules) = &filter_rules.filter_rules {
|
||||||
@@ -5752,7 +5735,6 @@ fn extract_prefix_suffix(filter: Option<&NotificationConfigurationFilter>) -> (S
|
|||||||
}
|
}
|
||||||
return (prefix, suffix);
|
return (prefix, suffix);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
(String::new(), String::new())
|
(String::new(), String::new())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -86,11 +86,11 @@ impl OperationHelper {
|
|||||||
.req_path(req.uri.path().to_string())
|
.req_path(req.uri.path().to_string())
|
||||||
.req_query(extract_req_params(req));
|
.req_query(extract_req_params(req));
|
||||||
|
|
||||||
if let Some(req_id) = req.headers.get("x-amz-request-id") {
|
if let Some(req_id) = req.headers.get("x-amz-request-id")
|
||||||
if let Ok(id_str) = req_id.to_str() {
|
&& let Ok(id_str) = req_id.to_str()
|
||||||
|
{
|
||||||
audit_builder = audit_builder.request_id(id_str);
|
audit_builder = audit_builder.request_id(id_str);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// initialize event builder
|
// initialize event builder
|
||||||
// object is a placeholder that must be set later using the `object()` method.
|
// object is a placeholder that must be set later using the `object()` method.
|
||||||
@@ -194,8 +194,9 @@ impl Drop for OperationHelper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Distribute event notification (only on success)
|
// Distribute event notification (only on success)
|
||||||
if self.api_builder.0.status.as_deref() == Some("success") {
|
if self.api_builder.0.status.as_deref() == Some("success")
|
||||||
if let Some(builder) = self.event_builder.take() {
|
&& let Some(builder) = self.event_builder.take()
|
||||||
|
{
|
||||||
let event_args = builder.build();
|
let event_args = builder.build();
|
||||||
// Avoid generating notifications for copy requests
|
// Avoid generating notifications for copy requests
|
||||||
if !event_args.is_replication_request() {
|
if !event_args.is_replication_request() {
|
||||||
@@ -206,4 +207,3 @@ impl Drop for OperationHelper {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|||||||
@@ -64,14 +64,13 @@ pub async fn del_opts(
|
|||||||
|
|
||||||
let vid = vid.map(|v| v.as_str().trim().to_owned());
|
let vid = vid.map(|v| v.as_str().trim().to_owned());
|
||||||
|
|
||||||
if let Some(ref id) = vid {
|
if let Some(ref id) = vid
|
||||||
if *id != Uuid::nil().to_string()
|
&& *id != Uuid::nil().to_string()
|
||||||
&& let Err(err) = Uuid::parse_str(id.as_str())
|
&& let Err(err) = Uuid::parse_str(id.as_str())
|
||||||
{
|
{
|
||||||
error!("del_opts: invalid version id: {} error: {}", id, err);
|
error!("del_opts: invalid version id: {} error: {}", id, err);
|
||||||
return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone()));
|
return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone()));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let mut opts = put_opts_from_headers(headers, metadata.clone()).map_err(|err| {
|
let mut opts = put_opts_from_headers(headers, metadata.clone()).map_err(|err| {
|
||||||
error!("del_opts: invalid argument: {} error: {}", object, err);
|
error!("del_opts: invalid argument: {} error: {}", object, err);
|
||||||
@@ -111,13 +110,12 @@ pub async fn get_opts(
|
|||||||
|
|
||||||
let vid = vid.map(|v| v.as_str().trim().to_owned());
|
let vid = vid.map(|v| v.as_str().trim().to_owned());
|
||||||
|
|
||||||
if let Some(ref id) = vid {
|
if let Some(ref id) = vid
|
||||||
if *id != Uuid::nil().to_string()
|
&& *id != Uuid::nil().to_string()
|
||||||
&& let Err(_err) = Uuid::parse_str(id.as_str())
|
&& let Err(_err) = Uuid::parse_str(id.as_str())
|
||||||
{
|
{
|
||||||
return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone()));
|
return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone()));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let mut opts = get_default_opts(headers, HashMap::new(), false)
|
let mut opts = get_default_opts(headers, HashMap::new(), false)
|
||||||
.map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?;
|
.map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?;
|
||||||
@@ -187,13 +185,12 @@ pub async fn put_opts(
|
|||||||
|
|
||||||
let vid = vid.map(|v| v.as_str().trim().to_owned());
|
let vid = vid.map(|v| v.as_str().trim().to_owned());
|
||||||
|
|
||||||
if let Some(ref id) = vid {
|
if let Some(ref id) = vid
|
||||||
if *id != Uuid::nil().to_string()
|
&& *id != Uuid::nil().to_string()
|
||||||
&& let Err(_err) = Uuid::parse_str(id.as_str())
|
&& let Err(_err) = Uuid::parse_str(id.as_str())
|
||||||
{
|
{
|
||||||
return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone()));
|
return Err(StorageError::InvalidVersionID(bucket.to_owned(), object.to_owned(), id.clone()));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
let mut opts = put_opts_from_headers(headers, metadata)
|
let mut opts = put_opts_from_headers(headers, metadata)
|
||||||
.map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?;
|
.map_err(|err| StorageError::InvalidArgument(bucket.to_owned(), object.to_owned(), err.to_string()))?;
|
||||||
@@ -512,13 +509,12 @@ fn skip_content_sha256_cksum(headers: &HeaderMap<HeaderValue>) -> bool {
|
|||||||
// such broken clients and content-length > 0.
|
// such broken clients and content-length > 0.
|
||||||
// For now, we'll assume strict compatibility is disabled
|
// For now, we'll assume strict compatibility is disabled
|
||||||
// In a real implementation, you would check a global config
|
// In a real implementation, you would check a global config
|
||||||
if let Some(content_length) = headers.get("content-length") {
|
if let Some(content_length) = headers.get("content-length")
|
||||||
if let Ok(length_str) = content_length.to_str() {
|
&& let Ok(length_str) = content_length.to_str()
|
||||||
if let Ok(length) = length_str.parse::<i64>() {
|
&& let Ok(length) = length_str.parse::<i64>()
|
||||||
|
{
|
||||||
return length > 0; // && !global_server_ctxt.strict_s3_compat
|
return length > 0; // && !global_server_ctxt.strict_s3_compat
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
_ => false,
|
_ => false,
|
||||||
@@ -546,11 +542,11 @@ fn get_content_sha256_cksum(headers: &HeaderMap<HeaderValue>, service_type: Serv
|
|||||||
};
|
};
|
||||||
|
|
||||||
// We found 'X-Amz-Content-Sha256' return the captured value.
|
// We found 'X-Amz-Content-Sha256' return the captured value.
|
||||||
if let Some(header_value) = content_sha256 {
|
if let Some(header_value) = content_sha256
|
||||||
if let Ok(value) = header_value.to_str() {
|
&& let Ok(value) = header_value.to_str()
|
||||||
|
{
|
||||||
return value.to_string();
|
return value.to_string();
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// We couldn't find 'X-Amz-Content-Sha256'.
|
// We couldn't find 'X-Amz-Content-Sha256'.
|
||||||
default_sha256_cksum
|
default_sha256_cksum
|
||||||
|
|||||||
@@ -75,11 +75,11 @@ fn increment_version(version: &str) -> Result<String, Box<dyn std::error::Error>
|
|||||||
let (major, minor, patch, pre_release) = parse_version(version)?;
|
let (major, minor, patch, pre_release) = parse_version(version)?;
|
||||||
|
|
||||||
// If there's a pre-release identifier, increment the pre-release version number
|
// If there's a pre-release identifier, increment the pre-release version number
|
||||||
if let Some(pre) = pre_release {
|
if let Some(pre) = pre_release
|
||||||
if let Some(new_pre) = increment_pre_release(&pre) {
|
&& let Some(new_pre) = increment_pre_release(&pre)
|
||||||
|
{
|
||||||
return Ok(format!("{major}.{minor}.{patch}-{new_pre}"));
|
return Ok(format!("{major}.{minor}.{patch}-{new_pre}"));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Otherwise increment patch version number
|
// Otherwise increment patch version number
|
||||||
Ok(format!("{major}.{minor}.{}", patch + 1))
|
Ok(format!("{major}.{minor}.{}", patch + 1))
|
||||||
@@ -107,11 +107,11 @@ pub fn parse_version(version: &str) -> VersionParseResult {
|
|||||||
fn increment_pre_release(pre_release: &str) -> Option<String> {
|
fn increment_pre_release(pre_release: &str) -> Option<String> {
|
||||||
// Handle pre-release versions like "alpha.19"
|
// Handle pre-release versions like "alpha.19"
|
||||||
let parts: Vec<&str> = pre_release.split('.').collect();
|
let parts: Vec<&str> = pre_release.split('.').collect();
|
||||||
if parts.len() == 2 {
|
if parts.len() == 2
|
||||||
if let Ok(num) = parts[1].parse::<u32>() {
|
&& let Ok(num) = parts[1].parse::<u32>()
|
||||||
|
{
|
||||||
return Some(format!("{}.{}", parts[0], num + 1));
|
return Some(format!("{}.{}", parts[0], num + 1));
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Handle pre-release versions like "alpha19"
|
// Handle pre-release versions like "alpha19"
|
||||||
if let Some(pos) = pre_release.rfind(|c: char| c.is_alphabetic()) {
|
if let Some(pos) = pre_release.rfind(|c: char| c.is_alphabetic()) {
|
||||||
|
|||||||
Reference in New Issue
Block a user