mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-27 00:38:16 +00:00
fix(scanner): cache object lock config (#4422)
* fix(scanner): cache object lock config * test(scanner): update lifecycle scanner items
This commit is contained in:
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use s3s::dto::BucketLifecycleConfiguration;
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
@@ -304,6 +304,8 @@ pub struct DataUsageCacheInfo {
|
||||
pub scan_checkpoint: Option<DataUsageScanCheckpoint>,
|
||||
#[serde(default)]
|
||||
pub pending_heals: Vec<PendingScannerHeal>,
|
||||
#[serde(default)]
|
||||
pub object_lock: Option<Arc<ObjectLockConfiguration>>,
|
||||
}
|
||||
|
||||
/// Data usage cache
|
||||
@@ -1272,6 +1274,7 @@ mod tests {
|
||||
assert_eq!(decoded.next_cycle, 7);
|
||||
assert!(decoded.scan_resume_after.is_none());
|
||||
assert!(decoded.scan_checkpoint.is_none());
|
||||
assert!(decoded.object_lock.is_none());
|
||||
assert!(decoded.pending_heals.is_empty());
|
||||
}
|
||||
|
||||
|
||||
@@ -669,6 +669,7 @@ pub struct ScannerItem {
|
||||
pub object_name: String,
|
||||
pub file_type: FileType,
|
||||
pub lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
|
||||
pub object_lock: Option<Arc<ObjectLockConfiguration>>,
|
||||
pub replication: Option<Arc<ReplicationConfig>>,
|
||||
pub heal_enabled: bool,
|
||||
pub heal_bitrot: bool,
|
||||
@@ -1801,6 +1802,7 @@ impl FolderScanner {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let active_object_lock = self.old_cache.info.object_lock.clone();
|
||||
|
||||
self.sleeper.sleep_folder().await;
|
||||
|
||||
@@ -2023,6 +2025,7 @@ impl FolderScanner {
|
||||
prefix: rustfs_utils::path::dir(&prefix),
|
||||
object_name: file_name,
|
||||
lifecycle: active_life_cycle.clone(),
|
||||
object_lock: active_object_lock.clone(),
|
||||
replication: active_replication.clone(),
|
||||
heal_enabled,
|
||||
heal_bitrot: self.scan_mode == HealScanMode::Deep,
|
||||
@@ -2987,6 +2990,7 @@ mod tests {
|
||||
object_name: "xl.meta".to_string(),
|
||||
file_type,
|
||||
lifecycle: None,
|
||||
object_lock: None,
|
||||
replication: None,
|
||||
heal_enabled: false,
|
||||
heal_bitrot: false,
|
||||
@@ -3017,6 +3021,7 @@ mod tests {
|
||||
object_name: "xl.meta".to_string(),
|
||||
file_type,
|
||||
lifecycle: None,
|
||||
object_lock: None,
|
||||
replication: None,
|
||||
heal_enabled: false,
|
||||
heal_bitrot: false,
|
||||
@@ -3994,6 +3999,7 @@ mod tests {
|
||||
object_name: "object".to_string(),
|
||||
file_type,
|
||||
lifecycle: None,
|
||||
object_lock: None,
|
||||
replication: None,
|
||||
heal_enabled: true,
|
||||
heal_bitrot: true,
|
||||
|
||||
@@ -28,7 +28,7 @@ use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, e
|
||||
use rustfs_config::{ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS};
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ReplicationConfiguration};
|
||||
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
@@ -92,6 +92,24 @@ fn scanner_metadata_transient_error(reason: impl std::fmt::Display, bucket: &str
|
||||
))
|
||||
}
|
||||
|
||||
async fn object_lock_config_for_scanner_item(item: &ScannerItem) -> Option<Arc<ObjectLockConfiguration>> {
|
||||
if let Some(config) = item.object_lock.clone() {
|
||||
return Some(config);
|
||||
}
|
||||
|
||||
get_object_lock_config(&item.bucket)
|
||||
.await
|
||||
.ok()
|
||||
.map(|(config, _)| Arc::new(config))
|
||||
}
|
||||
|
||||
fn object_lock_config_enabled(config: &ObjectLockConfiguration) -> bool {
|
||||
config
|
||||
.object_lock_enabled
|
||||
.as_ref()
|
||||
.is_some_and(|enabled| enabled.as_str() == ObjectLockEnabled::ENABLED)
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ScannerBucketScanPlan {
|
||||
buckets: Vec<BucketInfo>,
|
||||
@@ -1515,10 +1533,7 @@ impl ScannerIODisk for Disk {
|
||||
.insert(storageclass::RRS.to_string(), TierStats::default());
|
||||
}
|
||||
|
||||
let lock_config = match get_object_lock_config(&item.bucket).await {
|
||||
Ok((cfg, _)) => Some(Arc::new(cfg)),
|
||||
Err(_) => None,
|
||||
};
|
||||
let lock_config = object_lock_config_for_scanner_item(&item).await;
|
||||
|
||||
item.apply_actions(object_infos, lock_config, &mut size_summary).await;
|
||||
|
||||
@@ -1574,7 +1589,11 @@ impl ScannerIODisk for Disk {
|
||||
cache.info.replication = Some(Arc::new(ReplicationConfig::new(Some(replication_config), Some(targets))));
|
||||
}
|
||||
|
||||
// TODO: object lock
|
||||
if let Ok((object_lock_config, _)) = get_object_lock_config(&cache.info.name).await
|
||||
&& object_lock_config_enabled(&object_lock_config)
|
||||
{
|
||||
cache.info.object_lock = Some(Arc::new(object_lock_config));
|
||||
}
|
||||
|
||||
let Some(ecstore) = resolve_scanner_object_store_handle() else {
|
||||
error!(
|
||||
@@ -1689,6 +1708,47 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_item_object_lock_uses_cached_config() {
|
||||
let temp_dir = std::env::temp_dir();
|
||||
let cached = Arc::new(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
..Default::default()
|
||||
});
|
||||
let item = ScannerItem {
|
||||
path: temp_dir.join("object").to_string_lossy().to_string(),
|
||||
bucket: "bucket".to_string(),
|
||||
prefix: String::new(),
|
||||
object_name: "object".to_string(),
|
||||
file_type: std::fs::metadata(&temp_dir)
|
||||
.expect("temp dir metadata should be readable")
|
||||
.file_type(),
|
||||
lifecycle: None,
|
||||
object_lock: Some(cached.clone()),
|
||||
replication: None,
|
||||
heal_enabled: false,
|
||||
heal_bitrot: false,
|
||||
debug: false,
|
||||
};
|
||||
|
||||
let resolved = object_lock_config_for_scanner_item(&item)
|
||||
.await
|
||||
.expect("cached object-lock config should resolve");
|
||||
|
||||
assert!(Arc::ptr_eq(&resolved, &cached));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_lock_config_enabled_accepts_enabled_only() {
|
||||
let enabled = ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(object_lock_config_enabled(&enabled));
|
||||
assert!(!object_lock_config_enabled(&ObjectLockConfiguration::default()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn dirty_usage_snapshot_clear_preserves_newer_generation() {
|
||||
@@ -1960,6 +2020,7 @@ mod tests {
|
||||
object_name: STORAGE_FORMAT_FILE.to_string(),
|
||||
file_type,
|
||||
lifecycle: None,
|
||||
object_lock: None,
|
||||
replication: None,
|
||||
heal_enabled: false,
|
||||
heal_bitrot: false,
|
||||
@@ -2015,6 +2076,7 @@ mod tests {
|
||||
object_name: STORAGE_FORMAT_FILE.to_string(),
|
||||
file_type,
|
||||
lifecycle: None,
|
||||
object_lock: None,
|
||||
replication: None,
|
||||
heal_enabled: false,
|
||||
heal_bitrot: false,
|
||||
@@ -2085,6 +2147,7 @@ mod tests {
|
||||
object_name: STORAGE_FORMAT_FILE.to_string(),
|
||||
file_type,
|
||||
lifecycle: None,
|
||||
object_lock: None,
|
||||
replication: None,
|
||||
heal_enabled: false,
|
||||
heal_bitrot: false,
|
||||
|
||||
@@ -608,6 +608,7 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str
|
||||
object_name: STORAGE_FORMAT_FILE.to_string(),
|
||||
file_type,
|
||||
lifecycle,
|
||||
object_lock: None,
|
||||
replication: None,
|
||||
heal_enabled: false,
|
||||
heal_bitrot: false,
|
||||
@@ -644,6 +645,7 @@ async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) {
|
||||
object_name: STORAGE_FORMAT_FILE.to_string(),
|
||||
file_type,
|
||||
lifecycle: None,
|
||||
object_lock: None,
|
||||
replication: None,
|
||||
heal_enabled: false,
|
||||
heal_bitrot: false,
|
||||
|
||||
Reference in New Issue
Block a user