fix(scanner): cache object lock config (#4422)

* fix(scanner): cache object lock config

* test(scanner): update lifecycle scanner items
This commit is contained in:
Zhengchao An
2026-07-08 15:01:57 +08:00
committed by GitHub
parent c0d5f938f2
commit e0972f19ac
4 changed files with 81 additions and 7 deletions
+4 -1
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use s3s::dto::BucketLifecycleConfiguration;
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration};
use serde::{Deserialize, Serialize};
use std::{
collections::{HashMap, HashSet},
@@ -304,6 +304,8 @@ pub struct DataUsageCacheInfo {
pub scan_checkpoint: Option<DataUsageScanCheckpoint>,
#[serde(default)]
pub pending_heals: Vec<PendingScannerHeal>,
#[serde(default)]
pub object_lock: Option<Arc<ObjectLockConfiguration>>,
}
/// Data usage cache
@@ -1272,6 +1274,7 @@ mod tests {
assert_eq!(decoded.next_cycle, 7);
assert!(decoded.scan_resume_after.is_none());
assert!(decoded.scan_checkpoint.is_none());
assert!(decoded.object_lock.is_none());
assert!(decoded.pending_heals.is_empty());
}
+6
View File
@@ -669,6 +669,7 @@ pub struct ScannerItem {
pub object_name: String,
pub file_type: FileType,
pub lifecycle: Option<Arc<BucketLifecycleConfiguration>>,
pub object_lock: Option<Arc<ObjectLockConfiguration>>,
pub replication: Option<Arc<ReplicationConfig>>,
pub heal_enabled: bool,
pub heal_bitrot: bool,
@@ -1801,6 +1802,7 @@ impl FolderScanner {
} else {
None
};
let active_object_lock = self.old_cache.info.object_lock.clone();
self.sleeper.sleep_folder().await;
@@ -2023,6 +2025,7 @@ impl FolderScanner {
prefix: rustfs_utils::path::dir(&prefix),
object_name: file_name,
lifecycle: active_life_cycle.clone(),
object_lock: active_object_lock.clone(),
replication: active_replication.clone(),
heal_enabled,
heal_bitrot: self.scan_mode == HealScanMode::Deep,
@@ -2987,6 +2990,7 @@ mod tests {
object_name: "xl.meta".to_string(),
file_type,
lifecycle: None,
object_lock: None,
replication: None,
heal_enabled: false,
heal_bitrot: false,
@@ -3017,6 +3021,7 @@ mod tests {
object_name: "xl.meta".to_string(),
file_type,
lifecycle: None,
object_lock: None,
replication: None,
heal_enabled: false,
heal_bitrot: false,
@@ -3994,6 +3999,7 @@ mod tests {
object_name: "object".to_string(),
file_type,
lifecycle: None,
object_lock: None,
replication: None,
heal_enabled: true,
heal_bitrot: true,
+69 -6
View File
@@ -28,7 +28,7 @@ use rustfs_common::metrics::{Metric, Metrics, emit_scan_bucket_drive_complete, e
use rustfs_config::{ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS};
use rustfs_filemeta::FileMeta;
use rustfs_utils::path::path_join_buf;
use s3s::dto::{BucketLifecycleConfiguration, ReplicationConfiguration};
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ObjectLockEnabled, ReplicationConfiguration};
use std::collections::{HashMap, HashSet};
use std::path::Path;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
@@ -92,6 +92,24 @@ fn scanner_metadata_transient_error(reason: impl std::fmt::Display, bucket: &str
))
}
async fn object_lock_config_for_scanner_item(item: &ScannerItem) -> Option<Arc<ObjectLockConfiguration>> {
if let Some(config) = item.object_lock.clone() {
return Some(config);
}
get_object_lock_config(&item.bucket)
.await
.ok()
.map(|(config, _)| Arc::new(config))
}
fn object_lock_config_enabled(config: &ObjectLockConfiguration) -> bool {
config
.object_lock_enabled
.as_ref()
.is_some_and(|enabled| enabled.as_str() == ObjectLockEnabled::ENABLED)
}
#[derive(Clone)]
pub struct ScannerBucketScanPlan {
buckets: Vec<BucketInfo>,
@@ -1515,10 +1533,7 @@ impl ScannerIODisk for Disk {
.insert(storageclass::RRS.to_string(), TierStats::default());
}
let lock_config = match get_object_lock_config(&item.bucket).await {
Ok((cfg, _)) => Some(Arc::new(cfg)),
Err(_) => None,
};
let lock_config = object_lock_config_for_scanner_item(&item).await;
item.apply_actions(object_infos, lock_config, &mut size_summary).await;
@@ -1574,7 +1589,11 @@ impl ScannerIODisk for Disk {
cache.info.replication = Some(Arc::new(ReplicationConfig::new(Some(replication_config), Some(targets))));
}
// TODO: object lock
if let Ok((object_lock_config, _)) = get_object_lock_config(&cache.info.name).await
&& object_lock_config_enabled(&object_lock_config)
{
cache.info.object_lock = Some(Arc::new(object_lock_config));
}
let Some(ecstore) = resolve_scanner_object_store_handle() else {
error!(
@@ -1689,6 +1708,47 @@ mod tests {
}
}
#[tokio::test]
async fn scanner_item_object_lock_uses_cached_config() {
let temp_dir = std::env::temp_dir();
let cached = Arc::new(ObjectLockConfiguration {
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
..Default::default()
});
let item = ScannerItem {
path: temp_dir.join("object").to_string_lossy().to_string(),
bucket: "bucket".to_string(),
prefix: String::new(),
object_name: "object".to_string(),
file_type: std::fs::metadata(&temp_dir)
.expect("temp dir metadata should be readable")
.file_type(),
lifecycle: None,
object_lock: Some(cached.clone()),
replication: None,
heal_enabled: false,
heal_bitrot: false,
debug: false,
};
let resolved = object_lock_config_for_scanner_item(&item)
.await
.expect("cached object-lock config should resolve");
assert!(Arc::ptr_eq(&resolved, &cached));
}
#[test]
fn object_lock_config_enabled_accepts_enabled_only() {
let enabled = ObjectLockConfiguration {
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
..Default::default()
};
assert!(object_lock_config_enabled(&enabled));
assert!(!object_lock_config_enabled(&ObjectLockConfiguration::default()));
}
#[test]
#[serial]
fn dirty_usage_snapshot_clear_preserves_newer_generation() {
@@ -1960,6 +2020,7 @@ mod tests {
object_name: STORAGE_FORMAT_FILE.to_string(),
file_type,
lifecycle: None,
object_lock: None,
replication: None,
heal_enabled: false,
heal_bitrot: false,
@@ -2015,6 +2076,7 @@ mod tests {
object_name: STORAGE_FORMAT_FILE.to_string(),
file_type,
lifecycle: None,
object_lock: None,
replication: None,
heal_enabled: false,
heal_bitrot: false,
@@ -2085,6 +2147,7 @@ mod tests {
object_name: STORAGE_FORMAT_FILE.to_string(),
file_type,
lifecycle: None,
object_lock: None,
replication: None,
heal_enabled: false,
heal_bitrot: false,
@@ -608,6 +608,7 @@ async fn scan_object_with_lifecycle(disk_path: &Path, bucket: &str, object: &str
object_name: STORAGE_FORMAT_FILE.to_string(),
file_type,
lifecycle,
object_lock: None,
replication: None,
heal_enabled: false,
heal_bitrot: false,
@@ -644,6 +645,7 @@ async fn scan_object_metadata(disk_path: &Path, bucket: &str, object: &str) {
object_name: STORAGE_FORMAT_FILE.to_string(),
file_type,
lifecycle: None,
object_lock: None,
replication: None,
heal_enabled: false,
heal_bitrot: false,