gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-21 20:06:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(scanner): reduce per-object allocation churn (#6318)

Browse Source 
Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-08-21 00:47:48 +08:00
committed by GitHub
parent cee0d5cf9b
commit 762919b1ba
5 changed files with 109 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/scanner/src/scanner_folder/item_actions.rs
+57 -16
View File
@@ -274,18 +274,13 @@ impl ScannerItem {
/// Transform meta directory by splitting prefix and extracting object name
/// This converts a directory path like "bucket/dir1/dir2/file" to prefix="bucket/dir1/dir2" and object_name="file"
pub fn transform_meta_dir(&mut self) {
let prefix = self.prefix.clone(); // Clone to avoid borrow checker issues
let split: Vec<&str> = prefix.split(SLASH_SEPARATOR).collect();
if split.len() > 1 {
let prefix_parts: Vec<&str> = split[..split.len() - 1].to_vec();
self.prefix = path_join_buf(&prefix_parts);
let prefix = std::mem::take(&mut self.prefix);
if let Some((parent, object_name)) = prefix.rsplit_once(SLASH_SEPARATOR) {
self.prefix = path_join_buf(&[parent]);
self.object_name = object_name.to_string();
} else {
self.prefix = String::new();
self.object_name = prefix;
}
// Object name is the last element
self.object_name = split.last().unwrap_or(&"").to_string();
}
pub(super) fn metadata_object_path(&self) -> String {
@@ -301,13 +296,14 @@ impl ScannerItem {
versioning_config: VersioningConfiguration,
size_summary: &mut SizeSummary,
) {
let object_path = self.object_path();
if object_infos.is_empty() {
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object_path = %self.object_path(),
object_path = %object_path,
state = "no_object_versions",
"Scanner lifecycle action skipped"
);
@@ -318,7 +314,7 @@ impl ScannerItem {
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object_path = %self.object_path(),
object_path = %object_path,
state = "started",
"Scanner lifecycle evaluation started"
);
@@ -360,7 +356,7 @@ impl ScannerItem {
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object_path = %self.object_path(),
object_path = %object_path,
state = "no_lifecycle_config",
"Scanner lifecycle action finished without lifecycle rules"
);
@@ -385,7 +381,7 @@ impl ScannerItem {
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object_path = %self.object_path(),
object_path = %object_path,
state = "evaluate_failed",
error = %e,
"Scanner lifecycle action evaluation failed"
@@ -502,7 +498,7 @@ impl ScannerItem {
emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at);
if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) {
done_ilm(1)();
if !versioning_config.prefix_enabled(&self.object_path()) && event.action == IlmAction::DeleteAction {
if !versioning_config.prefix_enabled(&object_path) && event.action == IlmAction::DeleteAction {
remaining_versions -= 1;
size = 0;
}
@@ -570,7 +566,7 @@ impl ScannerItem {
trace_emit(|| {
TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerIlmAction)
.with_bucket(self.bucket.as_str())
.with_object(self.object_path())
.with_object(object_path.as_str())
.with_duration(trace_started_at.elapsed())
.with_attr("state", state)
.with_attr("action", action.as_str())
@@ -889,3 +885,48 @@ pub(super) async fn contains_erasure_part_file(path: &str) -> Result<bool, Scann
Ok(false)
}
#[cfg(test)]
mod tests {
use super::*;
fn scanner_item_with_prefix(prefix: &str) -> ScannerItem {
ScannerItem {
path: String::new(),
bucket: "bucket".to_string(),
prefix: prefix.to_string(),
object_name: String::new(),
file_type: std::fs::metadata(std::env::temp_dir())
.expect("temp dir metadata should be readable")
.file_type(),
lifecycle: None,
object_lock: None,
replication: None,
heal_enabled: false,
heal_bitrot: false,
debug: false,
}
}
#[test]
fn transform_meta_dir_splits_parent_and_object_without_extra_components() {
let mut item = scanner_item_with_prefix("bucket/prefix/object");
item.transform_meta_dir();
assert_eq!(item.prefix, "bucket/prefix");
assert_eq!(item.object_name, "object");
assert_eq!(item.object_path(), "bucket/prefix/object");
}
#[test]
fn transform_meta_dir_moves_single_component_into_object_name() {
let mut item = scanner_item_with_prefix("object");
item.transform_meta_dir();
assert_eq!(item.prefix, "");
assert_eq!(item.object_name, "object");
assert_eq!(item.object_path(), "object");
}
}
crates/scanner/src/scanner_io/io_disk.rs
+11 -10
View File
@@ -42,7 +42,8 @@ impl ScannerIODisk for Disk {
return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
}
let data = match self.read_metadata(&item.bucket, &item.object_path()).await {
let metadata_object_path = item.object_path();
let data = match self.read_metadata(&item.bucket, &metadata_object_path).await {
Ok(data) => data,
Err(e) if DiskError::is_err_object_not_found(&e) || DiskError::is_err_version_not_found(&e) => {
return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
@@ -51,23 +52,23 @@ impl ScannerIODisk for Disk {
return Err(scanner_metadata_transient_error(
format!("failed to read metadata: {e}"),
&item.bucket,
&item.object_path(),
&metadata_object_path,
));
}
};
item.transform_meta_dir();
let object_path = item.object_path();
let meta = FileMeta::load(&data).map_err(|e| {
scanner_metadata_corrupt_error(format!("failed to load metadata: {e}"), &item.bucket, &item.object_path())
})?;
let fivs = match meta.get_file_info_versions(item.bucket.as_str(), item.object_path().as_str(), false) {
let meta = FileMeta::load(&data)
.map_err(|e| scanner_metadata_corrupt_error(format!("failed to load metadata: {e}"), &item.bucket, &object_path))?;
let fivs = match meta.get_file_info_versions(item.bucket.as_str(), object_path.as_str(), false) {
Ok(versions) => versions,
Err(e) => {
return Err(scanner_metadata_corrupt_error(
format!("failed to resolve file info versions: {e}"),
&item.bucket,
&item.object_path(),
&object_path,
));
}
};
@@ -91,17 +92,17 @@ impl ScannerIODisk for Disk {
VersioningConfiguration::default()
}
};
let versioned = versioning_config.versioned(&item.object_path());
let versioned = versioning_config.versioned(&object_path);
let object_infos = fivs
.versions
.iter()
.map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), item.object_path().as_str(), versioned))
.map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), object_path.as_str(), versioned))
.collect::<Vec<ObjectInfo>>();
let free_version_infos = fivs
.free_versions
.iter()
.map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), item.object_path().as_str(), versioned))
.map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), object_path.as_str(), versioned))
.collect::<Vec<ObjectInfo>>();
let mut size_summary = SizeSummary::default();