gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-21 11:56:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into houseme/issue1856-admission-policy-observe

Browse Source
This commit is contained in:
houseme
2026-08-21 02:34:59 +08:00
committed by GitHub
parent 63cbe45bd5 762919b1ba
commit bfd7deb269
5 changed files with 109 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/object_api/types.rs
+23 -2
View File
@@ -719,14 +719,23 @@ impl ObjectInfo {
}
pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
let name = decode_dir_object(object);
let mut version_id = fi.version_id;
if versioned && version_id.is_none() {
version_id = Some(Uuid::nil())
}
Self::from_file_info_with_version_id(fi, bucket, object, version_id)
}
pub(crate) fn from_file_info_with_version_id(
fi: &FileInfo,
bucket: &str,
object: &str,
version_id: Option<Uuid>,
) -> ObjectInfo {
let name = decode_dir_object(object);
// etag
let (content_type, content_encoding, etag) = {
let content_type = fi.metadata.get("content-type").cloned();
@@ -1640,6 +1649,18 @@ mod tests {
assert_eq!(info.replication_decision, "arn=true;false;arn:replication::1:dest;rule-id");
}
#[test]
fn from_file_info_with_version_id_keeps_normalized_absent_version() {
let fi = FileInfo {
version_id: Some(Uuid::new_v4()),
..Default::default()
};
let info = ObjectInfo::from_file_info_with_version_id(&fi, "bucket", "object", None);
assert_eq!(info.version_id, None, "a normalized absent version must not be rewritten to nil");
}
#[test]
fn from_file_info_reports_effective_storage_class_for_legacy_metadata() {
for legacy_label in [
crates/ecstore/src/set_disk/ops/heal_walk.rs
+2 -16
View File
@@ -124,14 +124,7 @@ impl HealWalkCollector {
for fi in fiv.versions.iter().chain(fiv.free_versions.iter()) {
let version_uuid = fi.version_id.filter(|version_id| !version_id.is_nil());
let lifecycle_object_info = if self.include_lifecycle_object_info {
let mut lifecycle_fi = fi.clone();
lifecycle_fi.version_id = version_uuid;
Some(ObjectInfo::from_file_info(
&lifecycle_fi,
&self.bucket,
&entry.name,
version_uuid.is_some(),
))
Some(ObjectInfo::from_file_info_with_version_id(fi, &self.bucket, &entry.name, version_uuid))
} else {
None
};
@@ -198,14 +191,7 @@ impl HealWalkCollector {
let vid = version_uuid.map(|u| u.to_string());
if seen.insert(vid.clone()) {
let lifecycle_object_info = if self.include_lifecycle_object_info {
let mut lifecycle_fi = fi.clone();
lifecycle_fi.version_id = version_uuid;
Some(ObjectInfo::from_file_info(
&lifecycle_fi,
&self.bucket,
&entry.name,
version_uuid.is_some(),
))
Some(ObjectInfo::from_file_info_with_version_id(fi, &self.bucket, &entry.name, version_uuid))
} else {
None
};
crates/heal/src/heal/storage.rs
+16 -7
View File
@@ -1202,13 +1202,22 @@ impl HealStorageAPI for ECStoreHealStorage {
let version_id = obj.version_id.map(|u| u.to_string());
let mod_time_unix_nanos = obj.mod_time.map(|mod_time| mod_time.unix_timestamp_nanos());
let is_delete_marker = obj.delete_marker;
let lifecycle_object_info = include_lifecycle_object_info.then(|| obj.clone());
HealListItem {
name: obj.name,
version_id,
mod_time_unix_nanos,
lifecycle_object_info,
is_delete_marker,
if include_lifecycle_object_info {
HealListItem {
name: obj.name.clone(),
version_id,
mod_time_unix_nanos,
lifecycle_object_info: Some(obj),
is_delete_marker,
}
} else {
HealListItem {
name: obj.name,
version_id,
mod_time_unix_nanos,
lifecycle_object_info: None,
is_delete_marker,
}
}
})
.collect();
crates/scanner/src/scanner_folder/item_actions.rs
+57 -16
View File
@@ -274,18 +274,13 @@ impl ScannerItem {
/// Transform meta directory by splitting prefix and extracting object name
/// This converts a directory path like "bucket/dir1/dir2/file" to prefix="bucket/dir1/dir2" and object_name="file"
pub fn transform_meta_dir(&mut self) {
let prefix = self.prefix.clone(); // Clone to avoid borrow checker issues
let split: Vec<&str> = prefix.split(SLASH_SEPARATOR).collect();
if split.len() > 1 {
let prefix_parts: Vec<&str> = split[..split.len() - 1].to_vec();
self.prefix = path_join_buf(&prefix_parts);
let prefix = std::mem::take(&mut self.prefix);
if let Some((parent, object_name)) = prefix.rsplit_once(SLASH_SEPARATOR) {
self.prefix = path_join_buf(&[parent]);
self.object_name = object_name.to_string();
} else {
self.prefix = String::new();
self.object_name = prefix;
}
// Object name is the last element
self.object_name = split.last().unwrap_or(&"").to_string();
}
pub(super) fn metadata_object_path(&self) -> String {
@@ -301,13 +296,14 @@ impl ScannerItem {
versioning_config: VersioningConfiguration,
size_summary: &mut SizeSummary,
) {
let object_path = self.object_path();
if object_infos.is_empty() {
debug!(
target: "rustfs::scanner::folder",
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object_path = %self.object_path(),
object_path = %object_path,
state = "no_object_versions",
"Scanner lifecycle action skipped"
);
@@ -318,7 +314,7 @@ impl ScannerItem {
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object_path = %self.object_path(),
object_path = %object_path,
state = "started",
"Scanner lifecycle evaluation started"
);
@@ -360,7 +356,7 @@ impl ScannerItem {
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object_path = %self.object_path(),
object_path = %object_path,
state = "no_lifecycle_config",
"Scanner lifecycle action finished without lifecycle rules"
);
@@ -385,7 +381,7 @@ impl ScannerItem {
event = EVENT_SCANNER_LIFECYCLE_ACTION,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
object_path = %self.object_path(),
object_path = %object_path,
state = "evaluate_failed",
error = %e,
"Scanner lifecycle action evaluation failed"
@@ -502,7 +498,7 @@ impl ScannerItem {
emit_scanner_ilm_action_trace(&self.bucket, &oi.name, event.action, 1, queued, trace_started_at);
if record_scanner_ilm_action_if_queued(global_metrics(), event.action, 1, queued) {
done_ilm(1)();
if !versioning_config.prefix_enabled(&self.object_path()) && event.action == IlmAction::DeleteAction {
if !versioning_config.prefix_enabled(&object_path) && event.action == IlmAction::DeleteAction {
remaining_versions -= 1;
size = 0;
}
@@ -570,7 +566,7 @@ impl ScannerItem {
trace_emit(|| {
TraceEvent::new(TraceKind::Scanner, TraceFunc::ScannerIlmAction)
.with_bucket(self.bucket.as_str())
.with_object(self.object_path())
.with_object(object_path.as_str())
.with_duration(trace_started_at.elapsed())
.with_attr("state", state)
.with_attr("action", action.as_str())
@@ -889,3 +885,48 @@ pub(super) async fn contains_erasure_part_file(path: &str) -> Result<bool, Scann
Ok(false)
}
#[cfg(test)]
mod tests {
use super::*;
fn scanner_item_with_prefix(prefix: &str) -> ScannerItem {
ScannerItem {
path: String::new(),
bucket: "bucket".to_string(),
prefix: prefix.to_string(),
object_name: String::new(),
file_type: std::fs::metadata(std::env::temp_dir())
.expect("temp dir metadata should be readable")
.file_type(),
lifecycle: None,
object_lock: None,
replication: None,
heal_enabled: false,
heal_bitrot: false,
debug: false,
}
}
#[test]
fn transform_meta_dir_splits_parent_and_object_without_extra_components() {
let mut item = scanner_item_with_prefix("bucket/prefix/object");
item.transform_meta_dir();
assert_eq!(item.prefix, "bucket/prefix");
assert_eq!(item.object_name, "object");
assert_eq!(item.object_path(), "bucket/prefix/object");
}
#[test]
fn transform_meta_dir_moves_single_component_into_object_name() {
let mut item = scanner_item_with_prefix("object");
item.transform_meta_dir();
assert_eq!(item.prefix, "");
assert_eq!(item.object_name, "object");
assert_eq!(item.object_path(), "object");
}
}
crates/scanner/src/scanner_io/io_disk.rs
+11 -10
View File
@@ -42,7 +42,8 @@ impl ScannerIODisk for Disk {
return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
}
let data = match self.read_metadata(&item.bucket, &item.object_path()).await {
let metadata_object_path = item.object_path();
let data = match self.read_metadata(&item.bucket, &metadata_object_path).await {
Ok(data) => data,
Err(e) if DiskError::is_err_object_not_found(&e) || DiskError::is_err_version_not_found(&e) => {
return Err(StorageError::other(SCANNER_SKIP_FILE_ERROR.to_string()));
@@ -51,23 +52,23 @@ impl ScannerIODisk for Disk {
return Err(scanner_metadata_transient_error(
format!("failed to read metadata: {e}"),
&item.bucket,
&item.object_path(),
&metadata_object_path,
));
}
};
item.transform_meta_dir();
let object_path = item.object_path();
let meta = FileMeta::load(&data).map_err(|e| {
scanner_metadata_corrupt_error(format!("failed to load metadata: {e}"), &item.bucket, &item.object_path())
})?;
let fivs = match meta.get_file_info_versions(item.bucket.as_str(), item.object_path().as_str(), false) {
let meta = FileMeta::load(&data)
.map_err(|e| scanner_metadata_corrupt_error(format!("failed to load metadata: {e}"), &item.bucket, &object_path))?;
let fivs = match meta.get_file_info_versions(item.bucket.as_str(), object_path.as_str(), false) {
Ok(versions) => versions,
Err(e) => {
return Err(scanner_metadata_corrupt_error(
format!("failed to resolve file info versions: {e}"),
&item.bucket,
&item.object_path(),
&object_path,
));
}
};
@@ -91,17 +92,17 @@ impl ScannerIODisk for Disk {
VersioningConfiguration::default()
}
};
let versioned = versioning_config.versioned(&item.object_path());
let versioned = versioning_config.versioned(&object_path);
let object_infos = fivs
.versions
.iter()
.map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), item.object_path().as_str(), versioned))
.map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), object_path.as_str(), versioned))
.collect::<Vec<ObjectInfo>>();
let free_version_infos = fivs
.free_versions
.iter()
.map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), item.object_path().as_str(), versioned))
.map(|v| ObjectInfo::from_file_info(v, item.bucket.as_str(), object_path.as_str(), versioned))
.collect::<Vec<ObjectInfo>>();
let mut size_summary = SizeSummary::default();