gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-27 15:37:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

ilm feature add

Browse Source
This commit is contained in:
likewu
2025-06-22 23:04:40 +08:00
parent 8452c11e9a
commit cc71f40a6d
93 changed files with 12534 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/filemeta/src/fileinfo.rs
+39 -8
View File
@@ -18,6 +18,10 @@ pub const BLOCK_SIZE_V2: usize = 1024 * 1024; // 1M
pub const NULL_VERSION_ID: &str = "null";
// pub const RUSTFS_ERASURE_UPGRADED: &str = "x-rustfs-internal-erasure-upgraded";
pub const TIER_FV_ID: &str = "tier-free-versionID";
pub const TIER_FV_MARKER: &str = "tier-free-marker";
pub const TIER_SKIP_FV_ID: &str = "tier-skip-fvid";
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
pub struct ObjectPartInfo {
pub etag: String,
@@ -147,11 +151,10 @@ pub struct FileInfo {
pub version_id: Option<Uuid>,
pub is_latest: bool,
pub deleted: bool,
// Transition related fields
pub transition_status: Option<String>,
pub transitioned_obj_name: Option<String>,
pub transition_tier: Option<String>,
pub transition_version_id: Option<String>,
pub transition_status: String,
pub transitioned_objname: String,
pub transition_tier: String,
pub transition_version_id: Option<Uuid>,
pub expire_restored: bool,
pub data_dir: Option<Uuid>,
pub mod_time: Option<OffsetDateTime>,
@@ -220,7 +223,11 @@ impl FileInfo {
}
pub fn get_etag(&self) -> Option<String> {
self.metadata.get("etag").cloned()
if let Some(meta) = self.metadata.get("etag") {
Some(meta.clone())
} else {
None
}
}
pub fn write_quorum(&self, quorum: usize) -> usize {
@@ -301,6 +308,30 @@ impl FileInfo {
self.metadata.insert(RUSTFS_HEALING.to_string(), "true".to_string());
}
pub fn set_tier_free_version_id(&mut self, version_id: &str) {
self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID), version_id.to_string());
}
pub fn tier_free_version_id(&self) -> String {
self.metadata[&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_ID)].clone()
}
pub fn set_tier_free_version(&mut self) {
self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER), "".to_string());
}
pub fn set_skip_tier_free_version(&mut self) {
self.metadata.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID), "".to_string());
}
pub fn skip_tier_free_version(&self) -> bool {
self.metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_SKIP_FV_ID))
}
pub fn tier_free_version(&self) -> bool {
self.metadata.contains_key(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TIER_FV_MARKER))
}
pub fn set_inline_data(&mut self) {
self.metadata
.insert(format!("{}inline-data", RESERVED_METADATA_PREFIX_LOWER).to_owned(), "true".to_owned());
@@ -319,7 +350,7 @@ impl FileInfo {
/// Check if the object is remote (transitioned to another tier)
pub fn is_remote(&self) -> bool {
!self.transition_tier.as_ref().is_none_or(|s| s.is_empty())
!self.transition_tier.is_empty()
}
/// Get the data directory for this object
@@ -375,7 +406,7 @@ impl FileInfo {
pub fn transition_info_equals(&self, other: &FileInfo) -> bool {
self.transition_status == other.transition_status
&& self.transition_tier == other.transition_tier
&& self.transitioned_obj_name == other.transitioned_obj_name
&& self.transitioned_objname == other.transitioned_objname
&& self.transition_version_id == other.transition_version_id
}
crates/filemeta/src/filemeta.rs
+134 -4
View File
@@ -7,6 +7,7 @@ use crate::headers::{
};
use byteorder::ByteOrder;
use rmp::Marker;
use s3s::header::X_AMZ_RESTORE;
use serde::{Deserialize, Serialize};
use std::cmp::Ordering;
use std::hash::Hasher;
@@ -36,6 +37,19 @@ const _XL_FLAG_INLINE_DATA: u8 = 1 << 2;
const META_DATA_READ_DEFAULT: usize = 4 << 10;
const MSGP_UINT32_SIZE: usize = 5;
pub const TRANSITION_COMPLETE: &str = "complete";
pub const TRANSITION_PENDING: &str = "pending";
pub const FREE_VERSION: &str = "free-version";
pub const TRANSITION_STATUS: &str = "transition-status";
pub const TRANSITIONED_OBJECTNAME: &str = "transitioned-object";
pub const TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
pub const TRANSITION_TIER: &str = "transition-tier";
const X_AMZ_RESTORE_EXPIRY_DAYS: &str = "X-Amz-Restore-Expiry-Days";
const X_AMZ_RESTORE_REQUEST_DATE: &str = "X-Amz-Restore-Request-Date";
// type ScanHeaderVersionFn = Box<dyn Fn(usize, &[u8], &[u8]) -> Result<()>>;
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
@@ -466,6 +480,40 @@ impl FileMeta {
Err(Error::other("add_version failed"))
}
pub fn add_version_filemata(&mut self, ver: FileMetaVersion) -> Result<()> {
let mod_time = ver.get_mod_time().unwrap().nanosecond();
if !ver.valid() {
return Err(Error::other("attempted to add invalid version"));
}
let encoded = ver.marshal_msg()?;
if self.versions.len()+1 > 100 {
return Err(Error::other("You've exceeded the limit on the number of versions you can create on this object"));
}
self.versions.push(FileMetaShallowVersion {
header: FileMetaVersionHeader {
mod_time: Some(OffsetDateTime::from_unix_timestamp(-1)?),
..Default::default()
},
..Default::default()
});
let len = self.versions.len();
for (i, existing) in self.versions.iter().enumerate() {
if existing.header.mod_time.unwrap().nanosecond() <= mod_time {
let vers = self.versions[i..len-1].to_vec();
self.versions[i+1..].clone_from_slice(vers.as_slice());
self.versions[i] = FileMetaShallowVersion {
header: ver.header(),
meta: encoded,
};
return Ok(());
}
}
Err(Error::other("addVersion: Internal error, unable to add version"))
}
// delete_version deletes version, returns data_dir
pub fn delete_version(&mut self, fi: &FileInfo) -> Result<Option<Uuid>> {
let mut ventry = FileMetaVersion::default();
@@ -501,6 +549,42 @@ impl FileMeta {
}
}
for (i, version) in self.versions.iter().enumerate() {
if version.header.version_type != VersionType::Object || version.header.version_id != fi.version_id {
continue;
}
let mut ver = self.get_idx(i)?;
if fi.expire_restored {
ver.object.as_mut().unwrap().remove_restore_hdrs();
let _ = self.set_idx(i, ver.clone());
} else if fi.transition_status == TRANSITION_COMPLETE {
ver.object.as_mut().unwrap().set_transition(fi);
ver.object.as_mut().unwrap().reset_inline_data();
self.set_idx(i, ver.clone())?;
} else {
let vers = self.versions[i+1..].to_vec();
self.versions.extend(vers.iter().cloned());
let (free_version, to_free) = ver.object.as_ref().unwrap().init_free_version(fi);
if to_free {
self.add_version_filemata(free_version)?;
}
}
if fi.deleted {
self.add_version_filemata(ventry)?;
}
if self.shared_data_dir_count(ver.object.as_ref().unwrap().version_id, ver.object.as_ref().unwrap().data_dir) > 0 {
return Ok(None);
}
return Ok(ver.object.as_ref().unwrap().data_dir);
}
if fi.deleted {
self.add_version_filemata(ventry)?;
}
Err(Error::FileVersionNotFound)
}
@@ -1842,10 +1926,17 @@ impl MetaObject {
}
}
/// Set transition metadata
pub fn set_transition(&mut self, _fi: &FileInfo) {
// Implementation for object lifecycle transitions
// This would handle storage class transitions
pub fn set_transition(&mut self, fi: &FileInfo) {
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS), fi.transition_status.as_bytes().to_vec());
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME), fi.transitioned_objname.as_bytes().to_vec());
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID), fi.transition_version_id.unwrap().as_bytes().to_vec());
self.meta_sys.insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER), fi.transition_tier.as_bytes().to_vec());
}
pub fn remove_restore_hdrs(&mut self) {
self.meta_user.remove(X_AMZ_RESTORE.as_str());
self.meta_user.remove(X_AMZ_RESTORE_EXPIRY_DAYS);
self.meta_user.remove(X_AMZ_RESTORE_REQUEST_DATE);
}
pub fn uses_data_dir(&self) -> bool {
@@ -1881,6 +1972,45 @@ impl MetaObject {
let bytes = hash.to_le_bytes();
[bytes[0], bytes[1], bytes[2], bytes[3]]
}
pub fn init_free_version(&self, fi: &FileInfo) -> (FileMetaVersion, bool) {
if fi.skip_tier_free_version() {
return (FileMetaVersion::default(), false);
}
if let Some(status) = self.meta_sys.get(&format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_STATUS)) {
if *status == TRANSITION_COMPLETE.as_bytes().to_vec() {
let vid = Uuid::parse_str(&fi.tier_free_version_id());
if let Err(err) = vid {
panic!("Invalid Tier Object delete marker versionId {} {}", fi.tier_free_version_id(), err.to_string());
}
let vid = vid.unwrap();
let mut free_entry = FileMetaVersion {
version_type: VersionType::Delete,
write_version: 0,
..Default::default()
};
free_entry.delete_marker = Some(MetaDeleteMarker {
version_id: Some(vid),
mod_time: self.mod_time,
meta_sys: Some(HashMap::<String, Vec<u8>>::new()),
});
free_entry.delete_marker.as_mut().unwrap().meta_sys.as_mut().unwrap().insert(format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, FREE_VERSION), vec![]);
let tier_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITION_TIER);
let tier_obj_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_OBJECTNAME);
let tier_obj_vid_key = format!("{}{}", RESERVED_METADATA_PREFIX_LOWER, TRANSITIONED_VERSION_ID);
let aa = [tier_key, tier_obj_key, tier_obj_vid_key];
for (k, v) in &self.meta_sys {
if aa.contains(&k) {
free_entry.delete_marker.as_mut().unwrap().meta_sys.as_mut().unwrap().insert(k.clone(), v.clone());
}
}
return (free_entry, true);
}
}
(FileMetaVersion::default(), false)
}
}
impl From<FileInfo> for MetaObject {
crates/filemeta/src/lib.rs
+1 -1
View File
@@ -1,5 +1,5 @@
mod error;
mod fileinfo;
pub mod fileinfo;
mod filemeta;
mod filemeta_inline;
pub mod headers;