gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-22 12:26:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Enhance Object Version Management and Replication Status Handling (#1413)

Browse Source
This commit is contained in:
weisd
2026-01-07 10:44:35 +08:00
committed by GitHub
parent 3ce99939a3
commit 359c9d2d26
8 changed files with 246 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/filemeta/src/fileinfo.rs
+5 -1
View File
@@ -505,6 +505,10 @@ impl FileInfo {
ReplicationStatusType::Empty
}
}
pub fn shard_file_size(&self, total_length: i64) -> i64 {
self.erasure.shard_file_size(total_length)
}
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
@@ -590,7 +594,7 @@ impl RestoreStatusOps for RestoreStatus {
}
}
fn parse_restore_obj_status(restore_hdr: &str) -> Result<RestoreStatus> {
pub fn parse_restore_obj_status(restore_hdr: &str) -> Result<RestoreStatus> {
let tokens: Vec<&str> = restore_hdr.splitn(2, ",").collect();
let progress_tokens: Vec<&str> = tokens[0].splitn(2, "=").collect();
if progress_tokens.len() != 2 {
crates/filemeta/src/filemeta.rs
+83 -8
View File
@@ -14,7 +14,8 @@
use crate::{
ErasureAlgo, ErasureInfo, Error, FileInfo, FileInfoVersions, InlineData, ObjectPartInfo, RawFileInfo, ReplicationState,
ReplicationStatusType, Result, VersionPurgeStatusType, replication_statuses_map, version_purge_statuses_map,
ReplicationStatusType, Result, TIER_FV_ID, TIER_FV_MARKER, VersionPurgeStatusType, replication_statuses_map,
version_purge_statuses_map,
};
use byteorder::ByteOrder;
use bytes::Bytes;
@@ -909,6 +910,7 @@ impl FileMeta {
path: &str,
version_id: &str,
read_data: bool,
include_free_versions: bool,
all_parts: bool,
) -> Result<FileInfo> {
let vid = {
@@ -921,11 +923,35 @@ impl FileMeta {
let mut is_latest = true;
let mut succ_mod_time = None;
let mut non_free_versions = self.versions.len();
let mut found = false;
let mut found_free_version = None;
let mut found_fi = None;
for ver in self.versions.iter() {
let header = &ver.header;
// TODO: freeVersion
if header.free_version() {
non_free_versions -= 1;
if include_free_versions && found_free_version.is_none() {
let mut found_free_fi = FileMetaVersion::default();
if found_free_fi.unmarshal_msg(&ver.meta).is_ok() && found_free_fi.version_type != VersionType::Invalid {
let mut free_fi = found_free_fi.into_fileinfo(volume, path, all_parts);
free_fi.is_latest = true;
found_free_version = Some(free_fi);
}
}
if header.version_id != Some(vid) {
continue;
}
}
if found {
continue;
}
if !version_id.is_empty() && header.version_id != Some(vid) {
is_latest = false;
@@ -933,6 +959,8 @@ impl FileMeta {
continue;
}
found = true;
let mut fi = ver.into_fileinfo(volume, path, all_parts)?;
fi.is_latest = is_latest;
@@ -947,7 +975,25 @@ impl FileMeta {
.map(bytes::Bytes::from);
}
fi.num_versions = self.versions.len();
found_fi = Some(fi);
}
if !found {
if version_id.is_empty() {
if include_free_versions
&& non_free_versions == 0
&& let Some(free_version) = found_free_version
{
return Ok(free_version);
}
return Err(Error::FileNotFound);
} else {
return Err(Error::FileVersionNotFound);
}
}
if let Some(mut fi) = found_fi {
fi.num_versions = non_free_versions;
return Ok(fi);
}
@@ -1767,14 +1813,27 @@ impl MetaObject {
metadata.insert(k.to_owned(), v.to_owned());
}
let tier_fvidkey = format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_ID}").to_lowercase();
let tier_fvmarker_key = format!("{RESERVED_METADATA_PREFIX_LOWER}{TIER_FV_MARKER}").to_lowercase();
for (k, v) in &self.meta_sys {
if k == AMZ_STORAGE_CLASS && v == b"STANDARD" {
let lower_k = k.to_lowercase();
if lower_k == tier_fvidkey || lower_k == tier_fvmarker_key {
continue;
}
if lower_k == VERSION_PURGE_STATUS_KEY.to_lowercase() {
continue;
}
if lower_k == AMZ_STORAGE_CLASS.to_lowercase() && v == b"STANDARD" {
continue;
}
if k.starts_with(RESERVED_METADATA_PREFIX)
|| k.starts_with(RESERVED_METADATA_PREFIX_LOWER)
|| k == VERSION_PURGE_STATUS_KEY
|| lower_k == VERSION_PURGE_STATUS_KEY.to_lowercase()
{
metadata.insert(k.to_owned(), String::from_utf8(v.to_owned()).unwrap_or_default());
}
@@ -2511,15 +2570,31 @@ pub fn merge_file_meta_versions(
merged
}
pub async fn file_info_from_raw(ri: RawFileInfo, bucket: &str, object: &str, read_data: bool) -> Result<FileInfo> {
get_file_info(&ri.buf, bucket, object, "", FileInfoOpts { data: read_data }).await
pub fn file_info_from_raw(
ri: RawFileInfo,
bucket: &str,
object: &str,
read_data: bool,
include_free_versions: bool,
) -> Result<FileInfo> {
get_file_info(
&ri.buf,
bucket,
object,
"",
FileInfoOpts {
data: read_data,
include_free_versions,
},
)
}
pub struct FileInfoOpts {
pub data: bool,
pub include_free_versions: bool,
}
pub async fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opts: FileInfoOpts) -> Result<FileInfo> {
pub fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opts: FileInfoOpts) -> Result<FileInfo> {
let vid = {
if version_id.is_empty() {
None
@@ -2541,7 +2616,7 @@ pub async fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &st
});
}
let fi = meta.into_fileinfo(volume, path, version_id, opts.data, true)?;
let fi = meta.into_fileinfo(volume, path, version_id, opts.data, opts.include_free_versions, true)?;
Ok(fi)
}
crates/filemeta/src/metacache.rs
+16 -8
View File
@@ -12,7 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{Error, FileInfo, FileInfoVersions, FileMeta, FileMetaShallowVersion, Result, VersionType, merge_file_meta_versions};
use crate::{
Error, FileInfo, FileInfoOpts, FileInfoVersions, FileMeta, FileMetaShallowVersion, Result, VersionType, get_file_info,
merge_file_meta_versions,
};
use rmp::Marker;
use serde::{Deserialize, Serialize};
use std::cmp::Ordering;
@@ -141,8 +144,7 @@ impl MetaCacheEntry {
});
}
if self.cached.is_some() {
let fm = self.cached.as_ref().unwrap();
if let Some(fm) = &self.cached {
if fm.versions.is_empty() {
return Ok(FileInfo {
volume: bucket.to_owned(),
@@ -154,14 +156,20 @@ impl MetaCacheEntry {
});
}
let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?;
let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false, true)?;
return Ok(fi);
}
let mut fm = FileMeta::new();
fm.unmarshal_msg(&self.metadata)?;
let fi = fm.into_fileinfo(bucket, self.name.as_str(), "", false, false)?;
Ok(fi)
get_file_info(
&self.metadata,
bucket,
self.name.as_str(),
"",
FileInfoOpts {
data: false,
include_free_versions: false,
},
)
}
pub fn file_info_versions(&self, bucket: &str) -> Result<FileInfoVersions> {