gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-01 01:38:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(tiering): persist opaque remote versions

Browse Source
This commit is contained in:
马登山
2026-07-28 13:37:43 +08:00
parent 1e10d752b9
commit 837286f959
7 changed files with 158 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/filemeta/src/fileinfo.rs
+23
View File
@@ -230,6 +230,8 @@ pub struct FileInfo {
pub transitioned_objname: String,
pub transition_tier: String,
pub transition_version_id: Option<Uuid>,
#[serde(default)]
pub transition_version: Option<String>,
pub expire_restored: bool,
pub data_dir: Option<Uuid>,
pub mod_time: Option<OffsetDateTime>,
@@ -455,6 +457,10 @@ impl FileInfo {
if self.mod_time.is_none_or(|mod_time| mod_time <= OffsetDateTime::UNIX_EPOCH)
|| (!allow_nil_version_id && self.version_id.is_some_and(|version_id| version_id.is_nil()))
|| self.transition_version_id.is_some_and(|version_id| version_id.is_nil())
|| self
.transition_version
.as_ref()
.is_some_and(|version_id| version_id.is_empty())
|| self.size != 0
|| self.data_dir.is_some()
|| self.mode.is_some()
@@ -488,6 +494,7 @@ impl FileInfo {
|| !self.transitioned_objname.is_empty()
|| !self.transition_tier.is_empty()
|| self.transition_version_id.is_some()
|| self.transition_version.is_some()
|| self.expire_restored
|| self.size != 0
|| self.data_dir.is_some()
@@ -532,6 +539,11 @@ impl FileInfo {
/// return `None`.
pub fn validate(&self, mode: ValidationMode) -> Result<Option<ValidatedErasureLayout>> {
self.validate_collection_bounds()?;
if let (Some(version), Some(version_id)) = (&self.transition_version, self.transition_version_id)
&& Uuid::parse_str(version).ok() != Some(version_id)
{
return Err(Error::FileCorrupt);
}
let erasure_layout = match mode {
ValidationMode::RequireErasure => Some(self.validate_erasure_geometry()?),
@@ -824,6 +836,7 @@ impl FileInfo {
&& self.transition_tier == other.transition_tier
&& self.transitioned_objname == other.transitioned_objname
&& self.transition_version_id == other.transition_version_id
&& self.transition_version == other.transition_version
}
/// Check if metadata maps are equal
@@ -1328,6 +1341,15 @@ mod tests {
assert_file_corrupt(&fi, ValidationMode::DeleteOnly);
}
#[test]
fn metadata_read_validation_rejects_conflicting_transition_versions() {
let mut fi = one_shard_validation_fileinfo(1);
fi.transition_version_id = Some(Uuid::new_v4());
fi.transition_version = Some(Uuid::new_v4().to_string());
assert_file_corrupt(&fi, ValidationMode::RequireErasure);
}
#[test]
fn metadata_read_validation_requires_canonical_delete_marker_shape() {
let marker = FileInfo {
@@ -1699,6 +1721,7 @@ mod tests {
transitioned_objname,
transition_tier,
transition_version_id,
transition_version: transition_version_id.map(|version_id| version_id.to_string()),
expire_restored,
data_dir,
mod_time,
crates/filemeta/src/filemeta/version.rs
+88 -37
View File
@@ -43,6 +43,7 @@ const MSGPACK_FIXEXT8: u8 = 0xd7;
const MSGPACK_TIME_EXT_LEGACY: i8 = 5;
const MSGPACK_TIME_EXT_OFFICIAL: i8 = -1;
const MSGPACK_TIME_LEN: u8 = 12;
const MAX_TRANSITION_VERSION_LEN: usize = 1024;
/// Sentinel signature returned when a version has no computable body (invalid /
/// missing inner object). Mirrors MinIO's `signatureErr` so such versions never
@@ -251,23 +252,38 @@ fn parse_legacy_uuid_bytes(bytes: &[u8], field: &str) -> Result<Option<Uuid>> {
/// Decode a stored transitioned-version-id from a version's `meta_sys`.
///
/// RustFS writes it as 16 raw UUID bytes; MinIO-migrated tiered objects store
/// the remote tier's version id as a UUID *string*. Accept both, and treat any
/// absent / nil / otherwise-unparseable value as "no tier version" (matching the
/// tolerant pre-hardening behavior) rather than failing the whole object read —
/// a malformed tier id must not make an otherwise-readable object unreadable.
fn transitioned_version_id_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Option<Uuid> {
/// Legacy RustFS writes used 16 raw UUID bytes. New writes and MinIO-migrated
/// records use the provider's exact UTF-8 version text. Empty, nil UUID, and
/// malformed bytes are not usable remote versions.
fn transitioned_version_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Option<String> {
let value = get_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID)?;
if value.is_empty() {
return None;
}
if let Ok(id) = Uuid::from_slice(&value) {
return (!id.is_nil()).then_some(id);
return (!id.is_nil()).then(|| id.to_string());
}
std::str::from_utf8(&value)
.ok()
.and_then(|s| Uuid::parse_str(s.trim()).ok())
.filter(|id| !id.is_nil())
let value = String::from_utf8(value).ok()?;
if value.is_empty()
|| value.len() > MAX_TRANSITION_VERSION_LEN
|| value.chars().any(char::is_control)
|| Uuid::parse_str(&value).is_ok_and(|id| id.is_nil())
{
None
} else {
Some(value)
}
}
fn legacy_transitioned_version_id_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Option<Uuid> {
transitioned_version_from_meta_sys(meta_sys).and_then(|value| Uuid::parse_str(&value).ok())
}
fn transitioned_version_bytes(fi: &FileInfo) -> Option<Vec<u8>> {
fi.transition_version
.as_ref()
.map(|version| version.as_bytes().to_vec())
.or_else(|| fi.transition_version_id.map(|version_id| version_id.as_bytes().to_vec()))
}
fn parse_legacy_erasure_algo(value: &str) -> ErasureAlgo {
@@ -2398,7 +2414,8 @@ impl MetaObject {
let transitioned_objname = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_OBJECTNAME)
.map(|v| String::from_utf8_lossy(&v).to_string())
.unwrap_or_default();
let transition_version_id = transitioned_version_id_from_meta_sys(&self.meta_sys);
let transition_version = transitioned_version_from_meta_sys(&self.meta_sys);
let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
let transition_tier = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER)
.map(|v| String::from_utf8_lossy(&v).to_string())
.unwrap_or_default();
@@ -2419,6 +2436,7 @@ impl MetaObject {
transition_status,
transitioned_objname,
transition_version_id,
transition_version,
transition_tier,
..Default::default()
})
@@ -2431,12 +2449,8 @@ impl MetaObject {
SUFFIX_TRANSITIONED_OBJECTNAME,
fi.transitioned_objname.as_bytes().to_vec(),
);
if let Some(transition_version_id) = fi.transition_version_id.as_ref() {
insert_bytes(
&mut self.meta_sys,
SUFFIX_TRANSITIONED_VERSION_ID,
transition_version_id.as_bytes().to_vec(),
);
if let Some(transition_version) = transitioned_version_bytes(fi) {
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
}
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec());
if let Some(destination_id) = get_str(&fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID) {
@@ -2562,8 +2576,8 @@ impl From<FileInfo> for MetaObject {
);
}
if let Some(vid) = &value.transition_version_id {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, vid.as_bytes().to_vec());
if let Some(transition_version) = transitioned_version_bytes(&value) {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
}
if !value.transition_tier.is_empty() {
@@ -2706,7 +2720,8 @@ impl MetaDeleteMarker {
.map(|v| String::from_utf8_lossy(&v).to_string())
.unwrap_or_default();
fi.transition_version_id = transitioned_version_id_from_meta_sys(&self.meta_sys);
fi.transition_version = transitioned_version_from_meta_sys(&self.meta_sys);
fi.transition_version_id = legacy_transitioned_version_id_from_meta_sys(&self.meta_sys);
}
fi
@@ -2859,8 +2874,8 @@ impl From<FileInfo> for MetaDeleteMarker {
value.transitioned_objname.as_bytes().to_vec(),
);
}
if let Some(version_id) = value.transition_version_id {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, version_id.as_bytes().to_vec());
if let Some(transition_version) = transitioned_version_bytes(&value) {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
}
if !value.transition_tier.is_empty() {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec());
@@ -3412,7 +3427,7 @@ mod tests {
.insert("x-rustfs-internal-healing".to_string(), "true".to_string());
marker.metadata.insert("content-type".to_string(), "text/plain".to_string());
let remote_version_id = Uuid::new_v4();
marker.transition_version_id = Some(remote_version_id);
marker.transition_version = Some(remote_version_id.to_string());
let converted = MetaDeleteMarker::from(marker);
@@ -3420,7 +3435,19 @@ mod tests {
assert_eq!(converted.meta_sys.get("x-minio-internal-purgestatus"), Some(&b"pending".to_vec()));
assert_eq!(
get_bytes(&converted.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID),
Some(remote_version_id.as_bytes().to_vec())
Some(remote_version_id.to_string().into_bytes())
);
assert_eq!(
converted
.meta_sys
.get(&format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_ID}")),
Some(&remote_version_id.to_string().into_bytes())
);
assert_eq!(
converted
.meta_sys
.get(&format!("{}{SUFFIX_TRANSITIONED_VERSION_ID}", rustfs_utils::http::MINIO_INTERNAL_PREFIX)),
Some(&remote_version_id.to_string().into_bytes())
);
assert!(!converted.meta_sys.contains_key("x-rustfs-internal-healing"));
assert!(!converted.meta_sys.contains_key("content-type"));
@@ -4097,19 +4124,42 @@ mod tests {
.into_fileinfo("b", "k", false)
.expect("into_fileinfo");
assert_eq!(fi.transition_version_id, Some(id));
assert_eq!(fi.transition_version, Some(id.to_string()));
}
#[test]
fn meta_object_transition_version_id_unparseable_stays_readable_as_none() {
// A non-UUID / non-16-byte tier version id must NOT make the object
// unreadable; it is tolerated as "no tier version" (compat with
// pre-hardening behavior and foreign/edge metadata).
fn meta_object_transition_version_id_opaque_text_is_preserved() {
let mut sys = HashMap::new();
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"not-a-uuid".to_vec());
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"opaque-generation-42".to_vec());
let fi = make_meta_object_with_sys(sys)
.into_fileinfo("b", "k", false)
.expect("unparseable transition version id must not fail the object read");
.expect("opaque transition version id must decode");
assert_eq!(fi.transition_version_id, None);
assert_eq!(fi.transition_version.as_deref(), Some("opaque-generation-42"));
}
#[test]
fn meta_object_transition_version_id_invalid_utf8_yields_none() {
let mut sys = HashMap::new();
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, vec![0xff]);
let fi = make_meta_object_with_sys(sys)
.into_fileinfo("b", "k", false)
.expect("invalid transition version bytes must not fail the object read");
assert_eq!(fi.transition_version_id, None);
assert_eq!(fi.transition_version, None);
}
#[test]
fn meta_object_transition_version_id_unsafe_text_yields_none() {
for value in [b"opaque\0version".to_vec(), vec![b'x'; MAX_TRANSITION_VERSION_LEN + 1]] {
let mut sys = HashMap::new();
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, value);
let fi = make_meta_object_with_sys(sys)
.into_fileinfo("b", "k", false)
.expect("unsafe transition version text must not fail the object read");
assert_eq!(fi.transition_version_id, None);
assert_eq!(fi.transition_version, None);
}
}
#[test]
@@ -4123,6 +4173,7 @@ mod tests {
.into_fileinfo("b", "k", false)
.expect("string-form transition version id must decode");
assert_eq!(fi.transition_version_id, Some(id));
assert_eq!(fi.transition_version, Some(id.to_string()));
}
#[test]
@@ -4152,16 +4203,14 @@ mod tests {
}
.into_fileinfo("b", "k", false);
assert_eq!(fi.transition_version_id, Some(id));
assert_eq!(fi.transition_version, Some(id.to_string()));
}
#[test]
fn delete_marker_free_version_transition_version_id_unparseable_stays_readable() {
// A malformed tier version id must not make a free-version record corrupt:
// it decodes to None and stays readable. Otherwise free-version expiry
// fails and the remote-tier object leaks.
fn delete_marker_free_version_transition_version_id_opaque_text_is_preserved() {
let mut sys = HashMap::new();
insert_bytes(&mut sys, SUFFIX_FREE_VERSION, vec![]);
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"not-a-uuid".to_vec());
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, b"opaque-generation-42".to_vec());
insert_bytes(&mut sys, SUFFIX_TRANSITION_TIER, b"WARM".to_vec());
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_OBJECTNAME, b"remote-object".to_vec());
let fi = MetaDeleteMarker {
@@ -4172,8 +4221,9 @@ mod tests {
.into_fileinfo("b", "k", false);
assert_eq!(fi.transition_version_id, None);
assert_eq!(fi.transition_version.as_deref(), Some("opaque-generation-42"));
fi.validate_for_metadata_read()
.expect("free-version record with an unparseable tier id must remain readable");
.expect("free-version record with an opaque tier id must remain readable");
}
#[test]
@@ -4193,6 +4243,7 @@ mod tests {
.into_fileinfo("b", "k", false);
assert_eq!(fi.transition_version_id, Some(id));
assert_eq!(fi.transition_version, Some(id.to_string()));
}
#[test]