gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-21 20:06:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(tiering): gate remote version state safely

Browse Source
This commit is contained in:
马登山
2026-07-28 15:10:53 +08:00
parent a6695f6d30
commit 821e056f70
13 changed files with 515 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/filemeta/src/fileinfo.rs
+32
View File
@@ -219,6 +219,16 @@ impl ErasureInfo {
}
// #[derive(Debug, Clone)]
#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Copy, Default)]
#[serde(rename_all = "kebab-case")]
pub enum TransitionVersionState {
#[default]
Unknown,
KnownDisabled,
SuspendedNull,
Exact,
}
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
pub struct FileInfo {
pub volume: String,
@@ -232,6 +242,8 @@ pub struct FileInfo {
pub transition_version_id: Option<Uuid>,
#[serde(default)]
pub transition_version: Option<String>,
#[serde(default)]
pub transition_version_state: TransitionVersionState,
pub expire_restored: bool,
pub data_dir: Option<Uuid>,
pub mod_time: Option<OffsetDateTime>,
@@ -544,6 +556,20 @@ impl FileInfo {
{
return Err(Error::FileCorrupt);
}
let transition_state_valid = match self.transition_version_state {
TransitionVersionState::Unknown => true,
TransitionVersionState::KnownDisabled => self.transition_version.is_none() && self.transition_version_id.is_none(),
TransitionVersionState::SuspendedNull => {
self.transition_version.as_deref() == Some("null") && self.transition_version_id.is_none()
}
TransitionVersionState::Exact => self
.transition_version
.as_deref()
.is_some_and(|version| version != "null" && !version.is_empty()),
};
if !transition_state_valid {
return Err(Error::FileCorrupt);
}
let erasure_layout = match mode {
ValidationMode::RequireErasure => Some(self.validate_erasure_geometry()?),
@@ -837,6 +863,7 @@ impl FileInfo {
&& self.transitioned_objname == other.transitioned_objname
&& self.transition_version_id == other.transition_version_id
&& self.transition_version == other.transition_version
&& self.transition_version_state == other.transition_version_state
}
/// Check if metadata maps are equal
@@ -1722,6 +1749,11 @@ mod tests {
transition_tier,
transition_version_id,
transition_version: transition_version_id.map(|version_id| version_id.to_string()),
transition_version_state: if transition_version_id.is_some() {
TransitionVersionState::Exact
} else {
TransitionVersionState::Unknown
},
expire_restored,
data_dir,
mod_time,
crates/filemeta/src/filemeta/version.rs
+138 -13
View File
@@ -26,13 +26,14 @@ use super::msgp_decode::{
PrependByteReader, prealloc_hint, read_exact_vec, read_nil_or_array_len, read_nil_or_map_len, skip_msgp_value,
};
use super::*;
use crate::ChecksumInfo;
use crate::{ChecksumInfo, TransitionVersionState};
use rustfs_utils::HashAlgorithm;
use rustfs_utils::http::{
RUSTFS_INTERNAL_PREFIX, SUFFIX_CRC, SUFFIX_FREE_VERSION, SUFFIX_INLINE_DATA, SUFFIX_PURGESTATUS, SUFFIX_TIER_FV_ID,
SUFFIX_TIER_FV_MARKER, SUFFIX_TRANSITION_STATUS, SUFFIX_TRANSITION_TIER, SUFFIX_TRANSITION_TIER_DESTINATION_ID,
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, contains_key_bytes, get_bytes, get_consistent_bytes, get_str,
has_internal_suffix, insert_bytes, is_internal_key, remove_bytes, strip_internal_prefix,
SUFFIX_TRANSITIONED_OBJECTNAME, SUFFIX_TRANSITIONED_VERSION_ID, SUFFIX_TRANSITIONED_VERSION_STATE, contains_key_bytes,
get_bytes, get_consistent_bytes, get_str, has_internal_suffix, insert_bytes, is_internal_key, remove_bytes,
strip_internal_prefix,
};
const MSGPACK_EXT8: u8 = 0xc7;
@@ -255,28 +256,83 @@ fn parse_legacy_uuid_bytes(bytes: &[u8], field: &str) -> Result<Option<Uuid>> {
/// Legacy RustFS writes used 16 raw UUID bytes. New writes and MinIO-migrated
/// records use the provider's exact UTF-8 version text. Empty, nil UUID, and
/// malformed bytes are not usable remote versions.
fn transitioned_version_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Option<String> {
let value = get_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID)?;
fn transitioned_version_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Result<Option<String>> {
if !contains_key_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) {
return Ok(None);
}
let Some(value) = get_consistent_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_ID) else {
return Ok(None);
};
let value = value.to_vec();
if value.is_empty() {
return None;
return Ok(None);
}
if let Ok(id) = Uuid::from_slice(&value) {
return (!id.is_nil()).then(|| id.to_string());
return Ok((!id.is_nil()).then(|| id.to_string()));
}
let value = String::from_utf8(value).ok()?;
let Ok(value) = String::from_utf8(value) else {
return Ok(None);
};
if value.is_empty()
|| value.len() > MAX_TRANSITION_VERSION_LEN
|| value.chars().any(char::is_control)
|| Uuid::parse_str(&value).is_ok_and(|id| id.is_nil())
{
None
Ok(None)
} else {
Some(value)
Ok(Some(value))
}
}
fn transition_version_state_from_meta_sys(
meta_sys: &HashMap<String, Vec<u8>>,
version: Option<&str>,
) -> Result<TransitionVersionState> {
if !contains_key_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE) {
return Ok(TransitionVersionState::Unknown);
}
let value = get_consistent_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE).ok_or(Error::FileCorrupt)?;
let state = match value {
b"known-disabled" => TransitionVersionState::KnownDisabled,
b"suspended-null" => TransitionVersionState::SuspendedNull,
b"exact" => TransitionVersionState::Exact,
b"unknown" => TransitionVersionState::Unknown,
_ => return Err(Error::FileCorrupt),
};
let valid = match state {
TransitionVersionState::Unknown | TransitionVersionState::KnownDisabled => version.is_none(),
TransitionVersionState::SuspendedNull => version == Some("null"),
TransitionVersionState::Exact => version.is_some_and(|value| value != "null"),
};
valid.then_some(state).ok_or(Error::FileCorrupt)
}
fn transition_version_state_bytes(state: TransitionVersionState) -> &'static [u8] {
match state {
TransitionVersionState::Unknown => b"unknown",
TransitionVersionState::KnownDisabled => b"known-disabled",
TransitionVersionState::SuspendedNull => b"suspended-null",
TransitionVersionState::Exact => b"exact",
}
}
fn set_transition_version_state(meta_sys: &mut HashMap<String, Vec<u8>>, state: TransitionVersionState) {
if state == TransitionVersionState::Unknown {
remove_bytes(meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE);
} else {
insert_bytes(
meta_sys,
SUFFIX_TRANSITIONED_VERSION_STATE,
transition_version_state_bytes(state).to_vec(),
);
}
}
fn legacy_transitioned_version_id_from_meta_sys(meta_sys: &HashMap<String, Vec<u8>>) -> Option<Uuid> {
transitioned_version_from_meta_sys(meta_sys).and_then(|value| Uuid::parse_str(&value).ok())
transitioned_version_from_meta_sys(meta_sys)
.ok()
.flatten()
.and_then(|value| Uuid::parse_str(&value).ok())
}
fn transitioned_version_bytes(fi: &FileInfo) -> Option<Vec<u8>> {
@@ -2414,7 +2470,8 @@ impl MetaObject {
let transitioned_objname = get_bytes(&self.meta_sys, SUFFIX_TRANSITIONED_OBJECTNAME)
.map(|v| String::from_utf8_lossy(&v).to_string())
.unwrap_or_default();
let transition_version = transitioned_version_from_meta_sys(&self.meta_sys);
let transition_version = transitioned_version_from_meta_sys(&self.meta_sys)?;
let transition_version_state = transition_version_state_from_meta_sys(&self.meta_sys, transition_version.as_deref())?;
let transition_version_id = transition_version.as_deref().and_then(|value| Uuid::parse_str(value).ok());
let transition_tier = get_bytes(&self.meta_sys, SUFFIX_TRANSITION_TIER)
.map(|v| String::from_utf8_lossy(&v).to_string())
@@ -2437,6 +2494,7 @@ impl MetaObject {
transitioned_objname,
transition_version_id,
transition_version,
transition_version_state,
transition_tier,
..Default::default()
})
@@ -2452,6 +2510,7 @@ impl MetaObject {
if let Some(transition_version) = transitioned_version_bytes(fi) {
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
}
set_transition_version_state(&mut self.meta_sys, fi.transition_version_state);
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER, fi.transition_tier.as_bytes().to_vec());
if let Some(destination_id) = get_str(&fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID) {
insert_bytes(&mut self.meta_sys, SUFFIX_TRANSITION_TIER_DESTINATION_ID, destination_id.into_bytes());
@@ -2515,6 +2574,7 @@ impl MetaObject {
SUFFIX_TRANSITION_TIER,
SUFFIX_TRANSITIONED_OBJECTNAME,
SUFFIX_TRANSITIONED_VERSION_ID,
SUFFIX_TRANSITIONED_VERSION_STATE,
] {
if let Some(v) = get_bytes(&self.meta_sys, suffix) {
insert_bytes(&mut delete_marker.meta_sys, suffix, v);
@@ -2579,6 +2639,9 @@ impl From<FileInfo> for MetaObject {
if let Some(transition_version) = transitioned_version_bytes(&value) {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
}
if !value.transition_status.is_empty() {
set_transition_version_state(&mut meta_sys, value.transition_version_state);
}
if !value.transition_tier.is_empty() {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec());
@@ -2720,8 +2783,11 @@ impl MetaDeleteMarker {
.map(|v| String::from_utf8_lossy(&v).to_string())
.unwrap_or_default();
fi.transition_version = transitioned_version_from_meta_sys(&self.meta_sys);
fi.transition_version = transitioned_version_from_meta_sys(&self.meta_sys).ok().flatten();
fi.transition_version_id = legacy_transitioned_version_id_from_meta_sys(&self.meta_sys);
fi.transition_version_state =
transition_version_state_from_meta_sys(&self.meta_sys, fi.transition_version.as_deref())
.unwrap_or(TransitionVersionState::Unknown);
}
fi
@@ -2877,6 +2943,9 @@ impl From<FileInfo> for MetaDeleteMarker {
if let Some(transition_version) = transitioned_version_bytes(&value) {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITIONED_VERSION_ID, transition_version);
}
if !value.transition_status.is_empty() || value.tier_free_version() {
set_transition_version_state(&mut meta_sys, value.transition_version_state);
}
if !value.transition_tier.is_empty() {
insert_bytes(&mut meta_sys, SUFFIX_TRANSITION_TIER, value.transition_tier.as_bytes().to_vec());
}
@@ -4125,6 +4194,7 @@ mod tests {
.expect("into_fileinfo");
assert_eq!(fi.transition_version_id, Some(id));
assert_eq!(fi.transition_version, Some(id.to_string()));
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
}
#[test]
@@ -4136,6 +4206,61 @@ mod tests {
.expect("opaque transition version id must decode");
assert_eq!(fi.transition_version_id, None);
assert_eq!(fi.transition_version.as_deref(), Some("opaque-generation-42"));
assert_eq!(fi.transition_version_state, TransitionVersionState::Unknown);
}
#[test]
fn meta_object_transition_version_state_exact_round_trips_dual_keys() {
let id = sample_version_id();
let expected_version = id.to_string();
let fi = FileInfo {
transition_status: "complete".to_string(),
transition_version: Some(expected_version.clone()),
transition_version_state: TransitionVersionState::Exact,
..Default::default()
};
let object = MetaObject::from(fi);
assert_eq!(
object
.meta_sys
.get(&format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_STATE}"))
.map(Vec::as_slice),
Some(b"exact".as_slice())
);
assert_eq!(
object
.meta_sys
.get(&format!(
"{}{SUFFIX_TRANSITIONED_VERSION_STATE}",
rustfs_utils::http::MINIO_INTERNAL_PREFIX
))
.map(Vec::as_slice),
Some(b"exact".as_slice())
);
assert_eq!(
legacy_transitioned_version_id_from_meta_sys(&object.meta_sys),
Some(id),
"UUID exact writes must remain readable by the legacy UUID consumer"
);
let decoded = object.into_fileinfo("b", "k", false).expect("exact state should round trip");
assert_eq!(decoded.transition_version_state, TransitionVersionState::Exact);
assert_eq!(decoded.transition_version.as_deref(), Some(expected_version.as_str()));
}
#[test]
fn meta_object_transition_version_state_conflict_fails_closed() {
let mut sys = HashMap::new();
insert_bytes(&mut sys, SUFFIX_TRANSITIONED_VERSION_ID, sample_version_id().as_bytes().to_vec());
sys.insert(format!("{RUSTFS_INTERNAL_PREFIX}{SUFFIX_TRANSITIONED_VERSION_STATE}"), b"exact".to_vec());
sys.insert(
format!("{}{SUFFIX_TRANSITIONED_VERSION_STATE}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
b"known-disabled".to_vec(),
);
make_meta_object_with_sys(sys)
.into_fileinfo("b", "k", false)
.expect_err("conflicting state keys must fail closed");
}
#[test]