gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-09-06 03:59:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(replication): fence stale metadata status writeback (#7083)

Browse Source 
Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
cxymds
2026-09-03 16:05:58 +08:00
committed by GitHub
parent 8023cf3e26
commit 86ebcb325c
29 changed files with 3219 additions and 229 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/replication/src/filemeta.rs
+18
View File
@@ -849,6 +849,20 @@ pub fn parse_replicate_decision(_bucket: &str, s: &str) -> std::io::Result<Repli
// }
}
/// Source snapshot used to fence replication terminal-status publication.
///
/// Timestamp and mutation id remain opaque because supported RustFS/MinIO
/// writers may use different textual representations. `invalid` preserves the
/// distinction between truly absent legacy metadata and corrupt/conflicting
/// compatibility aliases.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ReplicationGenerationSnapshot {
pub timestamp: Option<String>,
pub mutation_id: Option<String>,
pub payload_fingerprint: Option<[u8; 32]>,
pub invalid: bool,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ReplicateObjectInfo {
pub name: String,
@@ -871,6 +885,10 @@ pub struct ReplicateObjectInfo {
pub target_statuses: HashMap<String, ReplicationStatusType>,
pub target_purge_statuses: HashMap<String, VersionPurgeStatusType>,
pub replication_timestamp: Option<OffsetDateTime>,
/// Exact persisted source snapshot used to fence terminal status
/// write-back after remote I/O.
#[serde(default)]
pub replication_generation: ReplicationGenerationSnapshot,
pub ssec: bool,
pub user_tags: String,
pub checksum: Option<Bytes>,
crates/replication/src/http.rs
+52 -2
View File
@@ -85,6 +85,24 @@ pub(crate) fn get_internal_metadata(map: &HashMap<String, String>, suffix: &str)
})
}
/// Return a non-empty internal value only when every RustFS/MinIO alias
/// present for the suffix agrees. A single alias remains valid for rolling
/// upgrade compatibility; conflicts fail closed at security boundaries.
pub(crate) fn get_consistent_internal_metadata<'a>(map: &'a HashMap<String, String>, suffix: &str) -> Option<&'a str> {
let (rustfs_key, minio_key) = internal_keys(suffix);
let mut value: Option<&String> = None;
for (key, candidate) in map {
if !key.eq_ignore_ascii_case(&rustfs_key) && !key.eq_ignore_ascii_case(&minio_key) {
continue;
}
if candidate.is_empty() || value.is_some_and(|current| current != candidate) {
return None;
}
value = Some(candidate);
}
value.map(String::as_str)
}
pub(crate) fn get_header_metadata(map: &HashMap<String, String>, suffix: &str) -> Option<String> {
let rustfs_key = rustfs_header_key(suffix);
let minio_key = minio_header_key(suffix);
@@ -114,8 +132,8 @@ pub(crate) fn insert_internal_metadata(map: &mut HashMap<String, String>, suffix
#[cfg(test)]
mod tests {
use super::{
SUFFIX_ACTUAL_SIZE, SUFFIX_REPLICATION_RESET_STATUS, get_header_metadata, get_internal_metadata, has_prefix_fold,
insert_internal_metadata, internal_key_rustfs, trim_etag,
SUFFIX_ACTUAL_SIZE, SUFFIX_REPLICATION_RESET_STATUS, get_consistent_internal_metadata, get_header_metadata,
get_internal_metadata, has_prefix_fold, insert_internal_metadata, internal_key_rustfs, trim_etag,
};
use std::collections::HashMap;
@@ -136,6 +154,38 @@ mod tests {
assert_eq!(get_internal_metadata(&metadata, SUFFIX_ACTUAL_SIZE).as_deref(), Some("12"));
}
#[test]
fn consistent_internal_metadata_accepts_single_or_matching_aliases_and_rejects_conflicts() {
for key in ["x-rustfs-internal-replication-status", "X-Minio-Internal-Replication-Status"] {
let metadata = HashMap::from([(key.to_string(), "arn:target=PENDING;".to_string())]);
assert_eq!(
get_consistent_internal_metadata(&metadata, "replication-status"),
Some("arn:target=PENDING;")
);
}
let matching = HashMap::from([
("x-rustfs-internal-replication-status".to_string(), "arn:target=COMPLETED;".to_string()),
("X-Minio-Internal-Replication-Status".to_string(), "arn:target=COMPLETED;".to_string()),
]);
assert_eq!(
get_consistent_internal_metadata(&matching, "replication-status"),
Some("arn:target=COMPLETED;")
);
let conflicting = HashMap::from([
("x-rustfs-internal-replication-status".to_string(), "arn:target=PENDING;".to_string()),
("x-minio-internal-replication-status".to_string(), "arn:target=REPLICA;".to_string()),
]);
assert!(get_consistent_internal_metadata(&conflicting, "replication-status").is_none());
let empty_alias = HashMap::from([
("x-rustfs-internal-replication-status".to_string(), "arn:target=PENDING;".to_string()),
("x-minio-internal-replication-status".to_string(), String::new()),
]);
assert!(get_consistent_internal_metadata(&empty_alias, "replication-status").is_none());
}
#[test]
fn internal_metadata_insert_writes_rustfs_and_minio_keys() {
let mut metadata = HashMap::new();
crates/replication/src/lib.rs
+4 -4
View File
@@ -48,10 +48,10 @@ pub use delete::{
pub use filemeta::{
NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING,
REPLICATE_INCOMING_DELETE, REPLICATE_MRF, REPLICATE_QUEUED, REPLICATION_RESET, REPLICATION_STATUS, ReplicateDecision,
ReplicateObjectInfo, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState,
ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision,
VersionPurgeStatusType, get_replication_state, parse_replicate_decision, replicate_decision_for_admitted_targets,
replication_statuses_map, target_reset_header, version_purge_statuses_map,
ReplicateObjectInfo, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction,
ReplicationGenerationSnapshot, ReplicationState, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation,
ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType, get_replication_state, parse_replicate_decision,
replicate_decision_for_admitted_targets, replication_statuses_map, target_reset_header, version_purge_statuses_map,
};
pub use mrf::{
MRF_ENVELOPE_FORMAT, MRF_ENVELOPE_VERSION, MRF_V2_FILE, MRF_V2_FORMAT, MRF_V2_NAMESPACE, MRF_V2_VERSION, MrfCapabilities,
crates/replication/src/operation.rs
+87 -18
View File
@@ -16,7 +16,7 @@ use std::collections::HashMap;
use crate::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER,
SUFFIX_REPLICATION_RESET_STATUS, SUFFIX_REPLICATION_STATUS, get_header_metadata, get_internal_metadata,
SUFFIX_REPLICATION_RESET_STATUS, SUFFIX_REPLICATION_STATUS, get_consistent_internal_metadata, get_header_metadata,
};
use s3s::dto::ReplicationConfiguration;
use time::OffsetDateTime;
@@ -99,15 +99,34 @@ impl MustReplicateOptions {
return true;
}
get_internal_metadata(&self.meta, SUFFIX_REPLICATION_STATUS)
.as_deref()
.and_then(|statuses| {
statuses.split(';').find_map(|entry| {
let (target_arn, status) = entry.split_once('=')?;
(target_arn == arn).then(|| ReplicationStatusType::from(status))
})
})
== Some(ReplicationStatusType::Completed)
let Some(statuses) = get_consistent_internal_metadata(&self.meta, SUFFIX_REPLICATION_STATUS) else {
return false;
};
let mut admitted = None;
for entry in statuses.split(';').filter(|entry| !entry.is_empty()) {
let Some((target_arn, status)) = entry.split_once('=') else {
continue;
};
if target_arn != arn {
continue;
}
// Per-target admission is authoritative only when the target has
// exactly one unambiguous historical status record.
if admitted.is_some() {
return false;
}
admitted = Some(ReplicationStatusType::from(status));
}
matches!(
admitted,
Some(
ReplicationStatusType::Pending
| ReplicationStatusType::Failed
| ReplicationStatusType::Completed
| ReplicationStatusType::CompletedLegacy
)
)
}
}
@@ -376,22 +395,72 @@ mod tests {
}
#[test]
fn metadata_replication_requires_completed_target_state() {
fn metadata_replication_requires_prior_target_admission() {
let arn = "arn:rustfs:replication:target";
for status in ["PENDING", "FAILED"] {
for status in ["PENDING", "FAILED", "COMPLETED", "COMPLETE"] {
let mut meta = HashMap::new();
insert_internal_metadata(&mut meta, SUFFIX_REPLICATION_STATUS, format!("{arn}={status};"));
let options = MustReplicateOptions::new(&meta, String::new(), ReplicationType::Metadata, false);
assert!(!options.metadata_target_is_eligible(arn));
assert!(
options.metadata_target_is_eligible(arn),
"known target state {status} must remain eligible"
);
assert!(!options.metadata_target_is_eligible("arn:rustfs:replication:missing"));
}
let mut meta = HashMap::new();
insert_internal_metadata(&mut meta, SUFFIX_REPLICATION_STATUS, format!("{arn}=COMPLETED;"));
let options = MustReplicateOptions::new(&meta, String::new(), ReplicationType::Metadata, false);
for status in ["", "REPLICA", "UNKNOWN"] {
let mut meta = HashMap::new();
insert_internal_metadata(&mut meta, SUFFIX_REPLICATION_STATUS, format!("{arn}={status};"));
let options = MustReplicateOptions::new(&meta, String::new(), ReplicationType::Metadata, false);
assert!(
!options.metadata_target_is_eligible(arn),
"unsupported target state {status:?} must fail closed"
);
}
assert!(options.metadata_target_is_eligible(arn));
assert!(!options.metadata_target_is_eligible("arn:rustfs:replication:missing"));
for key in ["x-rustfs-internal-replication-status", "X-Minio-Internal-Replication-Status"] {
let meta = HashMap::from([(key.to_string(), format!("{arn}=PENDING;"))]);
assert!(
MustReplicateOptions::new(&meta, String::new(), ReplicationType::Metadata, false)
.metadata_target_is_eligible(arn),
"a single compatibility alias must preserve rolling-upgrade admission"
);
}
let matching_aliases = HashMap::from([
("x-rustfs-internal-replication-status".to_string(), format!("{arn}=COMPLETED;")),
("X-Minio-Internal-Replication-Status".to_string(), format!("{arn}=COMPLETED;")),
]);
assert!(
MustReplicateOptions::new(&matching_aliases, String::new(), ReplicationType::Metadata, false)
.metadata_target_is_eligible(arn)
);
for conflicting in [
HashMap::from([
("x-rustfs-internal-replication-status".to_string(), format!("{arn}=PENDING;")),
("x-minio-internal-replication-status".to_string(), format!("{arn}=REPLICA;")),
]),
HashMap::from([
("x-rustfs-internal-replication-status".to_string(), format!("{arn}=PENDING;")),
("x-minio-internal-replication-status".to_string(), String::new()),
]),
HashMap::from([(
"x-rustfs-internal-replication-status".to_string(),
format!("{arn}=PENDING;{arn}=REPLICA;"),
)]),
HashMap::from([(
"x-rustfs-internal-replication-status".to_string(),
format!("{arn}=PENDING;{arn}=PENDING;"),
)]),
] {
assert!(
!MustReplicateOptions::new(&conflicting, String::new(), ReplicationType::Metadata, false)
.metadata_target_is_eligible(arn),
"conflicting, empty, or duplicate admission evidence must fail closed"
);
}
// A replica-side metadata edit (active-active) has no per-target
// internal status; eligibility is left to the ReplicaModifications rule.