gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-06 05:17:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(replication): propagate metadata changes

Browse Source 
Preserve metadata replication operations in the durable MRF and route tagging, retention, and legal-hold updates through the existing full-object replication transport. Keep ACL propagation outside the contract because the current object model has no durable object ACL state.

Refs #1616
This commit is contained in:
马登山
2026-08-02 19:06:24 +08:00
parent c104ba23d4
commit bed71291ba
8 changed files with 294 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/replication/replication_pool.rs
+44 -1
View File
@@ -22,7 +22,8 @@ use super::replication_filemeta_boundary::{
use super::replication_lock_boundary::ReplicationLockTiming;
use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
use super::replication_metadata_boundary::ReplicationMetadataStore;
use super::replication_object_config::{ReplicationConfig, check_replicate_delete};
use super::replication_object_config::{ReplicationConfig, check_replicate_delete, must_replicate};
use super::replication_object_decision_boundary::MustReplicateOptions;
use super::replication_queue_boundary::{
DeletedObjectReplicationInfo, LARGE_WORKER_COUNT, ReplicationBackpressureRecommendation, ReplicationBackpressureState,
ReplicationHealQueueAction, ReplicationHealQueueResult, ReplicationHealResyncDeletes, ReplicationOperation,
@@ -1021,6 +1022,28 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
queue_replication_heal(&entry.bucket, oi, entry.retry_count as u32).await;
queued_count += 1;
}
MrfOpKind::Metadata => {
let opts = ObjectOptions {
version_id: entry.version_id.map(|u| u.to_string()),
..Default::default()
};
let oi = match storage.get_object_info(&entry.bucket, &entry.object, &opts).await {
Ok(oi) => oi,
Err(e) => {
debug!(
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION,
bucket = %entry.bucket,
object = %entry.object,
error = %e,
"MRF metadata recovery: object not found, skipping"
);
continue;
}
};
queue_replication_metadata(&entry.bucket, oi, entry.retry_count as u32).await;
queued_count += 1;
}
}
}
@@ -1942,6 +1965,26 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
queue_replication_heal_internal(bucket, oi, rcfg_wrapper, retry_count).await;
}
pub async fn queue_replication_metadata(bucket: &str, oi: ObjectInfo, retry_count: u32) {
let dsc = must_replicate(
bucket,
&oi.name,
MustReplicateOptions::new(&oi.user_defined, (*oi.user_tags).clone(), ReplicationType::Metadata, false)
.with_replication_status(oi.replication_status.clone()),
)
.await;
if !dsc.replicate_any() {
return;
}
let mut roi = replicate_object_info_from_object_info(oi, dsc, ReplicationType::Metadata);
roi.retry_count = retry_count;
if let Some(pool) = runtime_sources::replication_pool() {
let _ = pool.queue_replica_task(roi).await;
}
}
/// queue_replication_heal_internal enqueues objects that failed replication OR eligible for resyncing through
/// an ongoing resync operation or via existing objects replication configuration setting.
pub(crate) async fn queue_replication_heal_internal(
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+69 -93
View File
@@ -96,7 +96,6 @@ const EVENT_REPLICATION_FORCE_DELETE_SKIPPED: &str = "replication_force_delete_s
const EVENT_RESYNC_TASK_FAILED: &str = "replication_resync_task_failed";
const EVENT_RESYNC_TARGET_OPERATION_FAILED: &str = "replication_resync_target_operation_failed";
const EVENT_RESYNC_RUNTIME_CHANNEL_FAILED: &str = "replication_resync_runtime_channel_failed";
const ERR_REPLICATION_METADATA_COPY_UNSUPPORTED: &str = "metadata-only replication is not implemented";
const REPLICATION_TARGET_OFFLINE_ERROR_MARKERS: &[&str] = &[
"dispatch failure",
"timeouterror",
@@ -2860,7 +2859,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
// The target already holds a matching object (reached here only via
// the version-id fallback ETag match above) — there is nothing to
// copy. Record it as synced and return, instead of falling into the
// metadata-unsupported failure branch below, which previously left
// metadata propagation path below, which previously left
// AWS-style targets permanently FAILED and never converging
// (backlog#860 / #799 B11).
if self.op_type == ReplicationType::ExistingObject && !tgt_client.reset_id.is_empty() {
@@ -2877,10 +2876,71 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
return rinfo;
}
// action == Metadata: metadata-only replication is not implemented.
if replication_action != ReplicationAction::All {
// The target client has no metadata-only operation. Reuse the existing
// object transport so metadata changes carry tags and object-lock state
// atomically with the source version.
let (put_opts, is_multipart) = match replication_put_object_options(&tgt_client.storage_class, &object_info) {
Ok((put_opts, is_mp)) => (put_opts, is_mp),
Err(e) => {
rinfo.error = Some(e.to_string());
warn!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
operation = "build_put_options",
error = %e,
"Replication target operation failed"
);
send_local_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: object_info,
user_agent: "Internal: [Replication]".to_string(),
..Default::default()
});
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
return rinfo;
}
};
let has_tagging_replication = !put_opts.user_tags.is_empty();
if let Some(err) = if is_multipart {
drop(gr);
let result = replicate_object_with_multipart(MultipartReplicationContext {
storage: storage.clone(),
cli: tgt_client.clone(),
src_bucket: &bucket,
dst_bucket: &tgt_client.bucket,
object: &object,
object_info: &object_info,
obj_opts: &obj_opts,
arn: &rinfo.arn,
put_opts,
})
.await;
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
if has_tagging_replication {
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
}
result.err()
} else {
gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
let byte_stream = async_read_to_bytestream(gr.stream);
let result = tgt_client
.put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
.await
.map_err(|e| std::io::Error::other(e.to_string()));
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
if has_tagging_replication {
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
}
result.err()
} {
rinfo.replication_status = ReplicationStatusType::Failed;
rinfo.error = Some(ERR_REPLICATION_METADATA_COPY_UNSUPPORTED.to_string());
rinfo.error = Some(err.to_string());
warn!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
@@ -2888,98 +2948,14 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
bucket = %bucket,
arn = %tgt_client.arn,
object = %object,
operation = "copy_object_metadata",
error = ERR_REPLICATION_METADATA_COPY_UNSUPPORTED,
operation = "put_object",
error = ?err,
"Replication target operation failed"
);
send_local_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: object_info,
user_agent: "Internal: [Replication]".to_string(),
..Default::default()
});
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
mark_replication_target_offline_if_needed(&tgt_client, &err).await;
return rinfo;
} else {
let (put_opts, is_multipart) = match replication_put_object_options(&tgt_client.storage_class, &object_info) {
Ok((put_opts, is_mp)) => (put_opts, is_mp),
Err(e) => {
rinfo.error = Some(e.to_string());
warn!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
operation = "build_put_options",
error = %e,
"Replication target operation failed"
);
send_local_event(EventArgs {
event_name: EventName::ObjectReplicationNotTracked.to_string(),
bucket_name: bucket.clone(),
object: object_info,
user_agent: "Internal: [Replication]".to_string(),
..Default::default()
});
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
return rinfo;
}
};
let has_tagging_replication = !put_opts.user_tags.is_empty();
if let Some(err) = if is_multipart {
drop(gr);
let result = replicate_object_with_multipart(MultipartReplicationContext {
storage: storage.clone(),
cli: tgt_client.clone(),
src_bucket: &bucket,
dst_bucket: &tgt_client.bucket,
object: &object,
object_info: &object_info,
obj_opts: &obj_opts,
arn: &rinfo.arn,
put_opts,
})
.await;
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
if has_tagging_replication {
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
}
result.err()
} else {
gr.stream = wrap_with_bandwidth_monitor(gr.stream, &put_opts, &bucket, &rinfo.arn);
let byte_stream = async_read_to_bytestream(gr.stream);
let result = tgt_client
.put_object(&tgt_client.bucket, &object, size, byte_stream, &put_opts)
.await
.map_err(|e| std::io::Error::other(e.to_string()));
record_proxy_request(&bucket, "PutObject", result.is_err()).await;
if has_tagging_replication {
record_proxy_request(&bucket, "PutObjectTagging", result.is_err()).await;
}
result.err()
} {
rinfo.replication_status = ReplicationStatusType::Failed;
rinfo.error = Some(err.to_string());
warn!(
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
component = LOG_COMPONENT_ECSTORE,
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
bucket = %bucket,
arn = %tgt_client.arn,
object = %object,
operation = "put_object",
error = ?err,
"Replication target operation failed"
);
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
mark_replication_target_offline_if_needed(&tgt_client, &err).await;
return rinfo;
}
}
rinfo
crates/ecstore/src/set_disk/ops/object.rs
+5
View File
@@ -2768,6 +2768,11 @@ impl SetDisks {
let (mut fi, _, disks) = self.get_object_fileinfo_gated(bucket, object, opts, false, false).await?;
fi.metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags.to_owned());
if let Some(eval_metadata) = &opts.eval_metadata {
for (key, value) in eval_metadata {
fi.metadata.insert(key.clone(), value.clone());
}
}
#[cfg(test)]
pause_object_tagging_commit(bucket, object).await;
crates/replication/src/filemeta.rs
+24 -2
View File
@@ -550,6 +550,8 @@ pub enum MrfOpKind {
#[default]
#[serde(rename = "object")]
Object,
#[serde(rename = "metadata")]
Metadata,
#[serde(rename = "delete")]
Delete,
}
@@ -810,7 +812,11 @@ impl ReplicationWorkerOperation for ReplicateObjectInfo {
version_id: self.version_id,
retry_count: retry_count_to_mrf(self.retry_count),
size: self.size,
op: MrfOpKind::Object,
op: if self.op_type == ReplicationType::Metadata {
MrfOpKind::Metadata
} else {
MrfOpKind::Object
},
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
@@ -865,7 +871,11 @@ impl ReplicateObjectInfo {
version_id: self.version_id,
retry_count: retry_count_to_mrf(self.retry_count),
size: self.size,
op: MrfOpKind::Object,
op: if self.op_type == ReplicationType::Metadata {
MrfOpKind::Metadata
} else {
MrfOpKind::Object
},
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
@@ -1067,6 +1077,18 @@ mod tests {
assert_eq!(entry.target_arns, vec!["arn:target-a".to_string()]);
}
#[test]
fn metadata_replication_mrf_entry_preserves_operation_kind() {
let info = ReplicateObjectInfo {
bucket: "bucket".to_string(),
name: "object".to_string(),
op_type: ReplicationType::Metadata,
..Default::default()
};
assert_eq!(info.to_mrf_entry().op, MrfOpKind::Metadata);
}
#[test]
fn target_state_reads_resync_timestamp_from_target_reset_header_key() {
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
crates/replication/src/mrf.rs
+24 -9
View File
@@ -57,10 +57,22 @@ mod tests {
use uuid::Uuid;
#[test]
fn mrf_file_round_trips_object_and_delete_entries() {
fn mrf_file_round_trips_object_metadata_and_delete_entries() {
let obj_vid = Uuid::new_v4();
let del_vid = Uuid::new_v4();
let entries = vec![
MrfReplicateEntry {
bucket: "bucket-a".to_string(),
object: "metadata-a".to_string(),
version_id: Some(obj_vid),
retry_count: 1,
size: 1024,
op: MrfOpKind::Metadata,
delete_marker_version_id: None,
delete_marker: false,
delete_marker_mtime: None,
target_arns: vec!["arn:target-a".to_string()],
},
MrfReplicateEntry {
bucket: "bucket-a".to_string(),
object: "object-a".to_string(),
@@ -90,17 +102,20 @@ mod tests {
let encoded = encode_mrf_file(&entries).expect("mrf file should encode");
let decoded = decode_mrf_file(&encoded).expect("mrf file should decode");
assert_eq!(decoded.len(), 2);
assert_eq!(decoded.len(), 3);
assert_eq!(decoded[0].version_id, Some(obj_vid));
assert_eq!(decoded[0].op, MrfOpKind::Object);
assert_eq!(decoded[0].target_arns, vec!["arn:target-a".to_string(), "arn:target-b".to_string()]);
assert_eq!(decoded[0].op, MrfOpKind::Metadata);
assert_eq!(decoded[0].target_arns, vec!["arn:target-a".to_string()]);
assert_eq!(decoded[0].delete_marker_mtime, None);
assert_eq!(decoded[1].delete_marker_version_id, Some(del_vid));
assert_eq!(decoded[1].op, MrfOpKind::Delete);
assert_eq!(decoded[1].target_arns, vec!["arn:target-a".to_string()]);
assert!(decoded[1].delete_marker);
assert_eq!(decoded[1].op, MrfOpKind::Object);
assert_eq!(decoded[1].target_arns, vec!["arn:target-a".to_string(), "arn:target-b".to_string()]);
assert_eq!(decoded[1].delete_marker_mtime, None);
assert_eq!(decoded[2].delete_marker_version_id, Some(del_vid));
assert_eq!(decoded[2].op, MrfOpKind::Delete);
assert_eq!(decoded[2].target_arns, vec!["arn:target-a".to_string()]);
assert!(decoded[2].delete_marker);
assert_eq!(
decoded[1].delete_marker_mtime,
decoded[2].delete_marker_mtime,
Some(1_705_312_200_123_456_789),
"delete-marker mtime must survive the MRF disk round-trip"
);
crates/replication/src/object.rs
+24
View File
@@ -164,6 +164,7 @@ mod tests {
replication_etags_match, target_is_newer_than_source_null_version,
};
use crate::filemeta::{ReplicationAction, ReplicationType};
use crate::http::AMZ_OBJECT_LOCK_MODE;
use std::collections::HashMap;
use time::{Duration, OffsetDateTime};
@@ -267,4 +268,27 @@ mod tests {
ReplicationAction::Metadata
);
}
#[test]
fn replication_action_detects_tags_and_object_lock_metadata_differences() {
let mut source_metadata = HashMap::new();
source_metadata.insert(AMZ_OBJECT_LOCK_MODE.to_string(), "GOVERNANCE".to_string());
let source = ReplicationSourceObject {
user_tags: "a=1&b=2",
user_defined: &source_metadata,
..source_object(&source_metadata)
};
let target_metadata = HashMap::new();
let target = ReplicationTargetObject {
tag_count: 1,
metadata: Some(&target_metadata),
..target_object(&target_metadata)
};
assert_eq!(
replication_action_for_target(&source, &target, ReplicationType::Metadata),
ReplicationAction::Metadata
);
}
}