fix(replication): harden MRF replay durability (#5659)

* feat(replication): add MRF envelope capabilities * fix(replication): retain failed MRF replay entries * fix(replication): retain transient MRF source failures * fix(replication): address MRF durability review feedback * fix(replication): preserve MRF recovery handoff * fix(replication): harden MRF recovery handoff
2026-08-12 08:06:54 +00:00 · 2026-08-03 15:35:00 +08:00
parent 371a3529e5
commit 9b4a73f315
3 changed files with 984 additions and 162 deletions
@@ -47,7 +47,10 @@ pub use filemeta::{
    VersionPurgeStatusType, get_replication_state, parse_replicate_decision, replicate_decision_for_admitted_targets,
    replication_statuses_map, target_reset_header, version_purge_statuses_map,
 };
-pub use mrf::{MrfOpKind, MrfReplicateEntry, decode_mrf_file, encode_mrf_file};
+pub use mrf::{
+    MrfCapabilities, MrfCapability, MrfEnvelope, MrfEnvelopeError, MrfOpKind, MrfProtocolCapabilities, MrfReplicateEntry,
+    decode_mrf_file, encode_mrf_file,
+};
 pub use multipart::{
    ReplicationMultipartPartInput, ReplicationMultipartPartPlan, ReplicationMultipartPlanError, ReplicationMultipartRange,
    replication_multipart_complete_actual_size, replication_multipart_part_plan,
@@ -13,6 +13,7 @@
 // limitations under the License.

 use byteorder::{ByteOrder, LittleEndian};
+use std::fmt;

 use crate::{Error, Result};

@@ -21,6 +22,324 @@ pub use crate::filemeta::{MrfOpKind, MrfReplicateEntry};
 pub const MRF_META_FORMAT: u16 = 1;
 pub const MRF_META_VERSION: u16 = 1;

+const MRF_ENVELOPE_MAGIC: [u8; 4] = *b"MRFE";
+const MRF_ENVELOPE_HEADER_LEN: usize = 24;
+pub const MRF_ENVELOPE_FORMAT: u16 = 1;
+pub const MRF_ENVELOPE_VERSION: u16 = 1;
+
+const CAPABILITY_OPERATION_KIND: u64 = 1 << 0;
+const CAPABILITY_TARGET_ARNS: u64 = 1 << 1;
+const CAPABILITY_FORCE_DELETE: u64 = 1 << 2;
+const CAPABILITY_DELETE_MARKER_MTIME: u64 = 1 << 3;
+const MRF_KNOWN_CAPABILITIES: u64 =
+    CAPABILITY_OPERATION_KIND | CAPABILITY_TARGET_ARNS | CAPABILITY_FORCE_DELETE | CAPABILITY_DELETE_MARKER_MTIME;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum MrfCapability {
+    OperationKind,
+    TargetArns,
+    ForceDelete,
+    DeleteMarkerMtime,
+}
+
+impl MrfCapability {
+    const fn bit(self) -> u64 {
+        match self {
+            Self::OperationKind => CAPABILITY_OPERATION_KIND,
+            Self::TargetArns => CAPABILITY_TARGET_ARNS,
+            Self::ForceDelete => CAPABILITY_FORCE_DELETE,
+            Self::DeleteMarkerMtime => CAPABILITY_DELETE_MARKER_MTIME,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub struct MrfCapabilities(u64);
+
+impl MrfCapabilities {
+    pub const fn current() -> Self {
+        Self(MRF_KNOWN_CAPABILITIES)
+    }
+
+    pub const fn empty() -> Self {
+        Self(0)
+    }
+
+    pub const fn with(capability: MrfCapability) -> Self {
+        Self(capability.bit())
+    }
+
+    pub const fn bits(self) -> u64 {
+        self.0
+    }
+
+    pub const fn contains(self, capability: MrfCapability) -> bool {
+        self.0 & capability.bit() != 0
+    }
+
+    pub const fn supports(self, required: Self) -> bool {
+        self.0 & required.0 == required.0
+    }
+
+    pub fn from_bits(bits: u64) -> std::result::Result<Self, MrfEnvelopeError> {
+        let unknown = bits & !MRF_KNOWN_CAPABILITIES;
+        if unknown != 0 {
+            return Err(MrfEnvelopeError::UnknownCapabilities { bits: unknown });
+        }
+        Ok(Self(bits))
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct MrfProtocolCapabilities {
+    version: u16,
+    min_reader_version: u16,
+    capabilities: MrfCapabilities,
+}
+
+impl MrfProtocolCapabilities {
+    pub const fn new(version: u16, min_reader_version: u16, capabilities: MrfCapabilities) -> Self {
+        Self {
+            version,
+            min_reader_version,
+            capabilities,
+        }
+    }
+
+    pub const fn current() -> Self {
+        Self {
+            version: MRF_ENVELOPE_VERSION,
+            min_reader_version: MRF_ENVELOPE_VERSION,
+            capabilities: MrfCapabilities::current(),
+        }
+    }
+
+    pub const fn version(self) -> u16 {
+        self.version
+    }
+
+    pub const fn min_reader_version(self) -> u16 {
+        self.min_reader_version
+    }
+
+    pub const fn capabilities(self) -> MrfCapabilities {
+        self.capabilities
+    }
+
+    pub fn negotiate(self, peer: Self) -> std::result::Result<Self, MrfEnvelopeError> {
+        MrfCapabilities::from_bits(self.capabilities.bits())?;
+        MrfCapabilities::from_bits(peer.capabilities.bits())?;
+        if self.min_reader_version > self.version {
+            return Err(MrfEnvelopeError::InvalidVersionRange {
+                version: self.version,
+                min_reader_version: self.min_reader_version,
+            });
+        }
+        if peer.min_reader_version > peer.version {
+            return Err(MrfEnvelopeError::InvalidVersionRange {
+                version: peer.version,
+                min_reader_version: peer.min_reader_version,
+            });
+        }
+        if self.version < MRF_ENVELOPE_VERSION {
+            return Err(MrfEnvelopeError::UnsupportedVersion { version: self.version });
+        }
+        if peer.version < MRF_ENVELOPE_VERSION {
+            return Err(MrfEnvelopeError::UnsupportedVersion { version: peer.version });
+        }
+        if peer.min_reader_version > self.version {
+            return Err(MrfEnvelopeError::RollbackFenced {
+                min_reader_version: peer.min_reader_version,
+                supported_version: self.version,
+            });
+        }
+        if self.min_reader_version > peer.version {
+            return Err(MrfEnvelopeError::RollbackFenced {
+                min_reader_version: self.min_reader_version,
+                supported_version: peer.version,
+            });
+        }
+        Ok(Self {
+            version: self.version.min(peer.version),
+            min_reader_version: self.min_reader_version.max(peer.min_reader_version),
+            capabilities: MrfCapabilities(self.capabilities.bits() & peer.capabilities.bits()),
+        })
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct MrfEnvelope {
+    protocol: MrfProtocolCapabilities,
+    payload: Vec<u8>,
+}
+
+impl MrfEnvelope {
+    pub fn new(protocol: MrfProtocolCapabilities, payload: Vec<u8>) -> std::result::Result<Self, MrfEnvelopeError> {
+        if protocol.version != MRF_ENVELOPE_VERSION {
+            return Err(MrfEnvelopeError::UnsupportedVersion {
+                version: protocol.version,
+            });
+        }
+        if protocol.min_reader_version > protocol.version {
+            return Err(MrfEnvelopeError::InvalidVersionRange {
+                version: protocol.version,
+                min_reader_version: protocol.min_reader_version,
+            });
+        }
+        MrfCapabilities::from_bits(protocol.capabilities.bits())?;
+        Ok(Self { protocol, payload })
+    }
+
+    pub const fn protocol(&self) -> MrfProtocolCapabilities {
+        self.protocol
+    }
+
+    pub fn payload(&self) -> &[u8] {
+        &self.payload
+    }
+
+    pub fn encode(&self) -> std::result::Result<Vec<u8>, MrfEnvelopeError> {
+        let payload_len: u32 = self.payload.len().try_into().map_err(|_| MrfEnvelopeError::PayloadTooLarge)?;
+        let mut data = Vec::with_capacity(MRF_ENVELOPE_HEADER_LEN + self.payload.len());
+        data.extend_from_slice(&MRF_ENVELOPE_MAGIC);
+        data.extend_from_slice(&MRF_ENVELOPE_FORMAT.to_le_bytes());
+        data.extend_from_slice(&self.protocol.version.to_le_bytes());
+        data.extend_from_slice(&self.protocol.min_reader_version.to_le_bytes());
+        data.extend_from_slice(&0u16.to_le_bytes());
+        data.extend_from_slice(&self.protocol.capabilities.bits().to_le_bytes());
+        data.extend_from_slice(&payload_len.to_le_bytes());
+        data.extend_from_slice(&self.payload);
+        Ok(data)
+    }
+
+    pub fn decode(data: &[u8], supported: MrfProtocolCapabilities) -> std::result::Result<Self, MrfEnvelopeError> {
+        if data.len() < MRF_ENVELOPE_HEADER_LEN {
+            return Err(MrfEnvelopeError::Truncated);
+        }
+        if data[..4] != MRF_ENVELOPE_MAGIC {
+            return Err(MrfEnvelopeError::InvalidMagic);
+        }
+        let format = LittleEndian::read_u16(&data[4..6]);
+        if format != MRF_ENVELOPE_FORMAT {
+            return Err(MrfEnvelopeError::UnsupportedFormat { format });
+        }
+        let version = LittleEndian::read_u16(&data[6..8]);
+        if version < MRF_ENVELOPE_VERSION {
+            return Err(MrfEnvelopeError::UnsupportedVersion { version });
+        }
+        let min_reader_version = LittleEndian::read_u16(&data[8..10]);
+        if min_reader_version > supported.version {
+            return Err(MrfEnvelopeError::RollbackFenced {
+                min_reader_version,
+                supported_version: supported.version,
+            });
+        }
+        if min_reader_version > version {
+            return Err(MrfEnvelopeError::InvalidVersionRange {
+                version,
+                min_reader_version,
+            });
+        }
+        let reserved = LittleEndian::read_u16(&data[10..12]);
+        if reserved != 0 {
+            return Err(MrfEnvelopeError::ReservedHeaderBits { bits: reserved });
+        }
+        let capabilities = MrfCapabilities::from_bits(LittleEndian::read_u64(&data[12..20]))?;
+        if !supported.capabilities.supports(capabilities) {
+            return Err(MrfEnvelopeError::MissingCapabilities {
+                required: capabilities.bits(),
+                available: supported.capabilities.bits(),
+            });
+        }
+        let payload_len = LittleEndian::read_u32(&data[20..24]);
+        let actual_len = data.len() - MRF_ENVELOPE_HEADER_LEN;
+        if usize::try_from(payload_len).map_err(|_| MrfEnvelopeError::PayloadTooLarge)? != actual_len {
+            return Err(MrfEnvelopeError::PayloadLengthMismatch {
+                declared: payload_len,
+                actual: actual_len,
+            });
+        }
+        Ok(Self {
+            protocol: MrfProtocolCapabilities {
+                version,
+                min_reader_version,
+                capabilities,
+            },
+            payload: data[MRF_ENVELOPE_HEADER_LEN..].to_vec(),
+        })
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum MrfEnvelopeError {
+    Truncated,
+    InvalidMagic,
+    UnsupportedFormat {
+        format: u16,
+    },
+    UnsupportedVersion {
+        version: u16,
+    },
+    InvalidVersionRange {
+        version: u16,
+        min_reader_version: u16,
+    },
+    RollbackFenced {
+        min_reader_version: u16,
+        supported_version: u16,
+    },
+    ReservedHeaderBits {
+        bits: u16,
+    },
+    UnknownCapabilities {
+        bits: u64,
+    },
+    MissingCapabilities {
+        required: u64,
+        available: u64,
+    },
+    PayloadLengthMismatch {
+        declared: u32,
+        actual: usize,
+    },
+    PayloadTooLarge,
+}
+
+impl fmt::Display for MrfEnvelopeError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Truncated => write!(f, "truncated MRF envelope"),
+            Self::InvalidMagic => write!(f, "invalid MRF envelope magic"),
+            Self::UnsupportedFormat { format } => write!(f, "unsupported MRF envelope format {format}"),
+            Self::UnsupportedVersion { version } => write!(f, "unsupported MRF envelope version {version}"),
+            Self::InvalidVersionRange {
+                version,
+                min_reader_version,
+            } => {
+                write!(f, "invalid MRF version range: version {version}, minimum reader {min_reader_version}")
+            }
+            Self::RollbackFenced {
+                min_reader_version,
+                supported_version,
+            } => write!(
+                f,
+                "MRF rollback fenced: reader version {supported_version} is below required {min_reader_version}"
+            ),
+            Self::ReservedHeaderBits { bits } => write!(f, "reserved MRF envelope header bits are set: 0x{bits:04x}"),
+            Self::UnknownCapabilities { bits } => write!(f, "unknown MRF capability bits 0x{bits:016x}"),
+            Self::MissingCapabilities { required, available } => {
+                write!(f, "MRF capabilities 0x{available:016x} do not satisfy required 0x{required:016x}")
+            }
+            Self::PayloadLengthMismatch { declared, actual } => {
+                write!(f, "MRF payload length is {actual}, expected {declared}")
+            }
+            Self::PayloadTooLarge => write!(f, "MRF payload exceeds the envelope length limit"),
+        }
+    }
+}
+
+impl std::error::Error for MrfEnvelopeError {}
+
 pub fn encode_mrf_file(entries: &[MrfReplicateEntry]) -> Result<Vec<u8>> {
    let payload = rmp_serde::to_vec_named(entries).map_err(|e| Error::Other(e.to_string()))?;
    let mut data = Vec::with_capacity(4 + payload.len());
@@ -56,6 +375,10 @@ mod tests {
    use super::*;
    use uuid::Uuid;

+    const ENVELOPE_FIXTURE: &[u8] = &[
+        b'M', b'R', b'F', b'E', 1, 0, 1, 0, 1, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 2, 3,
+    ];
+
    #[test]
    fn mrf_file_round_trips_object_metadata_and_delete_entries() {
        let obj_vid = Uuid::new_v4();
@@ -206,4 +529,128 @@ mod tests {

        assert!(matches!(decode_mrf_file(&data), Err(Error::CorruptedFormat)));
    }
+
+    #[test]
+    fn envelope_fixture_is_stable_and_round_trips() {
+        let envelope =
+            MrfEnvelope::new(MrfProtocolCapabilities::current(), vec![1, 2, 3]).expect("current MRF envelope should be valid");
+        assert_eq!(envelope.encode().expect("envelope should encode"), ENVELOPE_FIXTURE);
+        let decoded = MrfEnvelope::decode(ENVELOPE_FIXTURE, MrfProtocolCapabilities::current()).expect("fixture should decode");
+        assert_eq!(decoded.protocol(), MrfProtocolCapabilities::current());
+        assert_eq!(decoded.payload(), &[1, 2, 3]);
+    }
+
+    #[test]
+    fn envelope_accepts_a_forward_compatible_writer_version() {
+        let mut version = ENVELOPE_FIXTURE.to_vec();
+        version[6..8].copy_from_slice(&2u16.to_le_bytes());
+        let decoded =
+            MrfEnvelope::decode(&version, MrfProtocolCapabilities::current()).expect("compatible v2 envelope should decode");
+        assert_eq!(decoded.protocol().version(), 2);
+        assert_eq!(decoded.protocol().min_reader_version(), 1);
+
+        let mut fenced = version;
+        fenced[8..10].copy_from_slice(&2u16.to_le_bytes());
+        assert_eq!(
+            MrfEnvelope::decode(&fenced, MrfProtocolCapabilities::current()),
+            Err(MrfEnvelopeError::RollbackFenced {
+                min_reader_version: 2,
+                supported_version: 1,
+            })
+        );
+    }
+
+    #[test]
+    fn envelope_rejects_unknown_capability_bits() {
+        let mut capabilities = ENVELOPE_FIXTURE.to_vec();
+        capabilities[12..20].copy_from_slice(&(1u64 << 63).to_le_bytes());
+
+        assert_eq!(
+            MrfEnvelope::decode(&capabilities, MrfProtocolCapabilities::current()),
+            Err(MrfEnvelopeError::UnknownCapabilities { bits: 1u64 << 63 })
+        );
+
+        let mut reserved = ENVELOPE_FIXTURE.to_vec();
+        reserved[10..12].copy_from_slice(&1u16.to_le_bytes());
+        assert_eq!(
+            MrfEnvelope::decode(&reserved, MrfProtocolCapabilities::current()),
+            Err(MrfEnvelopeError::ReservedHeaderBits { bits: 1 })
+        );
+
+        let mut legacy = ENVELOPE_FIXTURE.to_vec();
+        legacy[6..8].copy_from_slice(&0u16.to_le_bytes());
+        assert_eq!(
+            MrfEnvelope::decode(&legacy, MrfProtocolCapabilities::current()),
+            Err(MrfEnvelopeError::UnsupportedVersion { version: 0 })
+        );
+    }
+
+    #[test]
+    fn envelope_rejects_rollback_and_missing_capabilities() {
+        let mut rollback = ENVELOPE_FIXTURE.to_vec();
+        rollback[8..10].copy_from_slice(&2u16.to_le_bytes());
+        assert_eq!(
+            MrfEnvelope::decode(&rollback, MrfProtocolCapabilities::current()),
+            Err(MrfEnvelopeError::RollbackFenced {
+                min_reader_version: 2,
+                supported_version: 1,
+            })
+        );
+
+        let required = MrfCapabilities::with(MrfCapability::TargetArns);
+        let envelope =
+            MrfEnvelope::new(MrfProtocolCapabilities::new(1, 1, required), Vec::new()).expect("known capability should be valid");
+        let encoded = envelope.encode().expect("envelope should encode");
+        assert_eq!(
+            MrfEnvelope::decode(&encoded, MrfProtocolCapabilities::new(1, 1, MrfCapabilities::empty())),
+            Err(MrfEnvelopeError::MissingCapabilities {
+                required: required.bits(),
+                available: 0,
+            })
+        );
+    }
+
+    #[test]
+    fn protocol_negotiation_fences_rollback() {
+        let current = MrfProtocolCapabilities::current();
+        let rollback = MrfProtocolCapabilities::new(1, 2, MrfCapabilities::current());
+        assert_eq!(
+            current.negotiate(rollback),
+            Err(MrfEnvelopeError::InvalidVersionRange {
+                version: 1,
+                min_reader_version: 2,
+            })
+        );
+    }
+
+    #[test]
+    fn protocol_negotiation_rejects_invalid_local_version_range() {
+        let invalid = MrfProtocolCapabilities::new(1, 2, MrfCapabilities::current());
+        assert_eq!(
+            invalid.negotiate(MrfProtocolCapabilities::current()),
+            Err(MrfEnvelopeError::InvalidVersionRange {
+                version: 1,
+                min_reader_version: 2,
+            })
+        );
+    }
+
+    #[test]
+    fn protocol_negotiation_intersects_capabilities() {
+        let local = MrfProtocolCapabilities::new(1, 1, MrfCapabilities::with(MrfCapability::TargetArns));
+        let peer = MrfProtocolCapabilities::new(1, 1, MrfCapabilities::with(MrfCapability::ForceDelete));
+        let negotiated = local.negotiate(peer).expect("same-version peers should negotiate");
+        assert_eq!(negotiated.capabilities(), MrfCapabilities::empty());
+    }
+
+    #[test]
+    fn protocol_negotiation_accepts_a_forward_compatible_peer() {
+        let reader = MrfProtocolCapabilities::current();
+        let writer = MrfProtocolCapabilities::new(2, 1, MrfCapabilities::current());
+        let negotiated = reader
+            .negotiate(writer)
+            .expect("v1 reader should negotiate with a compatible v2 writer");
+        assert_eq!(negotiated.version(), 1);
+        assert_eq!(negotiated.min_reader_version(), 1);
+    }
 }