// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use std::collections::HashMap; use std::fmt; use std::io::{Cursor, Read, Write}; use byteorder::{BigEndian, ByteOrder, LittleEndian}; use rmp::Marker; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; pub const RESYNC_META_FORMAT: u16 = 1; pub const RESYNC_META_VERSION: u16 = 1; const MSGP_TIME_EXT_TYPE: i8 = 5; const MSGP_TIME_LEN: u8 = 12; pub const WIRE_ZERO_TIME_UNIX: i64 = -62_135_596_800; pub type Result = std::result::Result; #[derive(Debug)] pub enum Error { CorruptedFormat, Other(String), } impl Error { fn other(err: impl Into) -> Self { Self::Other(err.into()) } } impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::CorruptedFormat => write!(f, "corrupted format"), Self::Other(err) => write!(f, "{err}"), } } } impl std::error::Error for Error {} impl From for Error { fn from(err: std::io::Error) -> Self { Self::other(err.to_string()) } } impl From for Error { fn from(err: std::string::FromUtf8Error) -> Self { Self::other(err.to_string()) } } impl From for Error { fn from(err: rmp::encode::ValueWriteError) -> Self { Self::other(err.to_string()) } } impl From for Error { fn from(err: rmp::decode::ValueReadError) -> Self { Self::other(err.to_string()) } } impl From for Error { fn from(err: rmp::decode::NumValueReadError) -> Self { Self::other(err.to_string()) } } impl From for Error { fn from(err: rmp_serde::decode::Error) -> Self { Self::other(err.to_string()) } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] pub enum ResyncStatusType { #[default] NoResync, ResyncPending, ResyncCanceled, ResyncStarted, ResyncCompleted, ResyncFailed, } impl ResyncStatusType { pub fn is_valid(&self) -> bool { *self != ResyncStatusType::NoResync } } pub fn should_auto_resume_resync(status: ResyncStatusType) -> bool { matches!(status, ResyncStatusType::ResyncPending | ResyncStatusType::ResyncStarted) } impl fmt::Display for ResyncStatusType { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let s = match self { ResyncStatusType::ResyncStarted => "Ongoing", ResyncStatusType::ResyncCompleted => "Completed", ResyncStatusType::ResyncFailed => "Failed", ResyncStatusType::ResyncPending => "Pending", ResyncStatusType::ResyncCanceled => "Canceled", ResyncStatusType::NoResync => "", }; write!(f, "{s}") } } #[derive(Debug, Clone, Default)] pub struct ResyncOpts { pub bucket: String, pub arn: String, pub resync_id: String, pub resync_before: Option, } #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct TargetReplicationResyncStatus { pub start_time: Option, pub last_update: Option, pub resync_id: String, pub resync_before_date: Option, pub resync_status: ResyncStatusType, pub failed_size: i64, pub failed_count: i64, pub replicated_size: i64, pub replicated_count: i64, pub bucket: String, pub object: String, pub error: Option, } impl TargetReplicationResyncStatus { pub fn new() -> Self { Self::default() } fn marshal_wire_msg(&self, wr: &mut Vec) -> Result<()> { rmp::encode::write_map_len(wr, 11)?; rmp::encode::write_str(wr, "st")?; write_msgp_time(wr, wire_time_or_default(self.start_time))?; rmp::encode::write_str(wr, "lst")?; write_msgp_time(wr, wire_time_or_default(self.last_update))?; rmp::encode::write_str(wr, "id")?; rmp::encode::write_str(wr, &self.resync_id)?; rmp::encode::write_str(wr, "rdt")?; write_msgp_time(wr, wire_time_or_default(self.resync_before_date))?; rmp::encode::write_str(wr, "rst")?; rmp::encode::write_i32(wr, resync_status_to_i32(self.resync_status))?; rmp::encode::write_str(wr, "fs")?; rmp::encode::write_i64(wr, self.failed_size)?; rmp::encode::write_str(wr, "frc")?; rmp::encode::write_i64(wr, self.failed_count)?; rmp::encode::write_str(wr, "rs")?; rmp::encode::write_i64(wr, self.replicated_size)?; rmp::encode::write_str(wr, "rrc")?; rmp::encode::write_i64(wr, self.replicated_count)?; rmp::encode::write_str(wr, "bkt")?; rmp::encode::write_str(wr, &self.bucket)?; rmp::encode::write_str(wr, "obj")?; rmp::encode::write_str(wr, &self.object)?; Ok(()) } fn unmarshal_wire_msg(rd: &mut R) -> Result { let mut out = Self::new(); let mut fields = rmp::decode::read_map_len(rd)?; while fields > 0 { fields -= 1; let key = read_msgp_str(rd)?; match key.as_str() { "st" => out.start_time = normalize_wire_time(read_msgp_time_or_nil(rd)?), "lst" => out.last_update = normalize_wire_time(read_msgp_time_or_nil(rd)?), "id" => out.resync_id = read_msgp_str(rd)?, "rdt" => out.resync_before_date = normalize_wire_time(read_msgp_time_or_nil(rd)?), "rst" => { let v: i32 = rmp::decode::read_int(rd)?; out.resync_status = resync_status_from_i32(v)?; } "fs" => out.failed_size = rmp::decode::read_int(rd)?, "frc" => out.failed_count = rmp::decode::read_int(rd)?, "rs" => out.replicated_size = rmp::decode::read_int(rd)?, "rrc" => out.replicated_count = rmp::decode::read_int(rd)?, "bkt" => out.bucket = read_msgp_str(rd)?, "obj" => out.object = read_msgp_str(rd)?, _ => skip_msgp_value(rd)?, } } Ok(out) } } pub fn resync_state_accepts_update(state: &TargetReplicationResyncStatus, opts: &ResyncOpts) -> bool { state.resync_id.is_empty() || opts.resync_id.is_empty() || state.resync_id == opts.resync_id } pub fn should_count_head_proxy_failure(is_not_found: bool, code: Option<&str>, raw_status: Option) -> bool { if is_not_found || matches!(code, Some("MethodNotAllowed" | "405")) { return false; } if matches!(raw_status, Some(404 | 405)) { return false; } !is_version_id_mismatch(code, raw_status) } // AWS returns 400 for root callers and 403 for IAM users when a UUID version ID // is rejected. The 403 case is safe: a real auth failure also returns 403 on the // versionId-less fallback, propagating as a hard error instead of silently skipping. pub fn is_version_id_mismatch(code: Option<&str>, raw_status: Option) -> bool { match code { Some(c) if !c.is_empty() => c == "InvalidArgument", _ => matches!(raw_status, Some(400 | 403)), } } #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct BucketReplicationResyncStatus { pub version: u16, pub targets_map: HashMap, pub id: i32, pub last_update: Option, } impl BucketReplicationResyncStatus { pub fn new() -> Self { Self { version: RESYNC_META_VERSION, ..Default::default() } } pub fn clone_tgt_stats(&self) -> HashMap { self.targets_map.clone() } pub fn marshal_msg(&self) -> Result> { let mut wr = Vec::new(); rmp::encode::write_map_len(&mut wr, 4)?; rmp::encode::write_str(&mut wr, "v")?; rmp::encode::write_i32(&mut wr, i32::from(self.version))?; rmp::encode::write_str(&mut wr, "brs")?; rmp::encode::write_map_len(&mut wr, self.targets_map.len() as u32)?; for (arn, status) in &self.targets_map { rmp::encode::write_str(&mut wr, arn)?; status.marshal_wire_msg(&mut wr)?; } rmp::encode::write_str(&mut wr, "id")?; rmp::encode::write_i32(&mut wr, self.id)?; rmp::encode::write_str(&mut wr, "lu")?; write_msgp_time(&mut wr, wire_time_or_default(self.last_update))?; Ok(wr) } pub fn unmarshal_msg(data: &[u8]) -> Result { let mut rd = Cursor::new(data); let mut out = Self::new(); let mut fields = rmp::decode::read_map_len(&mut rd)?; while fields > 0 { fields -= 1; let key = read_msgp_str(&mut rd)?; match key.as_str() { "v" => { let v: i32 = rmp::decode::read_int(&mut rd)?; out.version = u16::try_from(v).map_err(|_| Error::other("invalid resync version"))?; } "brs" => { let map_len = rmp::decode::read_map_len(&mut rd)?; let mut targets = HashMap::with_capacity(map_len as usize); for _ in 0..map_len { let arn = read_msgp_str(&mut rd)?; let status = TargetReplicationResyncStatus::unmarshal_wire_msg(&mut rd)?; targets.insert(arn, status); } out.targets_map = targets; } "id" => { out.id = rmp::decode::read_int::(&mut rd)?; } "lu" => { out.last_update = normalize_wire_time(read_msgp_time_or_nil(&mut rd)?); } _ => skip_msgp_value(&mut rd)?, } } Ok(out) } pub fn unmarshal_legacy_msg(data: &[u8]) -> Result { Ok(rmp_serde::from_slice(data)?) } } pub fn encode_resync_file(status: &BucketReplicationResyncStatus) -> Result> { let payload = status.marshal_msg()?; let mut data = Vec::with_capacity(4 + payload.len()); let mut major = [0u8; 2]; LittleEndian::write_u16(&mut major, RESYNC_META_FORMAT); data.extend_from_slice(&major); let mut minor = [0u8; 2]; LittleEndian::write_u16(&mut minor, RESYNC_META_VERSION); data.extend_from_slice(&minor); data.extend_from_slice(&payload); Ok(data) } pub fn decode_resync_file(data: &[u8]) -> Result { if data.len() <= 4 { return Err(Error::CorruptedFormat); } let mut major = [0u8; 2]; major.copy_from_slice(&data[0..2]); if LittleEndian::read_u16(&major) != RESYNC_META_FORMAT { return Err(Error::CorruptedFormat); } let mut minor = [0u8; 2]; minor.copy_from_slice(&data[2..4]); if LittleEndian::read_u16(&minor) != RESYNC_META_VERSION { return Err(Error::CorruptedFormat); } let status = match BucketReplicationResyncStatus::unmarshal_msg(&data[4..]) { Ok(v) => v, Err(_) => BucketReplicationResyncStatus::unmarshal_legacy_msg(&data[4..])?, }; if status.version != RESYNC_META_VERSION { return Err(Error::CorruptedFormat); } Ok(status) } fn wire_time_or_default(value: Option) -> OffsetDateTime { value.unwrap_or_else(wire_zero_time) } fn normalize_wire_time(value: Option) -> Option { match value { Some(v) if v == wire_zero_time() || v == OffsetDateTime::UNIX_EPOCH => None, other => other, } } fn wire_zero_time() -> OffsetDateTime { OffsetDateTime::from_unix_timestamp(WIRE_ZERO_TIME_UNIX).unwrap_or(OffsetDateTime::UNIX_EPOCH) } fn read_msgp_str(rd: &mut R) -> Result { let len = rmp::decode::read_str_len(rd)? as usize; let mut buf = vec![0u8; len]; rd.read_exact(&mut buf)?; Ok(String::from_utf8(buf)?) } fn read_msgp_time_or_nil(rd: &mut R) -> Result> { let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?; match marker { Marker::Null => Ok(None), Marker::Ext8 => Ok(Some(read_msgp_ext8_time(rd)?)), other => Err(Error::other(format!("expected time ext or nil, got marker: {other:?}"))), } } fn read_msgp_ext8_time(rd: &mut R) -> Result { let mut len_buf = [0u8; 1]; rd.read_exact(&mut len_buf)?; let len = len_buf[0] as usize; if len != MSGP_TIME_LEN as usize { return Err(Error::other(format!("invalid msgp time len: {len}"))); } let mut type_buf = [0u8; 1]; rd.read_exact(&mut type_buf)?; if type_buf[0] != MSGP_TIME_EXT_TYPE as u8 { return Err(Error::other(format!("invalid msgp time type: {}", type_buf[0]))); } let mut buf = [0u8; 12]; rd.read_exact(&mut buf)?; let sec = BigEndian::read_i64(&buf[0..8]); let nsec = BigEndian::read_u32(&buf[8..12]); OffsetDateTime::from_unix_timestamp(sec) .map_err(|_| Error::other("invalid timestamp"))? .replace_nanosecond(nsec) .map_err(|_| Error::other("invalid nanosecond")) } fn write_msgp_time(wr: &mut W, time: OffsetDateTime) -> Result<()> { wr.write_all(&[0xc7, MSGP_TIME_LEN, MSGP_TIME_EXT_TYPE as u8])?; let mut buf = [0u8; 12]; BigEndian::write_i64(&mut buf[0..8], time.unix_timestamp()); BigEndian::write_u32(&mut buf[8..12], time.nanosecond()); wr.write_all(&buf)?; Ok(()) } fn skip_msgp_value(rd: &mut R) -> Result<()> { let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?; let skip_len: usize = match marker { Marker::Null | Marker::False | Marker::True => 0, Marker::FixPos(_) | Marker::FixNeg(_) => 0, Marker::U8 => 1, Marker::U16 => 2, Marker::U32 => 4, Marker::U64 => 8, Marker::I8 => 1, Marker::I16 => 2, Marker::I32 => 4, Marker::I64 => 8, Marker::F32 => 4, Marker::F64 => 8, Marker::FixStr(n) => n as usize, Marker::Str8 | Marker::Bin8 => read_skip_len(rd, 1)?, Marker::Str16 | Marker::Bin16 => read_skip_len(rd, 2)?, Marker::Str32 | Marker::Bin32 => read_skip_len(rd, 4)?, Marker::FixArray(n) => { for _ in 0..n { skip_msgp_value(rd)?; } return Ok(()); } Marker::Array16 => { let n = read_skip_len(rd, 2)?; for _ in 0..n { skip_msgp_value(rd)?; } return Ok(()); } Marker::Array32 => { let n = read_skip_len(rd, 4)?; for _ in 0..n { skip_msgp_value(rd)?; } return Ok(()); } Marker::FixMap(n) => { for _ in 0..n { skip_msgp_value(rd)?; skip_msgp_value(rd)?; } return Ok(()); } Marker::Map16 => { let n = read_skip_len(rd, 2)?; for _ in 0..n { skip_msgp_value(rd)?; skip_msgp_value(rd)?; } return Ok(()); } Marker::Map32 => { let n = read_skip_len(rd, 4)?; for _ in 0..n { skip_msgp_value(rd)?; skip_msgp_value(rd)?; } return Ok(()); } Marker::FixExt1 => 1 + 1, Marker::FixExt2 => 1 + 2, Marker::FixExt4 => 1 + 4, Marker::FixExt8 => 1 + 8, Marker::FixExt16 => 1 + 16, Marker::Ext8 => 1 + read_skip_len(rd, 1)?, Marker::Ext16 => 1 + read_skip_len(rd, 2)?, Marker::Ext32 => 1 + read_skip_len(rd, 4)?, Marker::Reserved => 0, }; if skip_len > 0 { skip_exact(rd, skip_len)?; } Ok(()) } fn skip_exact(rd: &mut R, mut len: usize) -> Result<()> { let mut buf = [0u8; 1024]; while len > 0 { let take = len.min(buf.len()); rd.read_exact(&mut buf[..take])?; len -= take; } Ok(()) } fn read_skip_len(rd: &mut R, bytes: usize) -> Result { let mut buf = vec![0u8; bytes]; rd.read_exact(&mut buf)?; let len = match bytes { 1 => buf[0] as usize, 2 => u16::from_be_bytes([buf[0], buf[1]]) as usize, 4 => u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]) as usize, _ => return Err(Error::other("invalid MessagePack length width")), }; Ok(len) } fn resync_status_to_i32(status: ResyncStatusType) -> i32 { match status { ResyncStatusType::NoResync => 0, ResyncStatusType::ResyncPending => 1, ResyncStatusType::ResyncCanceled => 2, ResyncStatusType::ResyncStarted => 3, ResyncStatusType::ResyncCompleted => 4, ResyncStatusType::ResyncFailed => 5, } } fn resync_status_from_i32(code: i32) -> Result { match code { 0 => Ok(ResyncStatusType::NoResync), 1 => Ok(ResyncStatusType::ResyncPending), 2 => Ok(ResyncStatusType::ResyncCanceled), 3 => Ok(ResyncStatusType::ResyncStarted), 4 => Ok(ResyncStatusType::ResyncCompleted), 5 => Ok(ResyncStatusType::ResyncFailed), _ => Err(Error::other(format!("invalid resync status code: {code}"))), } } #[cfg(test)] mod tests { use super::*; #[test] fn resync_status_display_matches_admin_contract() { assert_eq!(ResyncStatusType::ResyncStarted.to_string(), "Ongoing"); assert_eq!(ResyncStatusType::ResyncCompleted.to_string(), "Completed"); assert_eq!(ResyncStatusType::NoResync.to_string(), ""); } #[test] fn auto_resume_resync_only_for_inflight_states() { assert!(should_auto_resume_resync(ResyncStatusType::ResyncPending)); assert!(should_auto_resume_resync(ResyncStatusType::ResyncStarted)); assert!(!should_auto_resume_resync(ResyncStatusType::NoResync)); assert!(!should_auto_resume_resync(ResyncStatusType::ResyncCanceled)); assert!(!should_auto_resume_resync(ResyncStatusType::ResyncCompleted)); assert!(!should_auto_resume_resync(ResyncStatusType::ResyncFailed)); } #[test] fn resync_state_accepts_update_only_for_matching_run() { let current = TargetReplicationResyncStatus { resync_id: "run-new".to_string(), ..Default::default() }; let matching = ResyncOpts { bucket: "bucket".to_string(), arn: "arn:replication::dest".to_string(), resync_id: "run-new".to_string(), resync_before: None, }; let stale = ResyncOpts { bucket: "bucket".to_string(), arn: "arn:replication::dest".to_string(), resync_id: "run-old".to_string(), resync_before: None, }; assert!(resync_state_accepts_update(&TargetReplicationResyncStatus::default(), &matching)); assert!(resync_state_accepts_update(¤t, &matching)); assert!(!resync_state_accepts_update(¤t, &stale)); } #[test] fn head_proxy_failure_ignores_expected_target_responses() { assert!(!should_count_head_proxy_failure(true, Some("NoSuchKey"), Some(404))); assert!(!should_count_head_proxy_failure(false, Some("MethodNotAllowed"), Some(405))); assert!(!should_count_head_proxy_failure(false, Some("405"), Some(405))); assert!(!should_count_head_proxy_failure(false, Some("InvalidArgument"), Some(400))); assert!(!should_count_head_proxy_failure(false, None, Some(400))); assert!(!should_count_head_proxy_failure(false, None, Some(403))); } #[test] fn head_proxy_failure_counts_unexpected_errors() { assert!(should_count_head_proxy_failure(false, Some("AccessDenied"), Some(403))); assert!(should_count_head_proxy_failure(false, None, Some(500))); } #[test] fn version_id_mismatch_detects_aws_rejections() { assert!(is_version_id_mismatch(Some("InvalidArgument"), Some(400))); assert!(is_version_id_mismatch(None, Some(400))); assert!(is_version_id_mismatch(Some(""), Some(400))); assert!(is_version_id_mismatch(None, Some(403))); assert!(is_version_id_mismatch(Some(""), Some(403))); assert!(!is_version_id_mismatch(Some("AccessDenied"), Some(403))); assert!(!is_version_id_mismatch(Some("NoSuchKey"), Some(404))); assert!(!is_version_id_mismatch(Some("MalformedXML"), Some(400))); assert!(!is_version_id_mismatch(Some("EntityTooLarge"), Some(400))); assert!(!is_version_id_mismatch(None, Some(500))); assert!(!is_version_id_mismatch(None, Some(404))); } #[test] fn resync_file_round_trips_status() { let mut status = BucketReplicationResyncStatus::new(); status.targets_map.insert( "arn:replication:a".to_string(), TargetReplicationResyncStatus { resync_id: "rid-1".to_string(), resync_status: ResyncStatusType::ResyncStarted, bucket: "bucket-a".to_string(), object: "object-a".to_string(), replicated_count: 7, ..Default::default() }, ); let data = encode_resync_file(&status).expect("resync status should encode"); let got = decode_resync_file(&data).expect("resync status should decode"); assert_eq!(got.version, RESYNC_META_VERSION); assert_eq!(got.targets_map["arn:replication:a"].resync_id, "rid-1"); assert_eq!(got.targets_map["arn:replication:a"].resync_status, ResyncStatusType::ResyncStarted); assert_eq!(got.targets_map["arn:replication:a"].replicated_count, 7); } #[test] fn skip_msgp_value_consumes_ext_type_and_payload() { let mut ext16 = Cursor::new(vec![0xc8, 0, 2, MSGP_TIME_EXT_TYPE as u8, 0xaa, 0xbb, 0x01]); skip_msgp_value(&mut ext16).expect("ext16 should skip"); assert!(matches!(rmp::decode::read_marker(&mut ext16), Ok(Marker::FixPos(1)))); let mut ext32 = Cursor::new(vec![0xc9, 0, 0, 0, 2, MSGP_TIME_EXT_TYPE as u8, 0xaa, 0xbb, 0x01]); skip_msgp_value(&mut ext32).expect("ext32 should skip"); assert!(matches!(rmp::decode::read_marker(&mut ext32), Ok(Marker::FixPos(1)))); } #[test] fn skip_msgp_value_consumes_fixext_type_and_payload() { for (marker, payload_len) in [(0xd4, 1), (0xd5, 2), (0xd6, 4), (0xd7, 8), (0xd8, 16)] { let mut data = vec![marker, MSGP_TIME_EXT_TYPE as u8]; data.resize(data.len() + payload_len, 0xaa); data.push(0x01); let mut cursor = Cursor::new(data); skip_msgp_value(&mut cursor).expect("fixext should skip"); assert!(matches!(rmp::decode::read_marker(&mut cursor), Ok(Marker::FixPos(1)))); } } }