mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-20 19:42:17 +00:00
Refactor: Introduce content checksums and improve multipart/object metadata handling (#671)
* feat: adapt to s3s typed etag support * refactor: move replication struct to rustfs_filemeta, fix filemeta transition bug * add head_object checksum, filter object metadata output * fix multipart checksum * fix multipart checksum * add content md5,sha256 check * fix test * fix cargo --------- Co-authored-by: overtrue <anzhengchao@gmail.com>
This commit is contained in:
@@ -284,6 +284,7 @@ impl FileInfo {
|
||||
Ok(t)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn add_object_part(
|
||||
&mut self,
|
||||
num: usize,
|
||||
@@ -292,6 +293,7 @@ impl FileInfo {
|
||||
mod_time: Option<OffsetDateTime>,
|
||||
actual_size: i64,
|
||||
index: Option<Bytes>,
|
||||
checksums: Option<HashMap<String, String>>,
|
||||
) {
|
||||
let part = ObjectPartInfo {
|
||||
etag,
|
||||
@@ -300,7 +302,7 @@ impl FileInfo {
|
||||
mod_time,
|
||||
actual_size,
|
||||
index,
|
||||
checksums: None,
|
||||
checksums,
|
||||
error: None,
|
||||
};
|
||||
|
||||
|
||||
+161
-11
@@ -15,9 +15,12 @@
|
||||
use crate::error::{Error, Result};
|
||||
use crate::fileinfo::{ErasureAlgo, ErasureInfo, FileInfo, FileInfoVersions, ObjectPartInfo, RawFileInfo};
|
||||
use crate::filemeta_inline::InlineData;
|
||||
use crate::{ReplicationStatusType, VersionPurgeStatusType};
|
||||
use crate::{
|
||||
ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_statuses_map, version_purge_statuses_map,
|
||||
};
|
||||
use byteorder::ByteOrder;
|
||||
use bytes::Bytes;
|
||||
use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS;
|
||||
use rustfs_utils::http::headers::{
|
||||
self, AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_STORAGE_CLASS, RESERVED_METADATA_PREFIX,
|
||||
RESERVED_METADATA_PREFIX_LOWER, VERSION_PURGE_STATUS_KEY,
|
||||
@@ -30,6 +33,7 @@ use std::hash::Hasher;
|
||||
use std::io::{Read, Write};
|
||||
use std::{collections::HashMap, io::Cursor};
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
use tokio::io::AsyncRead;
|
||||
use tracing::error;
|
||||
use uuid::Uuid;
|
||||
@@ -1742,7 +1746,25 @@ impl MetaObject {
|
||||
}
|
||||
}
|
||||
|
||||
// todo: ReplicationState,Delete
|
||||
let replication_state_internal = get_internal_replication_state(&metadata);
|
||||
|
||||
let mut deleted = false;
|
||||
|
||||
if let Some(v) = replication_state_internal.as_ref() {
|
||||
if !v.composite_version_purge_status().is_empty() {
|
||||
deleted = true;
|
||||
}
|
||||
|
||||
let st = v.composite_replication_status();
|
||||
if !st.is_empty() {
|
||||
metadata.insert(AMZ_BUCKET_REPLICATION_STATUS.to_string(), st.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let checksum = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}crc").as_str())
|
||||
.map(|v| Bytes::from(v.clone()));
|
||||
|
||||
let erasure = ErasureInfo {
|
||||
algorithm: self.erasure_algorithm.to_string(),
|
||||
@@ -1754,6 +1776,26 @@ impl MetaObject {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let transition_status = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}").as_str())
|
||||
.map(|v| String::from_utf8_lossy(v).to_string())
|
||||
.unwrap_or_default();
|
||||
let transitioned_objname = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_OBJECTNAME}").as_str())
|
||||
.map(|v| String::from_utf8_lossy(v).to_string())
|
||||
.unwrap_or_default();
|
||||
let transition_version_id = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_VERSION_ID}").as_str())
|
||||
.map(|v| Uuid::from_slice(v.as_slice()).unwrap_or_default());
|
||||
let transition_tier = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_TIER}").as_str())
|
||||
.map(|v| String::from_utf8_lossy(v).to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
FileInfo {
|
||||
version_id,
|
||||
erasure,
|
||||
@@ -1764,6 +1806,13 @@ impl MetaObject {
|
||||
volume: volume.to_string(),
|
||||
parts,
|
||||
metadata,
|
||||
replication_state_internal,
|
||||
deleted,
|
||||
checksum,
|
||||
transition_status,
|
||||
transitioned_objname,
|
||||
transition_version_id,
|
||||
transition_tier,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -1904,6 +1953,38 @@ impl From<FileInfo> for MetaObject {
|
||||
}
|
||||
}
|
||||
|
||||
if !value.transition_status.is_empty() {
|
||||
meta_sys.insert(
|
||||
format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}"),
|
||||
value.transition_status.as_bytes().to_vec(),
|
||||
);
|
||||
}
|
||||
|
||||
if !value.transitioned_objname.is_empty() {
|
||||
meta_sys.insert(
|
||||
format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_OBJECTNAME}"),
|
||||
value.transitioned_objname.as_bytes().to_vec(),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(vid) = &value.transition_version_id {
|
||||
meta_sys.insert(
|
||||
format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_VERSION_ID}"),
|
||||
vid.as_bytes().to_vec(),
|
||||
);
|
||||
}
|
||||
|
||||
if !value.transition_tier.is_empty() {
|
||||
meta_sys.insert(
|
||||
format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_TIER}"),
|
||||
value.transition_tier.as_bytes().to_vec(),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(content_hash) = value.checksum {
|
||||
meta_sys.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}crc"), content_hash.to_vec());
|
||||
}
|
||||
|
||||
Self {
|
||||
version_id: value.version_id,
|
||||
data_dir: value.data_dir,
|
||||
@@ -1927,6 +2008,50 @@ impl From<FileInfo> for MetaObject {
|
||||
}
|
||||
}
|
||||
|
||||
fn get_internal_replication_state(metadata: &HashMap<String, String>) -> Option<ReplicationState> {
|
||||
let mut rs = ReplicationState::default();
|
||||
let mut has = false;
|
||||
|
||||
for (k, v) in metadata.iter() {
|
||||
if k == VERSION_PURGE_STATUS_KEY {
|
||||
rs.version_purge_status_internal = Some(v.clone());
|
||||
rs.purge_targets = version_purge_statuses_map(v.as_str());
|
||||
has = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(sub_key) = k.strip_prefix(RESERVED_METADATA_PREFIX_LOWER) {
|
||||
match sub_key {
|
||||
"replica-timestamp" => {
|
||||
has = true;
|
||||
rs.replica_timestamp = Some(OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
|
||||
}
|
||||
"replica-status" => {
|
||||
has = true;
|
||||
rs.replica_status = ReplicationStatusType::from(v.as_str());
|
||||
}
|
||||
"replication-timestamp" => {
|
||||
has = true;
|
||||
rs.replication_timestamp = Some(OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH))
|
||||
}
|
||||
"replication-status" => {
|
||||
has = true;
|
||||
rs.replication_status_internal = Some(v.clone());
|
||||
rs.targets = replication_statuses_map(v.as_str());
|
||||
}
|
||||
_ => {
|
||||
if let Some(arn) = sub_key.strip_prefix("replication-reset-") {
|
||||
has = true;
|
||||
rs.reset_statuses_map.insert(arn.to_string(), v.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if has { Some(rs) } else { None }
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)]
|
||||
pub struct MetaDeleteMarker {
|
||||
#[serde(rename = "ID")]
|
||||
@@ -1939,24 +2064,51 @@ pub struct MetaDeleteMarker {
|
||||
|
||||
impl MetaDeleteMarker {
|
||||
pub fn free_version(&self) -> bool {
|
||||
self.meta_sys.contains_key(FREE_VERSION_META_HEADER)
|
||||
self.meta_sys
|
||||
.contains_key(format!("{RESERVED_METADATA_PREFIX_LOWER}{FREE_VERSION}").as_str())
|
||||
}
|
||||
|
||||
pub fn into_fileinfo(&self, volume: &str, path: &str, _all_parts: bool) -> FileInfo {
|
||||
let metadata = self.meta_sys.clone();
|
||||
let metadata = self
|
||||
.meta_sys
|
||||
.clone()
|
||||
.into_iter()
|
||||
.map(|(k, v)| (k, String::from_utf8_lossy(&v).to_string()))
|
||||
.collect();
|
||||
let replication_state_internal = get_internal_replication_state(&metadata);
|
||||
|
||||
FileInfo {
|
||||
let mut fi = FileInfo {
|
||||
version_id: self.version_id.filter(|&vid| !vid.is_nil()),
|
||||
name: path.to_string(),
|
||||
volume: volume.to_string(),
|
||||
deleted: true,
|
||||
mod_time: self.mod_time,
|
||||
metadata: metadata
|
||||
.into_iter()
|
||||
.map(|(k, v)| (k, String::from_utf8_lossy(&v).to_string()))
|
||||
.collect(),
|
||||
metadata,
|
||||
replication_state_internal,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if self.free_version() {
|
||||
fi.set_tier_free_version();
|
||||
fi.transition_tier = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_TIER}").as_str())
|
||||
.map(|v| String::from_utf8_lossy(v).to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
fi.transitioned_objname = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_OBJECTNAME}").as_str())
|
||||
.map(|v| String::from_utf8_lossy(v).to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
fi.transition_version_id = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_VERSION_ID}").as_str())
|
||||
.map(|v| Uuid::from_slice(v.as_slice()).unwrap_or_default());
|
||||
}
|
||||
|
||||
fi
|
||||
}
|
||||
|
||||
pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result<u64> {
|
||||
@@ -2160,8 +2312,6 @@ pub enum Flags {
|
||||
InlineData = 1 << 2,
|
||||
}
|
||||
|
||||
const FREE_VERSION_META_HEADER: &str = "free-version";
|
||||
|
||||
// mergeXLV2Versions
|
||||
pub fn merge_file_meta_versions(
|
||||
mut quorum: usize,
|
||||
|
||||
@@ -1,8 +1,36 @@
|
||||
use bytes::Bytes;
|
||||
use core::fmt;
|
||||
use regex::Regex;
|
||||
use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::any::Any;
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const REPLICATION_RESET: &str = "replication-reset";
|
||||
pub const REPLICATION_STATUS: &str = "replication-status";
|
||||
|
||||
// ReplicateQueued - replication being queued trail
|
||||
pub const REPLICATE_QUEUED: &str = "replicate:queue";
|
||||
|
||||
// ReplicateExisting - audit trail for existing objects replication
|
||||
pub const REPLICATE_EXISTING: &str = "replicate:existing";
|
||||
// ReplicateExistingDelete - audit trail for delete replication triggered for existing delete markers
|
||||
pub const REPLICATE_EXISTING_DELETE: &str = "replicate:existing:delete";
|
||||
|
||||
// ReplicateMRF - audit trail for replication from Most Recent Failures (MRF) queue
|
||||
pub const REPLICATE_MRF: &str = "replicate:mrf";
|
||||
// ReplicateIncoming - audit trail of inline replication
|
||||
pub const REPLICATE_INCOMING: &str = "replicate:incoming";
|
||||
// ReplicateIncomingDelete - audit trail of inline replication of deletes.
|
||||
pub const REPLICATE_INCOMING_DELETE: &str = "replicate:incoming:delete";
|
||||
|
||||
// ReplicateHeal - audit trail for healing of failed/pending replications
|
||||
pub const REPLICATE_HEAL: &str = "replicate:heal";
|
||||
// ReplicateHealDelete - audit trail of healing of failed/pending delete replications.
|
||||
pub const REPLICATE_HEAL_DELETE: &str = "replicate:heal:delete";
|
||||
|
||||
/// StatusType of Replication for x-amz-replication-status header
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, Hash)]
|
||||
@@ -492,3 +520,371 @@ impl ReplicatedInfos {
|
||||
ReplicationAction::None
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub struct MrfReplicateEntry {
|
||||
#[serde(rename = "bucket")]
|
||||
pub bucket: String,
|
||||
|
||||
#[serde(rename = "object")]
|
||||
pub object: String,
|
||||
|
||||
#[serde(skip_serializing, skip_deserializing)]
|
||||
pub version_id: Option<Uuid>,
|
||||
|
||||
#[serde(rename = "retryCount")]
|
||||
pub retry_count: i32,
|
||||
|
||||
#[serde(skip_serializing, skip_deserializing)]
|
||||
pub size: i64,
|
||||
}
|
||||
|
||||
pub trait ReplicationWorkerOperation: Any + Send + Sync {
|
||||
fn to_mrf_entry(&self) -> MrfReplicateEntry;
|
||||
fn as_any(&self) -> &dyn Any;
|
||||
fn get_bucket(&self) -> &str;
|
||||
fn get_object(&self) -> &str;
|
||||
fn get_size(&self) -> i64;
|
||||
fn is_delete_marker(&self) -> bool;
|
||||
fn get_op_type(&self) -> ReplicationType;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ReplicateTargetDecision {
|
||||
pub replicate: bool,
|
||||
pub synchronous: bool,
|
||||
pub arn: String,
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
impl ReplicateTargetDecision {
|
||||
pub fn new(arn: String, replicate: bool, sync: bool) -> Self {
|
||||
Self {
|
||||
replicate,
|
||||
synchronous: sync,
|
||||
arn,
|
||||
id: String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ReplicateTargetDecision {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{};{};{};{}", self.replicate, self.synchronous, self.arn, self.id)
|
||||
}
|
||||
}
|
||||
|
||||
/// ReplicateDecision represents replication decision for each target
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ReplicateDecision {
|
||||
pub targets_map: HashMap<String, ReplicateTargetDecision>,
|
||||
}
|
||||
|
||||
impl ReplicateDecision {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
targets_map: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if at least one target qualifies for replication
|
||||
pub fn replicate_any(&self) -> bool {
|
||||
self.targets_map.values().any(|t| t.replicate)
|
||||
}
|
||||
|
||||
/// Returns true if at least one target qualifies for synchronous replication
|
||||
pub fn is_synchronous(&self) -> bool {
|
||||
self.targets_map.values().any(|t| t.synchronous)
|
||||
}
|
||||
|
||||
/// Updates ReplicateDecision with target's replication decision
|
||||
pub fn set(&mut self, target: ReplicateTargetDecision) {
|
||||
self.targets_map.insert(target.arn.clone(), target);
|
||||
}
|
||||
|
||||
/// Returns a stringified representation of internal replication status with all targets marked as `PENDING`
|
||||
pub fn pending_status(&self) -> Option<String> {
|
||||
let mut result = String::new();
|
||||
for target in self.targets_map.values() {
|
||||
if target.replicate {
|
||||
result.push_str(&format!("{}={};", target.arn, ReplicationStatusType::Pending.as_str()));
|
||||
}
|
||||
}
|
||||
if result.is_empty() { None } else { Some(result) }
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ReplicateDecision {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let mut result = String::new();
|
||||
for (key, value) in &self.targets_map {
|
||||
result.push_str(&format!("{key}={value},"));
|
||||
}
|
||||
write!(f, "{}", result.trim_end_matches(','))
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ReplicateDecision {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
// parse k-v pairs of target ARN to stringified ReplicateTargetDecision delimited by ',' into a
|
||||
// ReplicateDecision struct
|
||||
pub fn parse_replicate_decision(_bucket: &str, s: &str) -> std::io::Result<ReplicateDecision> {
|
||||
let mut decision = ReplicateDecision::new();
|
||||
|
||||
if s.is_empty() {
|
||||
return Ok(decision);
|
||||
}
|
||||
|
||||
for p in s.split(',') {
|
||||
if p.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let slc = p.split('=').collect::<Vec<&str>>();
|
||||
if slc.len() != 2 {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!("invalid replicate decision format: {s}"),
|
||||
));
|
||||
}
|
||||
|
||||
let tgt_str = slc[1].trim_matches('"');
|
||||
let tgt = tgt_str.split(';').collect::<Vec<&str>>();
|
||||
if tgt.len() != 4 {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!("invalid replicate decision format: {s}"),
|
||||
));
|
||||
}
|
||||
|
||||
let tgt = ReplicateTargetDecision {
|
||||
replicate: tgt[0] == "true",
|
||||
synchronous: tgt[1] == "true",
|
||||
arn: tgt[2].to_string(),
|
||||
id: tgt[3].to_string(),
|
||||
};
|
||||
decision.targets_map.insert(slc[0].to_string(), tgt);
|
||||
}
|
||||
|
||||
Ok(decision)
|
||||
|
||||
// r = ReplicateDecision{
|
||||
// targetsMap: make(map[string]replicateTargetDecision),
|
||||
// }
|
||||
// if len(s) == 0 {
|
||||
// return
|
||||
// }
|
||||
// for _, p := range strings.Split(s, ",") {
|
||||
// if p == "" {
|
||||
// continue
|
||||
// }
|
||||
// slc := strings.Split(p, "=")
|
||||
// if len(slc) != 2 {
|
||||
// return r, errInvalidReplicateDecisionFormat
|
||||
// }
|
||||
// tgtStr := strings.TrimSuffix(strings.TrimPrefix(slc[1], `"`), `"`)
|
||||
// tgt := strings.Split(tgtStr, ";")
|
||||
// if len(tgt) != 4 {
|
||||
// return r, errInvalidReplicateDecisionFormat
|
||||
// }
|
||||
// r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: tgt[0] == "true", Synchronous: tgt[1] == "true", Arn: tgt[2], ID: tgt[3]}
|
||||
// }
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ReplicateObjectInfo {
|
||||
pub name: String,
|
||||
pub size: i64,
|
||||
pub actual_size: i64,
|
||||
pub bucket: String,
|
||||
pub version_id: Option<Uuid>,
|
||||
pub etag: Option<String>,
|
||||
pub mod_time: Option<OffsetDateTime>,
|
||||
pub replication_status: ReplicationStatusType,
|
||||
pub replication_status_internal: Option<String>,
|
||||
pub delete_marker: bool,
|
||||
pub version_purge_status_internal: Option<String>,
|
||||
pub version_purge_status: VersionPurgeStatusType,
|
||||
pub replication_state: Option<ReplicationState>,
|
||||
pub op_type: ReplicationType,
|
||||
pub event_type: String,
|
||||
pub dsc: ReplicateDecision,
|
||||
pub existing_obj_resync: ResyncDecision,
|
||||
pub target_statuses: HashMap<String, ReplicationStatusType>,
|
||||
pub target_purge_statuses: HashMap<String, VersionPurgeStatusType>,
|
||||
pub replication_timestamp: Option<OffsetDateTime>,
|
||||
pub ssec: bool,
|
||||
pub user_tags: String,
|
||||
pub checksum: Option<Bytes>,
|
||||
pub retry_count: u32,
|
||||
}
|
||||
|
||||
impl ReplicationWorkerOperation for ReplicateObjectInfo {
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn to_mrf_entry(&self) -> MrfReplicateEntry {
|
||||
MrfReplicateEntry {
|
||||
bucket: self.bucket.clone(),
|
||||
object: self.name.clone(),
|
||||
version_id: self.version_id,
|
||||
retry_count: self.retry_count as i32,
|
||||
size: self.size,
|
||||
}
|
||||
}
|
||||
|
||||
fn get_bucket(&self) -> &str {
|
||||
&self.bucket
|
||||
}
|
||||
|
||||
fn get_object(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
|
||||
fn get_size(&self) -> i64 {
|
||||
self.size
|
||||
}
|
||||
|
||||
fn is_delete_marker(&self) -> bool {
|
||||
self.delete_marker
|
||||
}
|
||||
|
||||
fn get_op_type(&self) -> ReplicationType {
|
||||
self.op_type
|
||||
}
|
||||
}
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref REPL_STATUS_REGEX: Regex = Regex::new(r"([^=].*?)=([^,].*?);").unwrap();
|
||||
}
|
||||
|
||||
impl ReplicateObjectInfo {
|
||||
/// Returns replication status of a target
|
||||
pub fn target_replication_status(&self, arn: &str) -> ReplicationStatusType {
|
||||
let binding = self.replication_status_internal.clone().unwrap_or_default();
|
||||
let captures = REPL_STATUS_REGEX.captures_iter(&binding);
|
||||
for cap in captures {
|
||||
if cap.len() == 3 && &cap[1] == arn {
|
||||
return ReplicationStatusType::from(&cap[2]);
|
||||
}
|
||||
}
|
||||
ReplicationStatusType::default()
|
||||
}
|
||||
|
||||
/// Returns the relevant info needed by MRF
|
||||
pub fn to_mrf_entry(&self) -> MrfReplicateEntry {
|
||||
MrfReplicateEntry {
|
||||
bucket: self.bucket.clone(),
|
||||
object: self.name.clone(),
|
||||
version_id: self.version_id,
|
||||
retry_count: self.retry_count as i32,
|
||||
size: self.size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// constructs a replication status map from string representation
|
||||
pub fn replication_statuses_map(s: &str) -> HashMap<String, ReplicationStatusType> {
|
||||
let mut targets = HashMap::new();
|
||||
let rep_stat_matches = REPL_STATUS_REGEX.captures_iter(s).map(|c| c.extract());
|
||||
for (_, [arn, status]) in rep_stat_matches {
|
||||
if arn.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let status = ReplicationStatusType::from(status);
|
||||
targets.insert(arn.to_string(), status);
|
||||
}
|
||||
targets
|
||||
}
|
||||
|
||||
// constructs a version purge status map from string representation
|
||||
pub fn version_purge_statuses_map(s: &str) -> HashMap<String, VersionPurgeStatusType> {
|
||||
let mut targets = HashMap::new();
|
||||
let purge_status_matches = REPL_STATUS_REGEX.captures_iter(s).map(|c| c.extract());
|
||||
for (_, [arn, status]) in purge_status_matches {
|
||||
if arn.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let status = VersionPurgeStatusType::from(status);
|
||||
targets.insert(arn.to_string(), status);
|
||||
}
|
||||
targets
|
||||
}
|
||||
|
||||
pub fn get_replication_state(rinfos: &ReplicatedInfos, prev_state: &ReplicationState, _vid: Option<String>) -> ReplicationState {
|
||||
let reset_status_map: Vec<(String, String)> = rinfos
|
||||
.targets
|
||||
.iter()
|
||||
.filter(|v| !v.resync_timestamp.is_empty())
|
||||
.map(|t| (target_reset_header(t.arn.as_str()), t.resync_timestamp.clone()))
|
||||
.collect();
|
||||
|
||||
let repl_statuses = rinfos.replication_status_internal();
|
||||
let vpurge_statuses = rinfos.version_purge_status_internal();
|
||||
|
||||
let mut reset_statuses_map = prev_state.reset_statuses_map.clone();
|
||||
for (key, value) in reset_status_map {
|
||||
reset_statuses_map.insert(key, value);
|
||||
}
|
||||
|
||||
ReplicationState {
|
||||
replicate_decision_str: prev_state.replicate_decision_str.clone(),
|
||||
reset_statuses_map,
|
||||
replica_timestamp: prev_state.replica_timestamp,
|
||||
replica_status: prev_state.replica_status.clone(),
|
||||
targets: replication_statuses_map(&repl_statuses.clone().unwrap_or_default()),
|
||||
replication_status_internal: repl_statuses,
|
||||
replication_timestamp: rinfos.replication_timestamp,
|
||||
purge_targets: version_purge_statuses_map(&vpurge_statuses.clone().unwrap_or_default()),
|
||||
version_purge_status_internal: vpurge_statuses,
|
||||
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn target_reset_header(arn: &str) -> String {
|
||||
format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}-{arn}")
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ResyncTargetDecision {
|
||||
pub replicate: bool,
|
||||
pub reset_id: String,
|
||||
pub reset_before_date: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
/// ResyncDecision is a struct representing a map with target's individual resync decisions
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResyncDecision {
|
||||
pub targets: HashMap<String, ResyncTargetDecision>,
|
||||
}
|
||||
|
||||
impl ResyncDecision {
|
||||
pub fn new() -> Self {
|
||||
Self { targets: HashMap::new() }
|
||||
}
|
||||
|
||||
/// Returns true if no targets with resync decision present
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.targets.is_empty()
|
||||
}
|
||||
|
||||
pub fn must_resync(&self) -> bool {
|
||||
self.targets.values().any(|v| v.replicate)
|
||||
}
|
||||
|
||||
pub fn must_resync_target(&self, tgt_arn: &str) -> bool {
|
||||
self.targets.get(tgt_arn).map(|v| v.replicate).unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ResyncDecision {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user