Refactor: Introduce content checksums and improve multipart/object metadata handling (#671)

* feat:  adapt to s3s typed etag support

* refactor: move replication struct to rustfs_filemeta, fix filemeta transition bug

* add head_object checksum, filter object metadata output

* fix multipart checksum

* fix multipart checksum

* add content md5,sha256 check

* fix test

* fix cargo

---------

Co-authored-by: overtrue <anzhengchao@gmail.com>
This commit is contained in:
weisd
2025-10-20 23:46:13 +08:00
committed by GitHub
parent 46797dc815
commit cd1e244c68
56 changed files with 3331 additions and 810 deletions
+3 -1
View File
@@ -284,6 +284,7 @@ impl FileInfo {
Ok(t)
}
#[allow(clippy::too_many_arguments)]
pub fn add_object_part(
&mut self,
num: usize,
@@ -292,6 +293,7 @@ impl FileInfo {
mod_time: Option<OffsetDateTime>,
actual_size: i64,
index: Option<Bytes>,
checksums: Option<HashMap<String, String>>,
) {
let part = ObjectPartInfo {
etag,
@@ -300,7 +302,7 @@ impl FileInfo {
mod_time,
actual_size,
index,
checksums: None,
checksums,
error: None,
};
+161 -11
View File
@@ -15,9 +15,12 @@
use crate::error::{Error, Result};
use crate::fileinfo::{ErasureAlgo, ErasureInfo, FileInfo, FileInfoVersions, ObjectPartInfo, RawFileInfo};
use crate::filemeta_inline::InlineData;
use crate::{ReplicationStatusType, VersionPurgeStatusType};
use crate::{
ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_statuses_map, version_purge_statuses_map,
};
use byteorder::ByteOrder;
use bytes::Bytes;
use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS;
use rustfs_utils::http::headers::{
self, AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_STORAGE_CLASS, RESERVED_METADATA_PREFIX,
RESERVED_METADATA_PREFIX_LOWER, VERSION_PURGE_STATUS_KEY,
@@ -30,6 +33,7 @@ use std::hash::Hasher;
use std::io::{Read, Write};
use std::{collections::HashMap, io::Cursor};
use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339;
use tokio::io::AsyncRead;
use tracing::error;
use uuid::Uuid;
@@ -1742,7 +1746,25 @@ impl MetaObject {
}
}
// todo: ReplicationState,Delete
let replication_state_internal = get_internal_replication_state(&metadata);
let mut deleted = false;
if let Some(v) = replication_state_internal.as_ref() {
if !v.composite_version_purge_status().is_empty() {
deleted = true;
}
let st = v.composite_replication_status();
if !st.is_empty() {
metadata.insert(AMZ_BUCKET_REPLICATION_STATUS.to_string(), st.to_string());
}
}
let checksum = self
.meta_sys
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}crc").as_str())
.map(|v| Bytes::from(v.clone()));
let erasure = ErasureInfo {
algorithm: self.erasure_algorithm.to_string(),
@@ -1754,6 +1776,26 @@ impl MetaObject {
..Default::default()
};
let transition_status = self
.meta_sys
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}").as_str())
.map(|v| String::from_utf8_lossy(v).to_string())
.unwrap_or_default();
let transitioned_objname = self
.meta_sys
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_OBJECTNAME}").as_str())
.map(|v| String::from_utf8_lossy(v).to_string())
.unwrap_or_default();
let transition_version_id = self
.meta_sys
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_VERSION_ID}").as_str())
.map(|v| Uuid::from_slice(v.as_slice()).unwrap_or_default());
let transition_tier = self
.meta_sys
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_TIER}").as_str())
.map(|v| String::from_utf8_lossy(v).to_string())
.unwrap_or_default();
FileInfo {
version_id,
erasure,
@@ -1764,6 +1806,13 @@ impl MetaObject {
volume: volume.to_string(),
parts,
metadata,
replication_state_internal,
deleted,
checksum,
transition_status,
transitioned_objname,
transition_version_id,
transition_tier,
..Default::default()
}
}
@@ -1904,6 +1953,38 @@ impl From<FileInfo> for MetaObject {
}
}
if !value.transition_status.is_empty() {
meta_sys.insert(
format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}"),
value.transition_status.as_bytes().to_vec(),
);
}
if !value.transitioned_objname.is_empty() {
meta_sys.insert(
format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_OBJECTNAME}"),
value.transitioned_objname.as_bytes().to_vec(),
);
}
if let Some(vid) = &value.transition_version_id {
meta_sys.insert(
format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_VERSION_ID}"),
vid.as_bytes().to_vec(),
);
}
if !value.transition_tier.is_empty() {
meta_sys.insert(
format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_TIER}"),
value.transition_tier.as_bytes().to_vec(),
);
}
if let Some(content_hash) = value.checksum {
meta_sys.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}crc"), content_hash.to_vec());
}
Self {
version_id: value.version_id,
data_dir: value.data_dir,
@@ -1927,6 +2008,50 @@ impl From<FileInfo> for MetaObject {
}
}
fn get_internal_replication_state(metadata: &HashMap<String, String>) -> Option<ReplicationState> {
let mut rs = ReplicationState::default();
let mut has = false;
for (k, v) in metadata.iter() {
if k == VERSION_PURGE_STATUS_KEY {
rs.version_purge_status_internal = Some(v.clone());
rs.purge_targets = version_purge_statuses_map(v.as_str());
has = true;
continue;
}
if let Some(sub_key) = k.strip_prefix(RESERVED_METADATA_PREFIX_LOWER) {
match sub_key {
"replica-timestamp" => {
has = true;
rs.replica_timestamp = Some(OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
}
"replica-status" => {
has = true;
rs.replica_status = ReplicationStatusType::from(v.as_str());
}
"replication-timestamp" => {
has = true;
rs.replication_timestamp = Some(OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH))
}
"replication-status" => {
has = true;
rs.replication_status_internal = Some(v.clone());
rs.targets = replication_statuses_map(v.as_str());
}
_ => {
if let Some(arn) = sub_key.strip_prefix("replication-reset-") {
has = true;
rs.reset_statuses_map.insert(arn.to_string(), v.clone());
}
}
}
}
}
if has { Some(rs) } else { None }
}
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)]
pub struct MetaDeleteMarker {
#[serde(rename = "ID")]
@@ -1939,24 +2064,51 @@ pub struct MetaDeleteMarker {
impl MetaDeleteMarker {
pub fn free_version(&self) -> bool {
self.meta_sys.contains_key(FREE_VERSION_META_HEADER)
self.meta_sys
.contains_key(format!("{RESERVED_METADATA_PREFIX_LOWER}{FREE_VERSION}").as_str())
}
pub fn into_fileinfo(&self, volume: &str, path: &str, _all_parts: bool) -> FileInfo {
let metadata = self.meta_sys.clone();
let metadata = self
.meta_sys
.clone()
.into_iter()
.map(|(k, v)| (k, String::from_utf8_lossy(&v).to_string()))
.collect();
let replication_state_internal = get_internal_replication_state(&metadata);
FileInfo {
let mut fi = FileInfo {
version_id: self.version_id.filter(|&vid| !vid.is_nil()),
name: path.to_string(),
volume: volume.to_string(),
deleted: true,
mod_time: self.mod_time,
metadata: metadata
.into_iter()
.map(|(k, v)| (k, String::from_utf8_lossy(&v).to_string()))
.collect(),
metadata,
replication_state_internal,
..Default::default()
};
if self.free_version() {
fi.set_tier_free_version();
fi.transition_tier = self
.meta_sys
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_TIER}").as_str())
.map(|v| String::from_utf8_lossy(v).to_string())
.unwrap_or_default();
fi.transitioned_objname = self
.meta_sys
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_OBJECTNAME}").as_str())
.map(|v| String::from_utf8_lossy(v).to_string())
.unwrap_or_default();
fi.transition_version_id = self
.meta_sys
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_VERSION_ID}").as_str())
.map(|v| Uuid::from_slice(v.as_slice()).unwrap_or_default());
}
fi
}
pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result<u64> {
@@ -2160,8 +2312,6 @@ pub enum Flags {
InlineData = 1 << 2,
}
const FREE_VERSION_META_HEADER: &str = "free-version";
// mergeXLV2Versions
pub fn merge_file_meta_versions(
mut quorum: usize,
+396
View File
@@ -1,8 +1,36 @@
use bytes::Bytes;
use core::fmt;
use regex::Regex;
use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER;
use serde::{Deserialize, Serialize};
use std::any::Any;
use std::collections::HashMap;
use std::time::Duration;
use time::OffsetDateTime;
use uuid::Uuid;
pub const REPLICATION_RESET: &str = "replication-reset";
pub const REPLICATION_STATUS: &str = "replication-status";
// ReplicateQueued - replication being queued trail
pub const REPLICATE_QUEUED: &str = "replicate:queue";
// ReplicateExisting - audit trail for existing objects replication
pub const REPLICATE_EXISTING: &str = "replicate:existing";
// ReplicateExistingDelete - audit trail for delete replication triggered for existing delete markers
pub const REPLICATE_EXISTING_DELETE: &str = "replicate:existing:delete";
// ReplicateMRF - audit trail for replication from Most Recent Failures (MRF) queue
pub const REPLICATE_MRF: &str = "replicate:mrf";
// ReplicateIncoming - audit trail of inline replication
pub const REPLICATE_INCOMING: &str = "replicate:incoming";
// ReplicateIncomingDelete - audit trail of inline replication of deletes.
pub const REPLICATE_INCOMING_DELETE: &str = "replicate:incoming:delete";
// ReplicateHeal - audit trail for healing of failed/pending replications
pub const REPLICATE_HEAL: &str = "replicate:heal";
// ReplicateHealDelete - audit trail of healing of failed/pending delete replications.
pub const REPLICATE_HEAL_DELETE: &str = "replicate:heal:delete";
/// StatusType of Replication for x-amz-replication-status header
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, Hash)]
@@ -492,3 +520,371 @@ impl ReplicatedInfos {
ReplicationAction::None
}
}
#[derive(Serialize, Deserialize, Debug)]
pub struct MrfReplicateEntry {
#[serde(rename = "bucket")]
pub bucket: String,
#[serde(rename = "object")]
pub object: String,
#[serde(skip_serializing, skip_deserializing)]
pub version_id: Option<Uuid>,
#[serde(rename = "retryCount")]
pub retry_count: i32,
#[serde(skip_serializing, skip_deserializing)]
pub size: i64,
}
pub trait ReplicationWorkerOperation: Any + Send + Sync {
fn to_mrf_entry(&self) -> MrfReplicateEntry;
fn as_any(&self) -> &dyn Any;
fn get_bucket(&self) -> &str;
fn get_object(&self) -> &str;
fn get_size(&self) -> i64;
fn is_delete_marker(&self) -> bool;
fn get_op_type(&self) -> ReplicationType;
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ReplicateTargetDecision {
pub replicate: bool,
pub synchronous: bool,
pub arn: String,
pub id: String,
}
impl ReplicateTargetDecision {
pub fn new(arn: String, replicate: bool, sync: bool) -> Self {
Self {
replicate,
synchronous: sync,
arn,
id: String::new(),
}
}
}
impl fmt::Display for ReplicateTargetDecision {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{};{};{};{}", self.replicate, self.synchronous, self.arn, self.id)
}
}
/// ReplicateDecision represents replication decision for each target
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReplicateDecision {
pub targets_map: HashMap<String, ReplicateTargetDecision>,
}
impl ReplicateDecision {
pub fn new() -> Self {
Self {
targets_map: HashMap::new(),
}
}
/// Returns true if at least one target qualifies for replication
pub fn replicate_any(&self) -> bool {
self.targets_map.values().any(|t| t.replicate)
}
/// Returns true if at least one target qualifies for synchronous replication
pub fn is_synchronous(&self) -> bool {
self.targets_map.values().any(|t| t.synchronous)
}
/// Updates ReplicateDecision with target's replication decision
pub fn set(&mut self, target: ReplicateTargetDecision) {
self.targets_map.insert(target.arn.clone(), target);
}
/// Returns a stringified representation of internal replication status with all targets marked as `PENDING`
pub fn pending_status(&self) -> Option<String> {
let mut result = String::new();
for target in self.targets_map.values() {
if target.replicate {
result.push_str(&format!("{}={};", target.arn, ReplicationStatusType::Pending.as_str()));
}
}
if result.is_empty() { None } else { Some(result) }
}
}
impl fmt::Display for ReplicateDecision {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let mut result = String::new();
for (key, value) in &self.targets_map {
result.push_str(&format!("{key}={value},"));
}
write!(f, "{}", result.trim_end_matches(','))
}
}
impl Default for ReplicateDecision {
fn default() -> Self {
Self::new()
}
}
// parse k-v pairs of target ARN to stringified ReplicateTargetDecision delimited by ',' into a
// ReplicateDecision struct
pub fn parse_replicate_decision(_bucket: &str, s: &str) -> std::io::Result<ReplicateDecision> {
let mut decision = ReplicateDecision::new();
if s.is_empty() {
return Ok(decision);
}
for p in s.split(',') {
if p.is_empty() {
continue;
}
let slc = p.split('=').collect::<Vec<&str>>();
if slc.len() != 2 {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("invalid replicate decision format: {s}"),
));
}
let tgt_str = slc[1].trim_matches('"');
let tgt = tgt_str.split(';').collect::<Vec<&str>>();
if tgt.len() != 4 {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("invalid replicate decision format: {s}"),
));
}
let tgt = ReplicateTargetDecision {
replicate: tgt[0] == "true",
synchronous: tgt[1] == "true",
arn: tgt[2].to_string(),
id: tgt[3].to_string(),
};
decision.targets_map.insert(slc[0].to_string(), tgt);
}
Ok(decision)
// r = ReplicateDecision{
// targetsMap: make(map[string]replicateTargetDecision),
// }
// if len(s) == 0 {
// return
// }
// for _, p := range strings.Split(s, ",") {
// if p == "" {
// continue
// }
// slc := strings.Split(p, "=")
// if len(slc) != 2 {
// return r, errInvalidReplicateDecisionFormat
// }
// tgtStr := strings.TrimSuffix(strings.TrimPrefix(slc[1], `"`), `"`)
// tgt := strings.Split(tgtStr, ";")
// if len(tgt) != 4 {
// return r, errInvalidReplicateDecisionFormat
// }
// r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: tgt[0] == "true", Synchronous: tgt[1] == "true", Arn: tgt[2], ID: tgt[3]}
// }
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReplicateObjectInfo {
pub name: String,
pub size: i64,
pub actual_size: i64,
pub bucket: String,
pub version_id: Option<Uuid>,
pub etag: Option<String>,
pub mod_time: Option<OffsetDateTime>,
pub replication_status: ReplicationStatusType,
pub replication_status_internal: Option<String>,
pub delete_marker: bool,
pub version_purge_status_internal: Option<String>,
pub version_purge_status: VersionPurgeStatusType,
pub replication_state: Option<ReplicationState>,
pub op_type: ReplicationType,
pub event_type: String,
pub dsc: ReplicateDecision,
pub existing_obj_resync: ResyncDecision,
pub target_statuses: HashMap<String, ReplicationStatusType>,
pub target_purge_statuses: HashMap<String, VersionPurgeStatusType>,
pub replication_timestamp: Option<OffsetDateTime>,
pub ssec: bool,
pub user_tags: String,
pub checksum: Option<Bytes>,
pub retry_count: u32,
}
impl ReplicationWorkerOperation for ReplicateObjectInfo {
fn as_any(&self) -> &dyn Any {
self
}
fn to_mrf_entry(&self) -> MrfReplicateEntry {
MrfReplicateEntry {
bucket: self.bucket.clone(),
object: self.name.clone(),
version_id: self.version_id,
retry_count: self.retry_count as i32,
size: self.size,
}
}
fn get_bucket(&self) -> &str {
&self.bucket
}
fn get_object(&self) -> &str {
&self.name
}
fn get_size(&self) -> i64 {
self.size
}
fn is_delete_marker(&self) -> bool {
self.delete_marker
}
fn get_op_type(&self) -> ReplicationType {
self.op_type
}
}
lazy_static::lazy_static! {
static ref REPL_STATUS_REGEX: Regex = Regex::new(r"([^=].*?)=([^,].*?);").unwrap();
}
impl ReplicateObjectInfo {
/// Returns replication status of a target
pub fn target_replication_status(&self, arn: &str) -> ReplicationStatusType {
let binding = self.replication_status_internal.clone().unwrap_or_default();
let captures = REPL_STATUS_REGEX.captures_iter(&binding);
for cap in captures {
if cap.len() == 3 && &cap[1] == arn {
return ReplicationStatusType::from(&cap[2]);
}
}
ReplicationStatusType::default()
}
/// Returns the relevant info needed by MRF
pub fn to_mrf_entry(&self) -> MrfReplicateEntry {
MrfReplicateEntry {
bucket: self.bucket.clone(),
object: self.name.clone(),
version_id: self.version_id,
retry_count: self.retry_count as i32,
size: self.size,
}
}
}
// constructs a replication status map from string representation
pub fn replication_statuses_map(s: &str) -> HashMap<String, ReplicationStatusType> {
let mut targets = HashMap::new();
let rep_stat_matches = REPL_STATUS_REGEX.captures_iter(s).map(|c| c.extract());
for (_, [arn, status]) in rep_stat_matches {
if arn.is_empty() {
continue;
}
let status = ReplicationStatusType::from(status);
targets.insert(arn.to_string(), status);
}
targets
}
// constructs a version purge status map from string representation
pub fn version_purge_statuses_map(s: &str) -> HashMap<String, VersionPurgeStatusType> {
let mut targets = HashMap::new();
let purge_status_matches = REPL_STATUS_REGEX.captures_iter(s).map(|c| c.extract());
for (_, [arn, status]) in purge_status_matches {
if arn.is_empty() {
continue;
}
let status = VersionPurgeStatusType::from(status);
targets.insert(arn.to_string(), status);
}
targets
}
pub fn get_replication_state(rinfos: &ReplicatedInfos, prev_state: &ReplicationState, _vid: Option<String>) -> ReplicationState {
let reset_status_map: Vec<(String, String)> = rinfos
.targets
.iter()
.filter(|v| !v.resync_timestamp.is_empty())
.map(|t| (target_reset_header(t.arn.as_str()), t.resync_timestamp.clone()))
.collect();
let repl_statuses = rinfos.replication_status_internal();
let vpurge_statuses = rinfos.version_purge_status_internal();
let mut reset_statuses_map = prev_state.reset_statuses_map.clone();
for (key, value) in reset_status_map {
reset_statuses_map.insert(key, value);
}
ReplicationState {
replicate_decision_str: prev_state.replicate_decision_str.clone(),
reset_statuses_map,
replica_timestamp: prev_state.replica_timestamp,
replica_status: prev_state.replica_status.clone(),
targets: replication_statuses_map(&repl_statuses.clone().unwrap_or_default()),
replication_status_internal: repl_statuses,
replication_timestamp: rinfos.replication_timestamp,
purge_targets: version_purge_statuses_map(&vpurge_statuses.clone().unwrap_or_default()),
version_purge_status_internal: vpurge_statuses,
..Default::default()
}
}
pub fn target_reset_header(arn: &str) -> String {
format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}-{arn}")
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ResyncTargetDecision {
pub replicate: bool,
pub reset_id: String,
pub reset_before_date: Option<OffsetDateTime>,
}
/// ResyncDecision is a struct representing a map with target's individual resync decisions
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ResyncDecision {
pub targets: HashMap<String, ResyncTargetDecision>,
}
impl ResyncDecision {
pub fn new() -> Self {
Self { targets: HashMap::new() }
}
/// Returns true if no targets with resync decision present
pub fn is_empty(&self) -> bool {
self.targets.is_empty()
}
pub fn must_resync(&self) -> bool {
self.targets.values().any(|v| v.replicate)
}
pub fn must_resync_target(&self, tgt_arn: &str) -> bool {
self.targets.get(tgt_arn).map(|v| v.replicate).unwrap_or(false)
}
}
impl Default for ResyncDecision {
fn default() -> Self {
Self::new()
}
}