gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-18 18:46:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: Introduce content checksums and improve multipart/object metadata handling (#671)

Browse Source 
* feat:  adapt to s3s typed etag support

* refactor: move replication struct to rustfs_filemeta, fix filemeta transition bug

* add head_object checksum, filter object metadata output

* fix multipart checksum

* fix multipart checksum

* add content md5,sha256 check

* fix test

* fix cargo

---------

Co-authored-by: overtrue <anzhengchao@gmail.com>
This commit is contained in:
weisd
2025-10-20 23:46:13 +08:00
committed by GitHub
parent 46797dc815
commit cd1e244c68
56 changed files with 3331 additions and 810 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/bucket/replication/mod.rs
-2
View File
@@ -17,12 +17,10 @@ pub mod datatypes;
mod replication_pool;
mod replication_resyncer;
mod replication_state;
mod replication_type;
mod rule;
pub use config::*;
pub use datatypes::*;
pub use replication_pool::*;
pub use replication_resyncer::*;
pub use replication_type::*;
pub use rule::*;
crates/ecstore/src/bucket/replication/replication_pool.rs
+8 -8
View File
@@ -1,9 +1,4 @@
use crate::StorageAPI;
use crate::bucket::replication::MrfReplicateEntry;
use crate::bucket::replication::ReplicateDecision;
use crate::bucket::replication::ReplicateObjectInfo;
use crate::bucket::replication::ReplicationWorkerOperation;
use crate::bucket::replication::ResyncDecision;
use crate::bucket::replication::ResyncOpts;
use crate::bucket::replication::ResyncStatusType;
use crate::bucket::replication::replicate_delete;
@@ -18,16 +13,21 @@ use crate::bucket::replication::replication_resyncer::{
BucketReplicationResyncStatus, DeletedObjectReplicationInfo, ReplicationResyncer,
};
use crate::bucket::replication::replication_state::ReplicationStats;
use crate::bucket::replication::replication_statuses_map;
use crate::bucket::replication::version_purge_statuses_map;
use crate::config::com::read_config;
use crate::error::Error as EcstoreError;
use crate::store_api::ObjectInfo;
use lazy_static::lazy_static;
use rustfs_filemeta::MrfReplicateEntry;
use rustfs_filemeta::ReplicateDecision;
use rustfs_filemeta::ReplicateObjectInfo;
use rustfs_filemeta::ReplicatedTargetInfo;
use rustfs_filemeta::ReplicationStatusType;
use rustfs_filemeta::ReplicationType;
use rustfs_filemeta::ReplicationWorkerOperation;
use rustfs_filemeta::ResyncDecision;
use rustfs_filemeta::replication_statuses_map;
use rustfs_filemeta::version_purge_statuses_map;
use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER;
use time::OffsetDateTime;
use time::format_description::well_known::Rfc3339;
@@ -996,7 +996,7 @@ pub async fn schedule_replication<S: StorageAPI>(oi: ObjectInfo, o: Arc<S>, dsc:
target_purge_statuses: purge_statuses,
replication_timestamp: tm,
user_tags: oi.user_tags,
checksum: vec![],
checksum: None,
retry_count: 0,
event_type: "".to_string(),
existing_obj_resync: ResyncDecision::default(),
crates/ecstore/src/bucket/replication/replication_resyncer.rs
+65 -16
View File
@@ -2,12 +2,8 @@ use crate::bucket::bucket_target_sys::{
AdvancedPutOptions, BucketTargetSys, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
};
use crate::bucket::metadata_sys;
use crate::bucket::replication::{MrfReplicateEntry, ReplicationWorkerOperation, ResyncStatusType};
use crate::bucket::replication::{
ObjectOpts, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATION_RESET, ReplicateObjectInfo,
ReplicationConfigurationExt as _, ResyncTargetDecision, get_replication_state, parse_replicate_decision,
replication_statuses_map, target_reset_header, version_purge_statuses_map,
};
use crate::bucket::replication::ResyncStatusType;
use crate::bucket::replication::{ObjectOpts, ReplicationConfigurationExt as _};
use crate::bucket::tagging::decode_tags_to_map;
use crate::bucket::target::BucketTargets;
use crate::bucket::versioning_sys::BucketVersioningSys;
@@ -29,14 +25,17 @@ use byteorder::ByteOrder;
use futures::future::join_all;
use http::HeaderMap;
use regex::Regex;
use rustfs_filemeta::{
ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType, ReplicationType,
VersionPurgeStatusType,
MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATION_RESET, ReplicateDecision, ReplicateObjectInfo,
ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType,
ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType,
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
};
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _,
RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER, RUSTFS_REPLICATION_AUTUAL_OBJECT_SIZE, SSEC_ALGORITHM_HEADER,
SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, headers,
RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER, RUSTFS_REPLICATION_AUTUAL_OBJECT_SIZE,
RUSTFS_REPLICATION_RESET_STATUS, SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, headers,
};
use rustfs_utils::path::path_join_buf;
use rustfs_utils::string::strings_has_prefix_fold;
@@ -56,9 +55,6 @@ use tokio::time::Duration as TokioDuration;
use tokio_util::sync::CancellationToken;
use tracing::{error, info, warn};
use super::replication_type::{ReplicateDecision, ReplicateTargetDecision, ResyncDecision};
use regex::Regex;
const REPLICATION_DIR: &str = ".replication";
const RESYNC_FILE_NAME: &str = "resync.bin";
const RESYNC_META_FORMAT: u16 = 1;
@@ -663,7 +659,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
replication_timestamp: None,
ssec: false, // TODO: add ssec support
user_tags: oi.user_tags.clone(),
checksum: Vec::new(),
checksum: oi.checksum.clone(),
retry_count: 0,
}
}
@@ -849,7 +845,7 @@ impl ReplicationConfig {
{
resync_decision.targets.insert(
decision.arn.clone(),
ResyncTargetDecision::resync_target(
resync_target(
&oi,
&target.arn,
&target.reset_id,
@@ -864,6 +860,59 @@ impl ReplicationConfig {
}
}
pub fn resync_target(
oi: &ObjectInfo,
arn: &str,
reset_id: &str,
reset_before_date: Option<OffsetDateTime>,
status: ReplicationStatusType,
) -> ResyncTargetDecision {
let rs = oi
.user_defined
.get(target_reset_header(arn).as_str())
.or(oi.user_defined.get(RUSTFS_REPLICATION_RESET_STATUS))
.map(|s| s.to_string());
let mut dec = ResyncTargetDecision::default();
let mod_time = oi.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
if rs.is_none() {
let reset_before_date = reset_before_date.unwrap_or(OffsetDateTime::UNIX_EPOCH);
if !reset_id.is_empty() && mod_time < reset_before_date {
dec.replicate = true;
return dec;
}
dec.replicate = status == ReplicationStatusType::Empty;
return dec;
}
if reset_id.is_empty() || reset_before_date.is_none() {
return dec;
}
let rs = rs.unwrap();
let reset_before_date = reset_before_date.unwrap();
let parts: Vec<&str> = rs.splitn(2, ';').collect();
if parts.len() != 2 {
return dec;
}
let new_reset = parts[0] == reset_id;
if !new_reset && status == ReplicationStatusType::Completed {
return dec;
}
dec.replicate = new_reset && mod_time < reset_before_date;
dec
}
pub struct MustReplicateOptions {
meta: HashMap<String, String>,
status: ReplicationStatusType,
@@ -933,7 +982,7 @@ pub async fn check_replicate_delete(
let rcfg = match get_replication_config(bucket).await {
Ok(Some(config)) => config,
Ok(None) => {
warn!("No replication config found for bucket: {}", bucket);
// warn!("No replication config found for bucket: {}", bucket);
return ReplicateDecision::default();
}
Err(err) => {
crates/ecstore/src/bucket/replication/replication_type.rs
-470
View File
@@ -1,470 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::error::{Error, Result};
use crate::store_api::ObjectInfo;
use regex::Regex;
use rustfs_filemeta::VersionPurgeStatusType;
use rustfs_filemeta::{ReplicatedInfos, ReplicationType};
use rustfs_filemeta::{ReplicationState, ReplicationStatusType};
use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER;
use rustfs_utils::http::RUSTFS_REPLICATION_RESET_STATUS;
use serde::{Deserialize, Serialize};
use std::any::Any;
use std::collections::HashMap;
use std::fmt;
use time::OffsetDateTime;
use uuid::Uuid;
pub const REPLICATION_RESET: &str = "replication-reset";
pub const REPLICATION_STATUS: &str = "replication-status";
// ReplicateQueued - replication being queued trail
pub const REPLICATE_QUEUED: &str = "replicate:queue";
// ReplicateExisting - audit trail for existing objects replication
pub const REPLICATE_EXISTING: &str = "replicate:existing";
// ReplicateExistingDelete - audit trail for delete replication triggered for existing delete markers
pub const REPLICATE_EXISTING_DELETE: &str = "replicate:existing:delete";
// ReplicateMRF - audit trail for replication from Most Recent Failures (MRF) queue
pub const REPLICATE_MRF: &str = "replicate:mrf";
// ReplicateIncoming - audit trail of inline replication
pub const REPLICATE_INCOMING: &str = "replicate:incoming";
// ReplicateIncomingDelete - audit trail of inline replication of deletes.
pub const REPLICATE_INCOMING_DELETE: &str = "replicate:incoming:delete";
// ReplicateHeal - audit trail for healing of failed/pending replications
pub const REPLICATE_HEAL: &str = "replicate:heal";
// ReplicateHealDelete - audit trail of healing of failed/pending delete replications.
pub const REPLICATE_HEAL_DELETE: &str = "replicate:heal:delete";
#[derive(Serialize, Deserialize, Debug)]
pub struct MrfReplicateEntry {
#[serde(rename = "bucket")]
pub bucket: String,
#[serde(rename = "object")]
pub object: String,
#[serde(skip_serializing, skip_deserializing)]
pub version_id: Option<Uuid>,
#[serde(rename = "retryCount")]
pub retry_count: i32,
#[serde(skip_serializing, skip_deserializing)]
pub size: i64,
}
pub trait ReplicationWorkerOperation: Any + Send + Sync {
fn to_mrf_entry(&self) -> MrfReplicateEntry;
fn as_any(&self) -> &dyn Any;
fn get_bucket(&self) -> &str;
fn get_object(&self) -> &str;
fn get_size(&self) -> i64;
fn is_delete_marker(&self) -> bool;
fn get_op_type(&self) -> ReplicationType;
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ReplicateTargetDecision {
pub replicate: bool,
pub synchronous: bool,
pub arn: String,
pub id: String,
}
impl ReplicateTargetDecision {
pub fn new(arn: String, replicate: bool, sync: bool) -> Self {
Self {
replicate,
synchronous: sync,
arn,
id: String::new(),
}
}
}
impl fmt::Display for ReplicateTargetDecision {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{};{};{};{}", self.replicate, self.synchronous, self.arn, self.id)
}
}
/// ReplicateDecision represents replication decision for each target
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReplicateDecision {
pub targets_map: HashMap<String, ReplicateTargetDecision>,
}
impl ReplicateDecision {
pub fn new() -> Self {
Self {
targets_map: HashMap::new(),
}
}
/// Returns true if at least one target qualifies for replication
pub fn replicate_any(&self) -> bool {
self.targets_map.values().any(|t| t.replicate)
}
/// Returns true if at least one target qualifies for synchronous replication
pub fn is_synchronous(&self) -> bool {
self.targets_map.values().any(|t| t.synchronous)
}
/// Updates ReplicateDecision with target's replication decision
pub fn set(&mut self, target: ReplicateTargetDecision) {
self.targets_map.insert(target.arn.clone(), target);
}
/// Returns a stringified representation of internal replication status with all targets marked as `PENDING`
pub fn pending_status(&self) -> Option<String> {
let mut result = String::new();
for target in self.targets_map.values() {
if target.replicate {
result.push_str(&format!("{}={};", target.arn, ReplicationStatusType::Pending.as_str()));
}
}
if result.is_empty() { None } else { Some(result) }
}
}
impl fmt::Display for ReplicateDecision {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let mut result = String::new();
for (key, value) in &self.targets_map {
result.push_str(&format!("{key}={value},"));
}
write!(f, "{}", result.trim_end_matches(','))
}
}
impl Default for ReplicateDecision {
fn default() -> Self {
Self::new()
}
}
// parse k-v pairs of target ARN to stringified ReplicateTargetDecision delimited by ',' into a
// ReplicateDecision struct
pub fn parse_replicate_decision(_bucket: &str, s: &str) -> Result<ReplicateDecision> {
let mut decision = ReplicateDecision::new();
if s.is_empty() {
return Ok(decision);
}
for p in s.split(',') {
if p.is_empty() {
continue;
}
let slc = p.split('=').collect::<Vec<&str>>();
if slc.len() != 2 {
return Err(Error::other(format!("invalid replicate decision format: {s}")));
}
let tgt_str = slc[1].trim_matches('"');
let tgt = tgt_str.split(';').collect::<Vec<&str>>();
if tgt.len() != 4 {
return Err(Error::other(format!("invalid replicate decision format: {s}")));
}
let tgt = ReplicateTargetDecision {
replicate: tgt[0] == "true",
synchronous: tgt[1] == "true",
arn: tgt[2].to_string(),
id: tgt[3].to_string(),
};
decision.targets_map.insert(slc[0].to_string(), tgt);
}
Ok(decision)
// r = ReplicateDecision{
// targetsMap: make(map[string]replicateTargetDecision),
// }
// if len(s) == 0 {
// return
// }
// for _, p := range strings.Split(s, ",") {
// if p == "" {
// continue
// }
// slc := strings.Split(p, "=")
// if len(slc) != 2 {
// return r, errInvalidReplicateDecisionFormat
// }
// tgtStr := strings.TrimSuffix(strings.TrimPrefix(slc[1], `"`), `"`)
// tgt := strings.Split(tgtStr, ";")
// if len(tgt) != 4 {
// return r, errInvalidReplicateDecisionFormat
// }
// r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: tgt[0] == "true", Synchronous: tgt[1] == "true", Arn: tgt[2], ID: tgt[3]}
// }
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct ResyncTargetDecision {
pub replicate: bool,
pub reset_id: String,
pub reset_before_date: Option<OffsetDateTime>,
}
pub fn target_reset_header(arn: &str) -> String {
format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}-{arn}")
}
impl ResyncTargetDecision {
pub fn resync_target(
oi: &ObjectInfo,
arn: &str,
reset_id: &str,
reset_before_date: Option<OffsetDateTime>,
status: ReplicationStatusType,
) -> Self {
let rs = oi
.user_defined
.get(target_reset_header(arn).as_str())
.or(oi.user_defined.get(RUSTFS_REPLICATION_RESET_STATUS))
.map(|s| s.to_string());
let mut dec = Self::default();
let mod_time = oi.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
if rs.is_none() {
let reset_before_date = reset_before_date.unwrap_or(OffsetDateTime::UNIX_EPOCH);
if !reset_id.is_empty() && mod_time < reset_before_date {
dec.replicate = true;
return dec;
}
dec.replicate = status == ReplicationStatusType::Empty;
return dec;
}
if reset_id.is_empty() || reset_before_date.is_none() {
return dec;
}
let rs = rs.unwrap();
let reset_before_date = reset_before_date.unwrap();
let parts: Vec<&str> = rs.splitn(2, ';').collect();
if parts.len() != 2 {
return dec;
}
let new_reset = parts[0] == reset_id;
if !new_reset && status == ReplicationStatusType::Completed {
return dec;
}
dec.replicate = new_reset && mod_time < reset_before_date;
dec
}
}
/// ResyncDecision is a struct representing a map with target's individual resync decisions
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ResyncDecision {
pub targets: HashMap<String, ResyncTargetDecision>,
}
impl ResyncDecision {
pub fn new() -> Self {
Self { targets: HashMap::new() }
}
/// Returns true if no targets with resync decision present
pub fn is_empty(&self) -> bool {
self.targets.is_empty()
}
pub fn must_resync(&self) -> bool {
self.targets.values().any(|v| v.replicate)
}
pub fn must_resync_target(&self, tgt_arn: &str) -> bool {
self.targets.get(tgt_arn).map(|v| v.replicate).unwrap_or(false)
}
}
impl Default for ResyncDecision {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReplicateObjectInfo {
pub name: String,
pub size: i64,
pub actual_size: i64,
pub bucket: String,
pub version_id: Option<Uuid>,
pub etag: Option<String>,
pub mod_time: Option<OffsetDateTime>,
pub replication_status: ReplicationStatusType,
pub replication_status_internal: Option<String>,
pub delete_marker: bool,
pub version_purge_status_internal: Option<String>,
pub version_purge_status: VersionPurgeStatusType,
pub replication_state: Option<ReplicationState>,
pub op_type: ReplicationType,
pub event_type: String,
pub dsc: ReplicateDecision,
pub existing_obj_resync: ResyncDecision,
pub target_statuses: HashMap<String, ReplicationStatusType>,
pub target_purge_statuses: HashMap<String, VersionPurgeStatusType>,
pub replication_timestamp: Option<OffsetDateTime>,
pub ssec: bool,
pub user_tags: String,
pub checksum: Vec<u8>,
pub retry_count: u32,
}
impl ReplicationWorkerOperation for ReplicateObjectInfo {
fn as_any(&self) -> &dyn Any {
self
}
fn to_mrf_entry(&self) -> MrfReplicateEntry {
MrfReplicateEntry {
bucket: self.bucket.clone(),
object: self.name.clone(),
version_id: self.version_id,
retry_count: self.retry_count as i32,
size: self.size,
}
}
fn get_bucket(&self) -> &str {
&self.bucket
}
fn get_object(&self) -> &str {
&self.name
}
fn get_size(&self) -> i64 {
self.size
}
fn is_delete_marker(&self) -> bool {
self.delete_marker
}
fn get_op_type(&self) -> ReplicationType {
self.op_type
}
}
lazy_static::lazy_static! {
static ref REPL_STATUS_REGEX: Regex = Regex::new(r"([^=].*?)=([^,].*?);").unwrap();
}
impl ReplicateObjectInfo {
/// Returns replication status of a target
pub fn target_replication_status(&self, arn: &str) -> ReplicationStatusType {
let binding = self.replication_status_internal.clone().unwrap_or_default();
let captures = REPL_STATUS_REGEX.captures_iter(&binding);
for cap in captures {
if cap.len() == 3 && &cap[1] == arn {
return ReplicationStatusType::from(&cap[2]);
}
}
ReplicationStatusType::default()
}
/// Returns the relevant info needed by MRF
pub fn to_mrf_entry(&self) -> MrfReplicateEntry {
MrfReplicateEntry {
bucket: self.bucket.clone(),
object: self.name.clone(),
version_id: self.version_id,
retry_count: self.retry_count as i32,
size: self.size,
}
}
}
// constructs a replication status map from string representation
pub fn replication_statuses_map(s: &str) -> HashMap<String, ReplicationStatusType> {
let mut targets = HashMap::new();
let rep_stat_matches = REPL_STATUS_REGEX.captures_iter(s).map(|c| c.extract());
for (_, [arn, status]) in rep_stat_matches {
if arn.is_empty() {
continue;
}
let status = ReplicationStatusType::from(status);
targets.insert(arn.to_string(), status);
}
targets
}
// constructs a version purge status map from string representation
pub fn version_purge_statuses_map(s: &str) -> HashMap<String, VersionPurgeStatusType> {
let mut targets = HashMap::new();
let purge_status_matches = REPL_STATUS_REGEX.captures_iter(s).map(|c| c.extract());
for (_, [arn, status]) in purge_status_matches {
if arn.is_empty() {
continue;
}
let status = VersionPurgeStatusType::from(status);
targets.insert(arn.to_string(), status);
}
targets
}
pub fn get_replication_state(rinfos: &ReplicatedInfos, prev_state: &ReplicationState, _vid: Option<String>) -> ReplicationState {
let reset_status_map: Vec<(String, String)> = rinfos
.targets
.iter()
.filter(|v| !v.resync_timestamp.is_empty())
.map(|t| (target_reset_header(t.arn.as_str()), t.resync_timestamp.clone()))
.collect();
let repl_statuses = rinfos.replication_status_internal();
let vpurge_statuses = rinfos.version_purge_status_internal();
let mut reset_statuses_map = prev_state.reset_statuses_map.clone();
for (key, value) in reset_status_map {
reset_statuses_map.insert(key, value);
}
ReplicationState {
replicate_decision_str: prev_state.replicate_decision_str.clone(),
reset_statuses_map,
replica_timestamp: prev_state.replica_timestamp,
replica_status: prev_state.replica_status.clone(),
targets: replication_statuses_map(&repl_statuses.clone().unwrap_or_default()),
replication_status_internal: repl_statuses,
replication_timestamp: rinfos.replication_timestamp,
purge_targets: version_purge_statuses_map(&vpurge_statuses.clone().unwrap_or_default()),
version_purge_status_internal: vpurge_statuses,
..Default::default()
}
}
crates/ecstore/src/client/api_put_object.rs
+2 -2
View File
@@ -30,7 +30,8 @@ use s3s::header::{
X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION,
};
//use crate::disk::{BufferReader, Reader};
use crate::checksum::ChecksumMode;
use crate::client::checksum::ChecksumMode;
use crate::client::utils::base64_encode;
use crate::client::{
api_error_response::{err_entity_too_large, err_invalid_argument},
api_put_object_common::optimal_part_info,
@@ -41,7 +42,6 @@ use crate::client::{
transition_api::{ReaderImpl, TransitionClient, UploadInfo},
utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
};
use rustfs_utils::crypto::base64_encode;
#[derive(Debug, Clone)]
pub struct AdvancedPutOptions {
crates/ecstore/src/client/api_put_object_multipart.rs
+3 -2
View File
@@ -25,7 +25,8 @@ use time::OffsetDateTime;
use tracing::warn;
use uuid::Uuid;
use crate::checksum::ChecksumMode;
use crate::client::checksum::ChecksumMode;
use crate::client::utils::base64_encode;
use crate::client::{
api_error_response::{
err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response,
@@ -38,7 +39,7 @@ use crate::client::{
constants::{ISO8601_DATEFORMAT, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE},
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
};
use rustfs_utils::{crypto::base64_encode, path::trim_etag};
use rustfs_utils::path::trim_etag;
use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
impl TransitionClient {
crates/ecstore/src/client/api_put_object_streaming.rs
+3 -2
View File
@@ -29,7 +29,7 @@ use tokio_util::sync::CancellationToken;
use tracing::warn;
use uuid::Uuid;
use crate::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
use crate::client::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
use crate::client::{
api_error_response::{err_invalid_argument, err_unexpected_eof, http_resp_to_error_response},
api_put_object::PutObjectOptions,
@@ -40,7 +40,8 @@ use crate::client::{
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
};
use rustfs_utils::{crypto::base64_encode, path::trim_etag};
use crate::client::utils::base64_encode;
use rustfs_utils::path::trim_etag;
use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
pub struct UploadedPartRes {
crates/ecstore/src/client/api_remove.rs
+2 -1
View File
@@ -20,7 +20,7 @@
use bytes::Bytes;
use http::{HeaderMap, HeaderValue, Method, StatusCode};
use rustfs_utils::{HashAlgorithm, crypto::base64_encode};
use rustfs_utils::HashAlgorithm;
use s3s::S3ErrorCode;
use s3s::dto::ReplicationStatus;
use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
@@ -29,6 +29,7 @@ use std::{collections::HashMap, sync::Arc};
use time::OffsetDateTime;
use tokio::sync::mpsc::{self, Receiver, Sender};
use crate::client::utils::base64_encode;
use crate::client::{
api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response},
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
crates/ecstore/src/client/api_s3_datatypes.rs
+2 -2
View File
@@ -23,9 +23,9 @@ use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use time::OffsetDateTime;
use crate::checksum::ChecksumMode;
use crate::client::checksum::ChecksumMode;
use crate::client::transition_api::ObjectMultipartInfo;
use rustfs_utils::crypto::base64_decode;
use crate::client::utils::base64_decode;
use super::transition_api;
crates/ecstore/src/client/checksum.rs
+351
View File
@@ -0,0 +1,351 @@
#![allow(clippy::map_entry)]
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_assignments)]
#![allow(unused_must_use)]
#![allow(clippy::all)]
use lazy_static::lazy_static;
use rustfs_checksums::ChecksumAlgorithm;
use std::collections::HashMap;
use crate::client::utils::base64_decode;
use crate::client::utils::base64_encode;
use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
use crate::{disk::DiskAPI, store_api::GetObjectReader};
use s3s::header::{
X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
};
use enumset::{EnumSet, EnumSetType, enum_set};
#[derive(Debug, EnumSetType, Default)]
#[enumset(repr = "u8")]
pub enum ChecksumMode {
#[default]
ChecksumNone,
ChecksumSHA256,
ChecksumSHA1,
ChecksumCRC32,
ChecksumCRC32C,
ChecksumCRC64NVME,
ChecksumFullObject,
}
lazy_static! {
static ref C_ChecksumMask: EnumSet<ChecksumMode> = {
let mut s = EnumSet::all();
s.remove(ChecksumMode::ChecksumFullObject);
s
};
static ref C_ChecksumFullObjectCRC32: EnumSet<ChecksumMode> =
enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject);
static ref C_ChecksumFullObjectCRC32C: EnumSet<ChecksumMode> =
enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject);
}
const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme";
impl ChecksumMode {
//pub const CRC64_NVME_POLYNOMIAL: i64 = 0xad93d23594c93659;
pub fn base(&self) -> ChecksumMode {
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
match s.as_u8() {
1_u8 => ChecksumMode::ChecksumNone,
2_u8 => ChecksumMode::ChecksumSHA256,
4_u8 => ChecksumMode::ChecksumSHA1,
8_u8 => ChecksumMode::ChecksumCRC32,
16_u8 => ChecksumMode::ChecksumCRC32C,
32_u8 => ChecksumMode::ChecksumCRC64NVME,
_ => panic!("enum err."),
}
}
pub fn is(&self, t: ChecksumMode) -> bool {
*self & t == t
}
pub fn key(&self) -> String {
//match c & checksumMask {
match self {
ChecksumMode::ChecksumCRC32 => {
return X_AMZ_CHECKSUM_CRC32.to_string();
}
ChecksumMode::ChecksumCRC32C => {
return X_AMZ_CHECKSUM_CRC32C.to_string();
}
ChecksumMode::ChecksumSHA1 => {
return X_AMZ_CHECKSUM_SHA1.to_string();
}
ChecksumMode::ChecksumSHA256 => {
return X_AMZ_CHECKSUM_SHA256.to_string();
}
ChecksumMode::ChecksumCRC64NVME => {
return AMZ_CHECKSUM_CRC64NVME.to_string();
}
_ => {
return "".to_string();
}
}
}
pub fn can_composite(&self) -> bool {
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
match s.as_u8() {
2_u8 => true,
4_u8 => true,
8_u8 => true,
16_u8 => true,
_ => false,
}
}
pub fn can_merge_crc(&self) -> bool {
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
match s.as_u8() {
8_u8 => true,
16_u8 => true,
32_u8 => true,
_ => false,
}
}
pub fn full_object_requested(&self) -> bool {
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
match s.as_u8() {
//C_ChecksumFullObjectCRC32 as u8 => true,
//C_ChecksumFullObjectCRC32C as u8 => true,
32_u8 => true,
_ => false,
}
}
pub fn key_capitalized(&self) -> String {
self.key()
}
pub fn raw_byte_len(&self) -> usize {
let u = EnumSet::from(*self).intersection(*C_ChecksumMask).as_u8();
if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 {
4
} else if u == ChecksumMode::ChecksumSHA1 as u8 {
use sha1::Digest;
sha1::Sha1::output_size() as usize
} else if u == ChecksumMode::ChecksumSHA256 as u8 {
use sha2::Digest;
sha2::Sha256::output_size() as usize
} else if u == ChecksumMode::ChecksumCRC64NVME as u8 {
8
} else {
0
}
}
pub fn hasher(&self) -> Result<Box<dyn rustfs_checksums::http::HttpChecksum>, std::io::Error> {
match /*C_ChecksumMask & **/self {
ChecksumMode::ChecksumCRC32 => {
return Ok(ChecksumAlgorithm::Crc32.into_impl());
}
ChecksumMode::ChecksumCRC32C => {
return Ok(ChecksumAlgorithm::Crc32c.into_impl());
}
ChecksumMode::ChecksumSHA1 => {
return Ok(ChecksumAlgorithm::Sha1.into_impl());
}
ChecksumMode::ChecksumSHA256 => {
return Ok(ChecksumAlgorithm::Sha256.into_impl());
}
ChecksumMode::ChecksumCRC64NVME => {
return Ok(ChecksumAlgorithm::Crc64Nvme.into_impl());
}
_ => return Err(std::io::Error::other("unsupported checksum type")),
}
}
pub fn is_set(&self) -> bool {
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
s.len() == 1
}
pub fn set_default(&mut self, t: ChecksumMode) {
if !self.is_set() {
*self = t;
}
}
pub fn encode_to_string(&self, b: &[u8]) -> Result<String, std::io::Error> {
if !self.is_set() {
return Ok("".to_string());
}
let mut h = self.hasher()?;
h.update(b);
let hash = h.finalize();
Ok(base64_encode(hash.as_ref()))
}
pub fn to_string(&self) -> String {
//match c & checksumMask {
match self {
ChecksumMode::ChecksumCRC32 => {
return "CRC32".to_string();
}
ChecksumMode::ChecksumCRC32C => {
return "CRC32C".to_string();
}
ChecksumMode::ChecksumSHA1 => {
return "SHA1".to_string();
}
ChecksumMode::ChecksumSHA256 => {
return "SHA256".to_string();
}
ChecksumMode::ChecksumNone => {
return "".to_string();
}
ChecksumMode::ChecksumCRC64NVME => {
return "CRC64NVME".to_string();
}
_ => {
return "<invalid>".to_string();
}
}
}
// pub fn check_sum_reader(&self, r: GetObjectReader) -> Result<Checksum, std::io::Error> {
// let mut h = self.hasher()?;
// Ok(Checksum::new(self.clone(), h.sum().as_bytes()))
// }
// pub fn check_sum_bytes(&self, b: &[u8]) -> Result<Checksum, std::io::Error> {
// let mut h = self.hasher()?;
// Ok(Checksum::new(self.clone(), h.sum().as_bytes()))
// }
pub fn composite_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
if !self.can_composite() {
return Err(std::io::Error::other("cannot do composite checksum"));
}
p.sort_by(|i, j| {
if i.part_num < j.part_num {
std::cmp::Ordering::Less
} else if i.part_num > j.part_num {
std::cmp::Ordering::Greater
} else {
std::cmp::Ordering::Equal
}
});
let c = self.base();
let crc_bytes = Vec::<u8>::with_capacity(p.len() * self.raw_byte_len() as usize);
let mut h = self.hasher()?;
h.update(crc_bytes.as_ref());
let hash = h.finalize();
Ok(Checksum {
checksum_type: self.clone(),
r: hash.as_ref().to_vec(),
computed: false,
})
}
pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
todo!();
}
}
#[derive(Default)]
pub struct Checksum {
checksum_type: ChecksumMode,
r: Vec<u8>,
computed: bool,
}
#[allow(dead_code)]
impl Checksum {
fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
if t.is_set() && b.len() == t.raw_byte_len() {
return Checksum {
checksum_type: t,
r: b.to_vec(),
computed: false,
};
}
Checksum::default()
}
#[allow(dead_code)]
fn new_checksum_string(t: ChecksumMode, s: &str) -> Result<Checksum, std::io::Error> {
let b = match base64_decode(s.as_bytes()) {
Ok(b) => b,
Err(err) => return Err(std::io::Error::other(err.to_string())),
};
if t.is_set() && b.len() == t.raw_byte_len() {
return Ok(Checksum {
checksum_type: t,
r: b,
computed: false,
});
}
Ok(Checksum::default())
}
fn is_set(&self) -> bool {
self.checksum_type.is_set() && self.r.len() == self.checksum_type.raw_byte_len()
}
fn encoded(&self) -> String {
if !self.is_set() {
return "".to_string();
}
base64_encode(&self.r)
}
#[allow(dead_code)]
fn raw(&self) -> Option<Vec<u8>> {
if !self.is_set() {
return None;
}
Some(self.r.clone())
}
}
pub fn add_auto_checksum_headers(opts: &mut PutObjectOptions) {
opts.user_metadata
.insert("X-Amz-Checksum-Algorithm".to_string(), opts.auto_checksum.to_string());
if opts.auto_checksum.full_object_requested() {
opts.user_metadata
.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
}
}
pub fn apply_auto_checksum(opts: &mut PutObjectOptions, all_parts: &mut [ObjectPart]) -> Result<(), std::io::Error> {
if opts.auto_checksum.can_composite() && !opts.auto_checksum.is(ChecksumMode::ChecksumFullObject) {
let crc = opts.auto_checksum.composite_checksum(all_parts)?;
opts.user_metadata = {
let mut hm = HashMap::new();
hm.insert(opts.auto_checksum.key(), crc.encoded());
hm
}
} else if opts.auto_checksum.can_merge_crc() {
let crc = opts.auto_checksum.full_object_checksum(all_parts)?;
opts.user_metadata = {
let mut hm = HashMap::new();
hm.insert(opts.auto_checksum.key_capitalized(), crc.encoded());
hm.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
hm
}
}
Ok(())
}
crates/ecstore/src/client/mod.rs
+1
View File
@@ -30,6 +30,7 @@ pub mod api_restore;
pub mod api_s3_datatypes;
pub mod api_stat;
pub mod bucket_cache;
pub mod checksum;
pub mod constants;
pub mod credentials;
pub mod object_api_utils;
crates/ecstore/src/client/transition_api.rs
+1 -1
View File
@@ -61,7 +61,7 @@ use crate::client::{
constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER},
credentials::{CredContext, Credentials, SignatureType, Static},
};
use crate::{checksum::ChecksumMode, store_api::GetObjectReader};
use crate::{client::checksum::ChecksumMode, store_api::GetObjectReader};
use rustfs_rio::HashReader;
use rustfs_utils::{
net::get_endpoint_url,
crates/ecstore/src/client/utils.rs
+8
View File
@@ -90,3 +90,11 @@ pub fn is_rustfs_header(header_key: &str) -> bool {
pub fn is_minio_header(header_key: &str) -> bool {
header_key.to_lowercase().starts_with("x-minio-")
}
pub fn base64_encode(input: &[u8]) -> String {
base64_simd::URL_SAFE_NO_PAD.encode_to_string(input)
}
pub fn base64_decode(input: &[u8]) -> Result<Vec<u8>, base64_simd::Error> {
base64_simd::URL_SAFE_NO_PAD.decode_to_vec(input)
}
crates/ecstore/src/erasure_coding/decode.rs
+4
View File
@@ -301,6 +301,10 @@ impl Erasure {
written += n;
}
if ret_err.is_some() {
return (written, ret_err);
}
if written < length {
ret_err = Some(Error::LessData.into());
}
crates/ecstore/src/erasure_coding/encode.rs
+3 -1
View File
@@ -145,7 +145,9 @@ impl Erasure {
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
}
}
Ok(_) => break,
Ok(_) => {
break;
}
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
break;
}
crates/ecstore/src/erasure_coding/erasure.rs
+7 -1
View File
@@ -468,15 +468,21 @@ impl Erasure {
let mut buf = vec![0u8; block_size];
match rustfs_utils::read_full(&mut *reader, &mut buf).await {
Ok(n) if n > 0 => {
warn!("encode_stream_callback_async read n={}", n);
total += n;
let res = self.encode_data(&buf[..n]);
on_block(res).await?
}
Ok(_) => break,
Ok(_) => {
warn!("encode_stream_callback_async read unexpected ok");
break;
}
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
warn!("encode_stream_callback_async read unexpected eof");
break;
}
Err(e) => {
warn!("encode_stream_callback_async read error={:?}", e);
on_block(Err(e)).await?;
break;
}
crates/ecstore/src/lib.rs
+1 -1
View File
@@ -44,7 +44,7 @@ mod store_init;
pub mod store_list_objects;
pub mod store_utils;
pub mod checksum;
// pub mod checksum;
pub mod client;
pub mod event;
pub mod event_notification;
crates/ecstore/src/pools.rs
+3 -1
View File
@@ -1262,6 +1262,8 @@ impl ECStore {
parts[i] = CompletePart {
part_num: pi.part_num,
etag: pi.etag,
..Default::default()
};
}
@@ -1289,7 +1291,7 @@ impl ECStore {
}
let reader = BufReader::new(rd.stream);
let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, false)?;
let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, None, false)?;
let mut data = PutObjReader::new(hrd);
if let Err(err) = self
crates/ecstore/src/rebalance.rs
+2 -1
View File
@@ -979,6 +979,7 @@ impl ECStore {
parts[i] = CompletePart {
part_num: pi.part_num,
etag: pi.etag,
..Default::default()
};
}
@@ -1005,7 +1006,7 @@ impl ECStore {
}
let reader = BufReader::new(rd.stream);
let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, false)?;
let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, None, false)?;
let mut data = PutObjReader::new(hrd);
if let Err(err) = self
crates/ecstore/src/set_disk.rs
+114 -83
View File
@@ -72,13 +72,13 @@ use rustfs_filemeta::{
};
use rustfs_lock::fast_lock::types::LockResult;
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
use rustfs_rio::{EtagResolvable, HashReader, TryGetIndex as _, WarpReader};
use rustfs_rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _, WarpReader};
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
use rustfs_utils::http::headers::RESERVED_METADATA_PREFIX_LOWER;
use rustfs_utils::{
HashAlgorithm,
crypto::{base64_decode, base64_encode, hex},
crypto::hex,
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf},
};
use rustfs_workers::workers::Workers;
@@ -158,10 +158,7 @@ impl SetDisks {
LockResult::Conflict {
current_owner,
current_mode,
} => format!(
"{mode} lock conflicted on {bucket}/{object}: held by {current_owner} as {:?}",
current_mode
),
} => format!("{mode} lock conflicted on {bucket}/{object}: held by {current_owner} as {current_mode:?}"),
LockResult::Acquired => format!("unexpected lock state while acquiring {mode} lock on {bucket}/{object}"),
}
}
@@ -922,9 +919,8 @@ impl SetDisks {
}
fn get_upload_id_dir(bucket: &str, object: &str, upload_id: &str) -> String {
// warn!("get_upload_id_dir upload_id {:?}", upload_id);
let upload_uuid = base64_decode(upload_id.as_bytes())
let upload_uuid = base64_simd::URL_SAFE_NO_PAD
.decode_to_vec(upload_id.as_bytes())
.and_then(|v| {
String::from_utf8(v).map_or(Ok(upload_id.to_owned()), |v| {
let parts: Vec<_> = v.splitn(2, '.').collect();
@@ -2950,6 +2946,7 @@ impl SetDisks {
part.mod_time,
part.actual_size,
part.index.clone(),
part.checksums.clone(),
);
if is_inline_buffer {
if let Some(writer) = writers[index].take() {
@@ -3528,9 +3525,9 @@ impl ObjectIO for SetDisks {
// }
if object_info.size == 0 {
if let Some(rs) = range {
let _ = rs.get_offset_length(object_info.size)?;
}
// if let Some(rs) = range {
// let _ = rs.get_offset_length(object_info.size)?;
// }
let reader = GetObjectReader {
stream: Box::new(Cursor::new(Vec::new())),
@@ -3712,7 +3709,7 @@ impl ObjectIO for SetDisks {
let stream = mem::replace(
&mut data.stream,
HashReader::new(Box::new(WarpReader::new(Cursor::new(Vec::new()))), 0, 0, None, false)?,
HashReader::new(Box::new(WarpReader::new(Cursor::new(Vec::new()))), 0, 0, None, None, false)?,
);
let (reader, w_size) = match Arc::new(erasure).encode(stream, &mut writers, write_quorum).await {
@@ -3729,7 +3726,12 @@ impl ObjectIO for SetDisks {
// }
if (w_size as i64) < data.size() {
return Err(Error::other("put_object write size < data.size()"));
warn!("put_object write size < data.size(), w_size={}, data.size={}", w_size, data.size());
return Err(Error::other(format!(
"put_object write size < data.size(), w_size={}, data.size={}",
w_size,
data.size()
)));
}
if user_defined.contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression")) {
@@ -3756,31 +3758,42 @@ impl ObjectIO for SetDisks {
}
}
if fi.checksum.is_none() {
if let Some(content_hash) = data.as_hash_reader().content_hash() {
fi.checksum = Some(content_hash.to_bytes(&[]));
}
}
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS) {
if sc == storageclass::STANDARD {
let _ = user_defined.remove(AMZ_STORAGE_CLASS);
}
}
let now = OffsetDateTime::now_utc();
let mod_time = if let Some(mod_time) = opts.mod_time {
Some(mod_time)
} else {
Some(OffsetDateTime::now_utc())
};
for (i, fi) in parts_metadatas.iter_mut().enumerate() {
fi.metadata = user_defined.clone();
for (i, pfi) in parts_metadatas.iter_mut().enumerate() {
pfi.metadata = user_defined.clone();
if is_inline_buffer {
if let Some(writer) = writers[i].take() {
fi.data = Some(writer.into_inline_data().map(bytes::Bytes::from).unwrap_or_default());
pfi.data = Some(writer.into_inline_data().map(bytes::Bytes::from).unwrap_or_default());
}
fi.set_inline_data();
pfi.set_inline_data();
}
fi.mod_time = Some(now);
fi.size = w_size as i64;
fi.versioned = opts.versioned || opts.version_suspended;
fi.add_object_part(1, etag.clone(), w_size, fi.mod_time, actual_size, index_op.clone());
pfi.mod_time = mod_time;
pfi.size = w_size as i64;
pfi.versioned = opts.versioned || opts.version_suspended;
pfi.add_object_part(1, etag.clone(), w_size, mod_time, actual_size, index_op.clone(), None);
pfi.checksum = fi.checksum.clone();
if opts.data_movement {
fi.set_data_moved();
pfi.set_data_moved();
}
}
@@ -3815,7 +3828,8 @@ impl ObjectIO for SetDisks {
fi.replication_state_internal = Some(opts.put_replication_state());
// TODO: version support
fi.is_latest = true;
Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended))
}
}
@@ -4430,8 +4444,6 @@ impl StorageAPI for SetDisks {
.await
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
// warn!("get object_info fi {:?}", &fi);
let oi = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
Ok(oi)
@@ -4759,6 +4771,11 @@ impl StorageAPI for SetDisks {
uploaded_parts.push(CompletePart {
part_num: p_info.part_num,
etag: p_info.etag,
checksum_crc32: None,
checksum_crc32c: None,
checksum_sha1: None,
checksum_sha256: None,
checksum_crc64nvme: None,
});
}
if let Err(err) = self.complete_multipart_upload(bucket, object, &res.upload_id, uploaded_parts, &ObjectOptions {
@@ -4834,64 +4851,24 @@ impl StorageAPI for SetDisks {
let write_quorum = fi.write_quorum(self.default_write_quorum());
let disks = self.disks.read().await;
if let Some(checksum) = fi.metadata.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM)
&& !checksum.is_empty()
&& data
.as_hash_reader()
.content_crc_type()
.is_none_or(|v| v.to_string() != *checksum)
{
return Err(Error::other(format!("checksum mismatch: {checksum}")));
}
let disks = self.disks.read().await.clone();
let disks = disks.clone();
let shuffle_disks = Self::shuffle_disks(&disks, &fi.erasure.distribution);
let part_suffix = format!("part.{part_id}");
let tmp_part = format!("{}x{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp());
let tmp_part_path = Arc::new(format!("{tmp_part}/{part_suffix}"));
// let mut writers = Vec::with_capacity(disks.len());
// let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
// let shared_size = erasure.shard_size(erasure.block_size);
// let futures = disks.iter().map(|disk| {
// let disk = disk.clone();
// let tmp_part_path = tmp_part_path.clone();
// tokio::spawn(async move {
// if let Some(disk) = disk {
// // let writer = disk.append_file(RUSTFS_META_TMP_BUCKET, &tmp_part_path).await?;
// // let filewriter = disk
// // .create_file("", RUSTFS_META_TMP_BUCKET, &tmp_part_path, data.content_length)
// // .await?;
// match new_bitrot_filewriter(
// disk.clone(),
// RUSTFS_META_TMP_BUCKET,
// &tmp_part_path,
// false,
// DEFAULT_BITROT_ALGO,
// shared_size,
// )
// .await
// {
// Ok(writer) => Ok(Some(writer)),
// Err(e) => Err(e),
// }
// } else {
// Ok(None)
// }
// })
// });
// for x in join_all(futures).await {
// let x = x??;
// writers.push(x);
// }
// let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
// let stream = replace(&mut data.stream, Box::new(empty()));
// let etag_stream = EtagReader::new(stream);
// let (w_size, mut etag) = Arc::new(erasure)
// .encode(etag_stream, &mut writers, data.content_length, write_quorum)
// .await?;
// if let Err(err) = close_bitrot_writers(&mut writers).await {
// error!("close_bitrot_writers err {:?}", err);
// }
let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
let mut writers = Vec::with_capacity(shuffle_disks.len());
@@ -4944,7 +4921,7 @@ impl StorageAPI for SetDisks {
let stream = mem::replace(
&mut data.stream,
HashReader::new(Box::new(WarpReader::new(Cursor::new(Vec::new()))), 0, 0, None, false)?,
HashReader::new(Box::new(WarpReader::new(Cursor::new(Vec::new()))), 0, 0, None, None, false)?,
);
let (reader, w_size) = Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?; // TODO: 出错，删除临时目录
@@ -4952,7 +4929,12 @@ impl StorageAPI for SetDisks {
let _ = mem::replace(&mut data.stream, reader);
if (w_size as i64) < data.size() {
return Err(Error::other("put_object_part write size < data.size()"));
warn!("put_object_part write size < data.size(), w_size={}, data.size={}", w_size, data.size());
return Err(Error::other(format!(
"put_object_part write size < data.size(), w_size={}, data.size={}",
w_size,
data.size()
)));
}
let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec());
@@ -5227,7 +5209,8 @@ impl StorageAPI for SetDisks {
uploads.push(MultipartInfo {
bucket: bucket.to_owned(),
object: object.to_owned(),
upload_id: base64_encode(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_id).as_bytes()),
upload_id: base64_simd::URL_SAFE_NO_PAD
.encode_to_string(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_id).as_bytes()),
initiated: Some(start_time),
..Default::default()
});
@@ -5348,6 +5331,14 @@ impl StorageAPI for SetDisks {
}
}
if let Some(checksum) = &opts.want_checksum {
user_defined.insert(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM.to_string(), checksum.checksum_type.to_string());
user_defined.insert(
rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE.to_string(),
checksum.checksum_type.obj_type().to_string(),
);
}
let (shuffle_disks, mut parts_metadatas) = Self::shuffle_disks_and_parts_metadata(&disks, &parts_metadata, &fi);
let mod_time = opts.mod_time.unwrap_or(OffsetDateTime::now_utc());
@@ -5362,7 +5353,8 @@ impl StorageAPI for SetDisks {
let upload_uuid = format!("{}x{}", Uuid::new_v4(), mod_time.unix_timestamp_nanos());
let upload_id = base64_encode(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_uuid).as_bytes());
let upload_id = base64_simd::URL_SAFE_NO_PAD
.encode_to_string(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_uuid).as_bytes());
let upload_path = Self::get_upload_id_dir(bucket, object, upload_uuid.as_str());
@@ -5379,7 +5371,11 @@ impl StorageAPI for SetDisks {
// evalDisks
Ok(MultipartUploadResult { upload_id })
Ok(MultipartUploadResult {
upload_id,
checksum_algo: user_defined.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM).cloned(),
checksum_type: user_defined.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE).cloned(),
})
}
#[tracing::instrument(skip(self))]
@@ -5467,6 +5463,25 @@ impl StorageAPI for SetDisks {
return Err(Error::other("part result number err"));
}
if let Some(cs) = fi.metadata.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM) {
let Some(checksum_type) = fi.metadata.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE) else {
return Err(Error::other("checksum type not found"));
};
if opts.want_checksum.is_some()
&& !opts.want_checksum.as_ref().is_some_and(|v| {
v.checksum_type
.is(rustfs_rio::ChecksumType::from_string_with_obj_type(cs, checksum_type))
})
{
return Err(Error::other(format!(
"checksum type mismatch, got {:?}, want {:?}",
opts.want_checksum.as_ref().unwrap(),
rustfs_rio::ChecksumType::from_string_with_obj_type(cs, checksum_type)
)));
}
}
for (i, part) in object_parts.iter().enumerate() {
if let Some(err) = &part.error {
error!("complete_multipart_upload part error: {:?}", &err);
@@ -5487,6 +5502,7 @@ impl StorageAPI for SetDisks {
part.mod_time,
part.actual_size,
part.index.clone(),
part.checksums.clone(),
);
}
@@ -6422,10 +6438,20 @@ mod tests {
CompletePart {
part_num: 1,
etag: Some("d41d8cd98f00b204e9800998ecf8427e".to_string()),
checksum_crc32: None,
checksum_crc32c: None,
checksum_sha1: None,
checksum_sha256: None,
checksum_crc64nvme: None,
},
CompletePart {
part_num: 2,
etag: Some("098f6bcd4621d373cade4e832627b4f6".to_string()),
checksum_crc32: None,
checksum_crc32c: None,
checksum_sha1: None,
checksum_sha256: None,
checksum_crc64nvme: None,
},
];
@@ -6442,6 +6468,11 @@ mod tests {
let single_part = vec![CompletePart {
part_num: 1,
etag: Some("d41d8cd98f00b204e9800998ecf8427e".to_string()),
checksum_crc32: None,
checksum_crc32c: None,
checksum_sha1: None,
checksum_sha256: None,
checksum_crc64nvme: None,
}];
let single_result = get_complete_multipart_md5(&single_part);
assert!(single_result.ends_with("-1"));
crates/ecstore/src/store.rs
+2 -3
View File
@@ -59,7 +59,6 @@ use rustfs_common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_
use rustfs_common::heal_channel::{HealItemType, HealOpts};
use rustfs_filemeta::FileInfo;
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_utils::crypto::base64_decode;
use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf};
use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
use std::cmp::Ordering;
@@ -2421,7 +2420,7 @@ fn check_list_multipart_args(
}
}
if let Err(_e) = base64_decode(upload_id_marker.as_bytes()) {
if let Err(_e) = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id_marker.as_bytes()) {
return Err(StorageError::MalformedUploadID(upload_id_marker.to_owned()));
}
}
@@ -2448,7 +2447,7 @@ fn check_new_multipart_args(bucket: &str, object: &str) -> Result<()> {
}
fn check_multipart_object_args(bucket: &str, object: &str, upload_id: &str) -> Result<()> {
if let Err(e) = base64_decode(upload_id.as_bytes()) {
if let Err(e) = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id.as_bytes()) {
return Err(StorageError::MalformedUploadID(format!("{bucket}/{object}-{upload_id},err:{e}")));
};
check_object_args(bucket, object)
crates/ecstore/src/store_api.rs
+59 -9
View File
@@ -13,9 +13,6 @@
// limitations under the License.
use crate::bucket::metadata_sys::get_versioning_config;
use crate::bucket::replication::REPLICATION_RESET;
use crate::bucket::replication::REPLICATION_STATUS;
use crate::bucket::replication::{ReplicateDecision, replication_statuses_map, version_purge_statuses_map};
use crate::bucket::versioning::VersioningApi as _;
use crate::disk::DiskStore;
use crate::error::{Error, Result};
@@ -25,12 +22,15 @@ use crate::{
bucket::lifecycle::lifecycle::ExpirationOptions,
bucket::lifecycle::{bucket_lifecycle_ops::TransitionedObject, lifecycle::TransitionOptions},
};
use bytes::Bytes;
use http::{HeaderMap, HeaderValue};
use rustfs_common::heal_channel::HealOpts;
use rustfs_filemeta::{
FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, ReplicationState, ReplicationStatusType, VersionPurgeStatusType,
FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, REPLICATION_RESET, REPLICATION_STATUS, ReplicateDecision, ReplicationState,
ReplicationStatusType, VersionPurgeStatusType, replication_statuses_map, version_purge_statuses_map,
};
use rustfs_madmin::heal_commands::HealResultItem;
use rustfs_rio::Checksum;
use rustfs_rio::{DecompressReader, HashReader, LimitReader, WarpReader};
use rustfs_utils::CompressionAlgorithm;
use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX_LOWER};
@@ -92,11 +92,28 @@ impl PutObjReader {
PutObjReader { stream }
}
pub fn as_hash_reader(&self) -> &HashReader {
&self.stream
}
pub fn from_vec(data: Vec<u8>) -> Self {
use sha2::{Digest, Sha256};
let content_length = data.len() as i64;
let sha256hex = if content_length > 0 {
Some(hex_simd::encode_to_string(Sha256::digest(&data), hex_simd::AsciiCase::Lower))
} else {
None
};
PutObjReader {
stream: HashReader::new(Box::new(WarpReader::new(Cursor::new(data))), content_length, content_length, None, false)
.unwrap(),
stream: HashReader::new(
Box::new(WarpReader::new(Cursor::new(data))),
content_length,
content_length,
None,
sha256hex,
false,
)
.unwrap(),
}
}
@@ -374,6 +391,8 @@ pub struct ObjectOptions {
pub lifecycle_audit_event: LcAuditEvent,
pub eval_metadata: Option<HashMap<String, String>>,
pub want_checksum: Option<Checksum>,
}
impl ObjectOptions {
@@ -456,6 +475,8 @@ pub struct BucketInfo {
#[derive(Debug, Default, Clone)]
pub struct MultipartUploadResult {
pub upload_id: String,
pub checksum_algo: Option<String>,
pub checksum_type: Option<String>,
}
#[derive(Debug, Default, Clone)]
@@ -471,13 +492,24 @@ pub struct PartInfo {
pub struct CompletePart {
pub part_num: usize,
pub etag: Option<String>,
// pub size: Option<usize>,
pub checksum_crc32: Option<String>,
pub checksum_crc32c: Option<String>,
pub checksum_sha1: Option<String>,
pub checksum_sha256: Option<String>,
pub checksum_crc64nvme: Option<String>,
}
impl From<s3s::dto::CompletedPart> for CompletePart {
fn from(value: s3s::dto::CompletedPart) -> Self {
Self {
part_num: value.part_number.unwrap_or_default() as usize,
etag: value.e_tag.map(|e| e.value().to_owned()),
etag: value.e_tag.map(|v| v.value().to_owned()),
checksum_crc32: value.checksum_crc32,
checksum_crc32c: value.checksum_crc32c,
checksum_sha1: value.checksum_sha1,
checksum_sha256: value.checksum_sha256,
checksum_crc64nvme: value.checksum_crc64nvme,
}
}
}
@@ -517,7 +549,7 @@ pub struct ObjectInfo {
pub version_purge_status_internal: Option<String>,
pub version_purge_status: VersionPurgeStatusType,
pub replication_decision: String,
pub checksum: Vec<u8>,
pub checksum: Option<Bytes>,
}
impl Clone for ObjectInfo {
@@ -554,7 +586,7 @@ impl Clone for ObjectInfo {
version_purge_status_internal: self.version_purge_status_internal.clone(),
version_purge_status: self.version_purge_status.clone(),
replication_decision: self.replication_decision.clone(),
checksum: Default::default(),
checksum: self.checksum.clone(),
expires: self.expires,
}
}
@@ -694,6 +726,7 @@ impl ObjectInfo {
inlined,
user_defined: metadata,
transitioned_object,
checksum: fi.checksum.clone(),
..Default::default()
}
}
@@ -884,6 +917,23 @@ impl ObjectInfo {
..Default::default()
}
}
pub fn decrypt_checksums(&self, part: usize, _headers: &HeaderMap) -> Result<(HashMap<String, String>, bool)> {
if part > 0 {
if let Some(checksums) = self.parts.iter().find(|p| p.number == part).and_then(|p| p.checksums.clone()) {
return Ok((checksums, true));
}
}
// TODO: decrypt checksums
if let Some(data) = &self.checksum {
let (checksums, is_multipart) = rustfs_rio::read_checksums(data.as_ref(), 0);
return Ok((checksums, is_multipart));
}
Ok((HashMap::new(), false))
}
}
#[derive(Debug, Default)]